Update GenSec tool to always create output file without check file content.
[efi/basetools/.git] / gcc / mingw-gcc-build.py
1 #!/usr/bin/env python
2
3 #
4 # Automation of instructions from:
5 #   http://mingw-w64.svn.sourceforge.net/viewvc/mingw-w64/trunk/mingw-w64-doc/
6 #     howto-build/mingw-w64-howto-build.txt?revision=216&view=markup
7 #
8
9 from optparse import OptionParser
10 import os
11 import shutil
12 import subprocess
13 import sys
14 import tarfile
15 import urllib
16 import urlparse
17 try:
18     from hashlib import md5
19 except Exception:
20     from md5 import md5
21
22 #
23 # Version and Copyright
24 #
25 VersionNumber = "0.01"
26 __version__ = "%prog Version " + VersionNumber
27 __copyright__ = "Copyright (c) 2008, Intel Corporation.  All rights reserved."
28
29 class Config:
30     """class Config
31
32     Stores the configuration options for the rest of the script.
33
34     Handles the command line options, and allows the code within
35     the script to easily interact with the 'config' requested by
36     the user.
37     """
38
39     def __init__(self):
40         self.base_dir = os.getcwd()
41         (self.options, self.args) = self.CheckOptions()
42         self.__init_dirs__()
43
44     def CheckOptions(self):
45         Parser = \
46             OptionParser(
47                 description=__copyright__,
48                 version=__version__,
49                 prog="mingw-gcc-build",
50                 usage="%prog [options] [target]"
51                 )
52         Parser.add_option(
53             "--arch",
54             action = "store", type = "string",
55             default = '',
56             dest = "arch",
57             help = "Processor architecture to build gcc for."
58             )
59         Parser.add_option(
60             "--src-dir",
61             action = "store", type = "string", dest = "src_dir",
62             default = os.path.join(self.base_dir, 'src'),
63             help = "Directory to download/extract binutils/gcc sources"
64             )
65         Parser.add_option(
66             "--build-dir",
67             action = "store", type = "string", dest = "build_dir",
68             default = os.path.join(self.base_dir, 'build'),
69             help = "Directory to download/extract binutils/gcc sources"
70             )
71         Parser.add_option(
72             "--prefix",
73             action = "store", type = "string", dest = "prefix",
74             default = os.path.join(self.base_dir, 'install'),
75             help = "Prefix to install binutils/gcc into"
76             )
77         Parser.add_option(
78             "--symlinks",
79             action = "store", type = "string", dest = "symlinks",
80             default = os.path.join(self.base_dir, 'symlinks'),
81             help = "Directory to create binutils/gcc symbolic links into."
82             )
83         Parser.add_option(
84             "-v", "--verbose",
85             action="store_true",
86             type=None, help="Print verbose messages"
87             )
88
89         (Opt, Args) = Parser.parse_args()
90
91         self.arch = Opt.arch.lower()
92         allowedArchs = ('ia32', 'x64', 'ipf')
93         if self.arch not in allowedArchs:
94             Parser.error(
95                 'Please use --arch to specify one of: %s' %
96                     ', '.join(allowedArchs)
97                 )
98         self.target_arch = {'ia32': 'i686', 'x64': 'x86_64', 'ipf': 'ia64'}[self.arch]
99         self.target_sys = {'ia32': 'pc', 'x64': 'pc', 'ipf': 'pc'}[self.arch]
100         self.target_bin = {'ia32': 'mingw32', 'x64': 'mingw32', 'ipf': 'elf'}[self.arch]
101         self.target_combo = '-'.join((self.target_arch, self.target_sys, self.target_bin))
102
103         return (Opt, Args)
104
105     def __init_dirs__(self):
106         self.src_dir = os.path.realpath(os.path.expanduser(self.options.src_dir))
107         self.build_dir = os.path.realpath(os.path.expanduser(self.options.build_dir))
108         self.prefix = os.path.realpath(os.path.expanduser(self.options.prefix))
109         self.symlinks = os.path.realpath(os.path.expanduser(self.options.symlinks))
110
111     def IsConfigOk(self):
112                 
113         print "Current directory:"
114         print "   ", self.base_dir
115         print "Sources download/extraction:", self.Relative(self.src_dir)
116         print "Build directory            :", self.Relative(self.build_dir)
117         print "Prefix (install) directory :", self.Relative(self.prefix)
118         print "Create symlinks directory  :", self.Relative(self.symlinks)
119         print
120         answer = raw_input("Is this configuration ok? (default = no): ")
121         if (answer.lower() not in ('y', 'yes')):
122             print
123             print "Please try using --help and then change the configuration."
124             return False
125
126         if self.arch.lower() == 'ipf':
127             print
128             print 'Please note that the IPF compiler built by this script has'
129             print 'not yet been validated!'
130             print
131             answer = raw_input("Are you sure you want to build it? (default = no): ")
132             if (answer.lower() not in ('y', 'yes')):
133                 print
134                 print "Please try using --help and then change the configuration."
135                 return False
136
137         print
138         return True
139
140     def Relative(self, path):
141         if path.startswith(self.base_dir):
142             return '.' + path[len(self.base_dir):]
143         return path
144
145     def MakeDirs(self):
146         for path in (self.src_dir, self.build_dir,self.prefix, self.symlinks):
147             if not os.path.exists(path):
148                 os.makedirs(path)
149
150 class SourceFiles:
151     """class SourceFiles
152
153     Handles the downloading of source files used by the script.
154     """
155
156     def __init__(self, config):
157         self.config = config
158         self.source_files = self.source_files[config.arch]
159
160     source_files_common = {
161         'binutils': {
162             'url': 'http://www.kernel.org/pub/linux/devel/binutils/' + \
163                    'binutils-$version.tar.bz2',
164             'version': '2.18.50.0.5',
165             'md5': 'daee18dbbf0a6ccfc186141bee18bf62',
166             },
167         }
168
169     source_files_x64 = {
170         'gcc': {
171             'url': 'http://gcc-ca.internet.bs/releases/' + \
172                    'gcc-$version/gcc-$version.tar.bz2',
173             'version': '4.3.0',
174             'md5': '197ed8468b38db1d3481c3111691d85b',
175             },
176         'mingw_hdr': {
177             'url': 'http://superb-west.dl.sourceforge.net/sourceforge/' + \
178                    'mingw-w64/mingw-w64-snapshot-$version.tar.bz2',
179             'extract-dir': os.path.join('trunk', 'mingw-w64-headers'),
180             'version': '20080310',
181             'md5': '235b2d15c2411f7d213c0c0977b2162f',
182             },
183         }
184
185     source_files_ia32 = {
186         'gcc': {
187             'url': 'http://superb-east.dl.sourceforge.net/sourceforge/' + \
188                    'mingw/gcc-$version-mingw-$minor_version-src.tar.gz',
189             'version': '4.3.0',
190             'minor_version': 'alpha-20080403',
191             'extract-dir': 'gcc-$version',
192             'md5': '27961d80e304f4ef32c980833c6e8e44',
193             'configure-params': ('--with-gnu-as', '--with-gnu-ld', '--with-newlib',
194                                  '--verbose', '--disable-libssp', '--disable-nls',
195                                  '--enable-languages=c,c++'
196                                 )
197             },
198         'mingw_hdr': {
199             'url': 'http://superb-west.dl.sourceforge.net/sourceforge/' + \
200                    'mingw/mingw-runtime-$version-src.tar.gz',
201             'extract-dir': 'mingw-runtime-$version',
202             'version': '3.14',
203             'md5': '7d049a8331efcfe34600c0cda6934ac6',
204             },
205         }
206
207     source_files_ipf = source_files_x64.copy()
208     source_files_ipf['gcc']['configure-params'] = (
209         '--with-gnu-as', '--with-gnu-ld', '--with-newlib',
210         '--verbose', '--disable-libssp', '--disable-nls',
211         '--enable-languages=c,c++'
212         )
213
214     source_files = {
215         'ia32': [source_files_common, source_files_ia32],
216         'x64': [source_files_common, source_files_x64],
217         'ipf': [source_files_common, source_files_ipf],
218         }
219
220     for arch in source_files:
221         mergedSourceFiles = {}
222         for source_files_dict in source_files[arch]:
223             mergedSourceFiles.update(source_files_dict)
224         for downloadItem in mergedSourceFiles:
225             fdata = mergedSourceFiles[downloadItem]
226             fdata['filename'] = fdata['url'].split('/')[-1]
227             if 'extract-dir' not in fdata:
228                 for ext in ('.tar.gz', '.tar.bz2', '.zip'):
229                     if fdata['filename'].endswith(ext):
230                         fdata['extract-dir'] = fdata['filename'][:-len(ext)]
231                         break
232             replaceables = ('extract-dir', 'filename', 'url')
233             for replaceItem in fdata:
234                 if replaceItem in replaceables: continue
235                 if type(fdata[replaceItem]) != str: continue
236                 for replaceable in replaceables:
237                     if type(fdata[replaceable]) != str: continue
238                     if replaceable in fdata:
239                         fdata[replaceable] = \
240                             fdata[replaceable].replace(
241                                 '$' + replaceItem,
242                                 fdata[replaceItem]
243                                 )
244         source_files[arch] = mergedSourceFiles
245     #print 'source_files:', source_files
246
247     def GetAll(self):
248
249         def progress(received, blockSize, fileSize):
250             if fileSize < 0: return
251             wDots = (100 * received * blockSize) / fileSize / 10
252             if wDots > self.dots:
253                 for i in range(wDots - self.dots):
254                     print '.',
255                     sys.stdout.flush()
256                     self.dots += 1
257
258         maxRetries = 3
259         for (fname, fdata) in self.source_files.items():
260             for retries in range(maxRetries):
261                 try:
262                     self.dots = 0
263                     local_file = os.path.join(self.config.src_dir, fdata['filename'])
264                     url = fdata['url']
265                     print 'Downloading %s:' % fname,
266                     if retries > 0:
267                         print '(retry)',
268                     sys.stdout.flush()
269
270                     completed = False
271                     if os.path.exists(local_file):
272                         md5_pass = self.checkHash(fdata)
273                         if md5_pass:
274                             print '[md5 match]',
275                         else:
276                             print '[md5 mismatch]',
277                         sys.stdout.flush()
278                         completed = md5_pass
279
280                     if not completed:
281                         urllib.urlretrieve(url, local_file, progress)
282
283                     #
284                     # BUGBUG: Suggest proxy to user if download fails.
285                     #
286                     # export http_proxy=http://proxyservername.mycompany.com:911
287                     # export ftp_proxy=http://proxyservername.mycompany.com:911
288
289                     if not completed and os.path.exists(local_file):
290                         md5_pass = self.checkHash(fdata)
291                         if md5_pass:
292                             print '[md5 match]',
293                         else:
294                             print '[md5 mismatch]',
295                         sys.stdout.flush()
296                         completed = md5_pass
297
298                     if completed:
299                         print '[done]'
300                         break
301                     else:
302                         print '[failed]'
303                         raise Exception()
304                 
305                 except KeyboardInterrupt:
306                     print '[KeyboardInterrupt]'
307                     return False
308
309                 except:
310                     pass
311
312             if not completed: return False
313
314         return True
315
316     def checkHash(self, fdata):
317         local_file = os.path.join(self.config.src_dir, fdata['filename'])
318         expect_md5 = fdata['md5']
319         data = open(local_file).read()
320         md5sum = md5()
321         md5sum.update(data)
322         return md5sum.hexdigest().lower() == expect_md5.lower()
323
324     def GetModules(self):
325         return self.source_files.keys()
326
327     def GetFilenameOf(self, module):
328         return self.source_files[module]['filename']
329
330     def GetMd5Of(self, module):
331         return self.source_files[module]['md5']
332
333     def GetExtractDirOf(self, module):
334         return self.source_files[module]['extract-dir']
335
336     def GetAdditionalParameters(self, module, step):
337         key = step + '-params'
338         if key in self.source_files[module]:
339             return self.source_files[module][key]
340         else:
341             return tuple()
342
343 class Extracter:
344     """class Extracter
345
346     Handles the extraction of the source files from their downloaded
347     archive files.
348     """
349
350     def __init__(self, source_files, config):
351         self.source_files = source_files
352         self.config = config
353
354     def Extract(self, module):
355         src = self.config.src_dir
356         extractDst = os.path.join(src, self.config.arch)
357         local_file = os.path.join(src, self.source_files.GetFilenameOf(module))
358         moduleMd5 = self.source_files.GetMd5Of(module)
359         extracted = os.path.join(extractDst, os.path.split(local_file)[1] + '.extracted')
360         if not os.path.exists(extractDst):
361             os.mkdir(extractDst)
362
363         extractedMd5 = None
364         if os.path.exists(extracted):
365             extractedMd5 = open(extracted).read()
366
367         if extractedMd5 != moduleMd5:
368             print 'Extracting %s:' % self.config.Relative(local_file)
369             tar = tarfile.open(local_file)
370             tar.extractall(extractDst)
371             open(extracted, 'w').write(moduleMd5)
372         else:
373             pass
374             #print 'Previously extracted', self.config.Relative(local_file)
375
376     def ExtractAll(self):
377         for module in self.source_files.GetModules():
378             self.Extract(module)
379
380 class Builder:
381     """class Builder
382
383     Builds and installs the GCC tool suite.
384     """
385
386     def __init__(self, source_files, config):
387         self.source_files = source_files
388         self.config = config
389
390     def Build(self):
391         self.BuildModule('binutils')
392         self.CopyIncludeDirectory()
393         self.BuildModule('gcc')
394         self.MakeSymLinks()
395
396     def IsBuildStepComplete(self, step):
397         return \
398             os.path.exists(
399                 os.path.join(
400                     self.config.build_dir, self.config.arch, step + '.completed'
401                     )
402                 )
403
404     def MarkBuildStepComplete(self, step):
405         open(
406             os.path.join(
407                 self.config.build_dir, self.config.arch, step + '.completed'
408                 ),
409             "w"
410             ).close()
411
412     def CopyIncludeDirectory(self):
413         linkdst = os.path.join(self.config.prefix, 'mingw')
414         src = os.path.join(
415             self.config.src_dir,
416             self.config.arch,
417             self.source_files.GetExtractDirOf('mingw_hdr'),
418             'include'
419             )
420         dst_parent = os.path.join(self.config.prefix, self.config.target_combo)
421         dst = os.path.join(dst_parent, 'include')
422         if not os.path.exists(dst):
423             if not os.path.exists(dst_parent):
424                 os.makedirs(dst_parent)
425             print 'Copying headers to', self.config.Relative(dst)
426             shutil.copytree(src, dst, True)
427         if not os.path.lexists(linkdst):
428             print 'Making symlink at', self.config.Relative(linkdst)
429             os.symlink(self.config.target_combo, linkdst)
430
431     def BuildModule(self, module):
432         base_dir = os.getcwd()
433         build_dir = os.path.join(self.config.build_dir, self.config.arch, module)
434         module_dir = self.source_files.GetExtractDirOf(module)
435         module_dir = os.path.realpath(os.path.join('src', self.config.arch, module_dir))
436         configure = os.path.join(module_dir, 'configure')
437         prefix = self.config.prefix
438         if not os.path.exists(build_dir):
439             os.makedirs(build_dir)
440         os.chdir(build_dir)
441
442         cmd = (
443             configure,
444             '--target=%s' % self.config.target_combo,
445             '--prefix=' + prefix,
446             '--with-sysroot=' + prefix,
447             )
448         if os.path.exists('/opt/local/include/gmp.h'):
449             cmd += ('--with-gmp=/opt/local',)
450         if module == 'gcc': cmd += ('--oldincludedir=/opt/local/include',)
451         cmd += self.source_files.GetAdditionalParameters(module, 'configure')
452         self.RunCommand(cmd, module, 'config', skipable=True)
453
454         cmd = ('make',)
455         if module == 'gcc':
456             cmd += ('all-gcc',)
457         self.RunCommand(cmd, module, 'build')
458
459         cmd = ('make',)
460         if module == 'gcc':
461             cmd += ('install-gcc',)
462         else:
463             cmd += ('install',)
464         self.RunCommand(cmd, module, 'install')
465
466         os.chdir(base_dir)
467
468         print '%s module is now built and installed' % module
469
470     def RunCommand(self, cmd, module, stage, skipable=False):
471         if skipable:
472             if self.IsBuildStepComplete('%s.%s' % (module, stage)):
473                 return
474
475         popen = lambda cmd: \
476             subprocess.Popen(
477                 cmd,
478                 stdin=subprocess.PIPE,
479                 stdout=subprocess.PIPE,
480                 stderr=subprocess.STDOUT
481                 )
482
483         print '%s [%s] ...' % (module, stage),
484         sys.stdout.flush()
485         p = popen(cmd)
486         output = p.stdout.read()
487         p.wait()
488         if p.returncode != 0:
489             print '[failed!]'
490             logFile = os.path.join(self.config.build_dir, 'log.txt')
491             f = open(logFile, "w")
492             f.write(output)
493             f.close()
494             raise Exception, 'Failed to %s %s\n' % (stage, module) + \
495                 'See output log at %s' % self.config.Relative(logFile)
496         else:
497             print '[done]'
498
499         if skipable:
500             self.MarkBuildStepComplete('%s.%s' % (module, stage))
501
502     def MakeSymLinks(self):
503         links_dir = os.path.join(self.config.symlinks, self.config.arch)
504         if not os.path.exists(links_dir):
505             os.makedirs(links_dir)
506         startPrinted = False
507         for link in ('ar', 'ld', 'gcc'):
508             src = os.path.join(
509                 self.config.prefix, 'bin', self.config.target_combo + '-' + link
510                 )
511             linkdst = os.path.join(links_dir, link)
512             if not os.path.lexists(linkdst):
513                 if not startPrinted:
514                     print 'Making symlinks in %s:' % self.config.Relative(links_dir),
515                     startPrinted = True
516                 print link,
517                 os.symlink(src, linkdst)
518
519         if startPrinted:
520             print '[done]'
521
522 class App:
523     """class App
524
525     The main body of the application.
526     """
527
528     def __init__(self):
529         config = Config()
530
531         if not config.IsConfigOk():
532             return
533
534         config.MakeDirs()
535
536         sources = SourceFiles(config)
537         result = sources.GetAll()
538         if result:
539             print 'All files have been downloaded & verified'
540         else:
541             print 'An error occured while downloading a file'
542             return
543
544         Extracter(sources, config).ExtractAll()
545
546         Builder(sources, config).Build()
547
548 App()
549