mingw-gcc-build.py: Add ability to skip building GCC or binutils
[mirror/efi/basetools/.git] / gcc / mingw-gcc-build.py
1 #!/usr/bin/env python
2
3 ## @file
4 #
5 # Automation of instructions from:
6 #   http://mingw-w64.svn.sourceforge.net/viewvc/mingw-w64/trunk/mingw-w64-doc/
7 #     howto-build/mingw-w64-howto-build.txt?revision=216&view=markup
8 #
9 # Copyright (c) 2008 - 2010, Intel Corporation
10 # All rights reserved. This program and the accompanying materials
11 # are licensed and made available under the terms and conditions of the BSD License
12 # which accompanies this distribution.    The full text of the license may be found at
13 # http://opensource.org/licenses/bsd-license.php
14 #
15 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
16 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
17 #
18
19
20 from optparse import OptionParser
21 import os
22 import shutil
23 import subprocess
24 import sys
25 import tarfile
26 import urllib
27 import urlparse
28 try:
29     from hashlib import md5
30 except Exception:
31     from md5 import md5
32
33 if sys.version_info < (2, 5):
34     #
35     # This script (and edk2 BaseTools) require Python 2.5 or newer
36     #
37     print 'Python version 2.5 or later is required.'
38     sys.exit(-1)
39
40 #
41 # Version and Copyright
42 #
43 VersionNumber = "0.01"
44 __version__ = "%prog Version " + VersionNumber
45 __copyright__ = "Copyright (c) 2008 - 2010, Intel Corporation.  All rights reserved."
46
47 class Config:
48     """class Config
49
50     Stores the configuration options for the rest of the script.
51
52     Handles the command line options, and allows the code within
53     the script to easily interact with the 'config' requested by
54     the user.
55     """
56
57     def __init__(self):
58         self.base_dir = os.getcwd()
59         (self.options, self.args) = self.CheckOptions()
60         self.__init_dirs__()
61
62     def CheckOptions(self):
63         Parser = \
64             OptionParser(
65                 description=__copyright__,
66                 version=__version__,
67                 prog="mingw-gcc-build",
68                 usage="%prog [options] [target]"
69                 )
70         Parser.add_option(
71             "--arch",
72             action = "store", type = "string",
73             default = '',
74             dest = "arch",
75             help = "Processor architecture to build gcc for."
76             )
77         Parser.add_option(
78             "--src-dir",
79             action = "store", type = "string", dest = "src_dir",
80             default = os.path.join(self.base_dir, 'src'),
81             help = "Directory to download/extract binutils/gcc sources"
82             )
83         Parser.add_option(
84             "--build-dir",
85             action = "store", type = "string", dest = "build_dir",
86             default = os.path.join(self.base_dir, 'build'),
87             help = "Directory to download/extract binutils/gcc sources"
88             )
89         Parser.add_option(
90             "--prefix",
91             action = "store", type = "string", dest = "prefix",
92             default = os.path.join(self.base_dir, 'install'),
93             help = "Prefix to install binutils/gcc into"
94             )
95         Parser.add_option(
96             "--skip-binutils",
97             action = "store_true", dest = "skip_binutils",
98             default = False,
99             help = "Will skip building binutils"
100             )
101         Parser.add_option(
102             "--skip-gcc",
103             action = "store_true", dest = "skip_gcc",
104             default = False,
105             help = "Will skip building GCC"
106             )
107         Parser.add_option(
108             "--symlinks",
109             action = "store", type = "string", dest = "symlinks",
110             default = os.path.join(self.base_dir, 'symlinks'),
111             help = "Directory to create binutils/gcc symbolic links into."
112             )
113         Parser.add_option(
114             "-v", "--verbose",
115             action="store_true",
116             type=None, help="Print verbose messages"
117             )
118
119         (Opt, Args) = Parser.parse_args()
120
121         self.arch = Opt.arch.lower()
122         allowedArchs = ('ia32', 'x64', 'ipf')
123         if self.arch not in allowedArchs:
124             Parser.error(
125                 'Please use --arch to specify one of: %s' %
126                     ', '.join(allowedArchs)
127                 )
128         self.target_arch = {'ia32': 'i686', 'x64': 'x86_64', 'ipf': 'ia64'}[self.arch]
129         self.target_sys = {'ia32': 'pc', 'x64': 'pc', 'ipf': 'pc'}[self.arch]
130         self.target_bin = {'ia32': 'mingw32', 'x64': 'mingw32', 'ipf': 'elf'}[self.arch]
131         self.target_combo = '-'.join((self.target_arch, self.target_sys, self.target_bin))
132
133         return (Opt, Args)
134
135     def __init_dirs__(self):
136         self.src_dir = os.path.realpath(os.path.expanduser(self.options.src_dir))
137         self.build_dir = os.path.realpath(os.path.expanduser(self.options.build_dir))
138         self.prefix = os.path.realpath(os.path.expanduser(self.options.prefix))
139         self.symlinks = os.path.realpath(os.path.expanduser(self.options.symlinks))
140
141     def IsConfigOk(self):
142
143         building = []
144         if not self.options.skip_binutils:
145             building.append('binutils')
146         if not self.options.skip_gcc:
147             building.append('gcc')
148         if len(building) == 0:
149             print "Nothing will be built!"
150             print
151             print "Please try using --help and then change the configuration."
152             return False
153
154         print "Current directory:"
155         print "   ", self.base_dir
156         print "Sources download/extraction:", self.Relative(self.src_dir)
157         print "Build directory            :", self.Relative(self.build_dir)
158         print "Prefix (install) directory :", self.Relative(self.prefix)
159         print "Create symlinks directory  :", self.Relative(self.symlinks)
160         print "Building                   :", ', '.join(building)
161         print
162         answer = raw_input("Is this configuration ok? (default = no): ")
163         if (answer.lower() not in ('y', 'yes')):
164             print
165             print "Please try using --help and then change the configuration."
166             return False
167
168         if self.arch.lower() == 'ipf':
169             print
170             print 'Please note that the IPF compiler built by this script has'
171             print 'not yet been validated!'
172             print
173             answer = raw_input("Are you sure you want to build it? (default = no): ")
174             if (answer.lower() not in ('y', 'yes')):
175                 print
176                 print "Please try using --help and then change the configuration."
177                 return False
178
179         print
180         return True
181
182     def Relative(self, path):
183         if path.startswith(self.base_dir):
184             return '.' + path[len(self.base_dir):]
185         return path
186
187     def MakeDirs(self):
188         for path in (self.src_dir, self.build_dir,self.prefix, self.symlinks):
189             if not os.path.exists(path):
190                 os.makedirs(path)
191
192 class SourceFiles:
193     """class SourceFiles
194
195     Handles the downloading of source files used by the script.
196     """
197
198     def __init__(self, config):
199         self.config = config
200         self.source_files = self.source_files[config.arch]
201
202         if config.options.skip_binutils:
203             del self.source_files['binutils']
204
205         if config.options.skip_gcc:
206             del self.source_files['gcc']
207             del self.source_files['mingw_hdr']
208
209     source_files_common = {
210         'binutils': {
211             'url': 'http://www.kernel.org/pub/linux/devel/binutils/' + \
212                    'binutils-$version.tar.bz2',
213             'version': '2.20.51.0.5',
214             'md5': '6d2de7cdf7a8389e70b124e3d73b4d37',
215             },
216         }
217
218     source_files_x64 = {
219         'gcc': {
220             'url': 'http://ftpmirror.gnu.org/gcc/' + \
221                    'gcc-$version/gcc-$version.tar.bz2',
222             'version': '4.3.0',
223             'md5': '197ed8468b38db1d3481c3111691d85b',
224             },
225         'mingw_hdr': {
226             'url': 'http://sourceforge.net/projects/' + \
227                    'mingw-w64/files/mingw-w64/mingw-w64-snapshot/' + \
228                    'mingw-w64-trunk-snapshot-$version.tar.bz2/download',
229             'extract-dir': os.path.join('trunk', 'mingw-w64-headers'),
230             'version': '20091222',
231             'md5': 'fbcf282d1a05df121088d775e02095d6',
232             },
233         }
234
235     source_files_ia32 = {
236         'gcc': source_files_x64['gcc'],
237         'mingw_hdr': {
238             'url': 'http://sourceforge.net/projects/' + \
239                    'mingw/files/MinGW%20Runtime/' + \
240                    'mingwrt-$version/' + \
241                    'mingwrt-$version-mingw32-src.tar.gz/download',
242             'extract-dir': 'mingwrt-$version-mingw32',
243             'version': '3.15.2',
244             'md5': '7bf0525f158213f3ac990ea68a5ec34d',
245             },
246         }
247
248     source_files_ipf = source_files_x64.copy()
249     source_files_ipf['gcc']['configure-params'] = (
250         '--with-gnu-as', '--with-gnu-ld', '--with-newlib',
251         '--verbose', '--disable-libssp', '--disable-nls',
252         '--enable-languages=c,c++'
253         )
254
255     source_files = {
256         'ia32': [source_files_common, source_files_ia32],
257         'x64': [source_files_common, source_files_x64],
258         'ipf': [source_files_common, source_files_ipf],
259         }
260
261     for arch in source_files:
262         mergedSourceFiles = {}
263         for source_files_dict in source_files[arch]:
264             mergedSourceFiles.update(source_files_dict)
265         for downloadItem in mergedSourceFiles:
266             fdata = mergedSourceFiles[downloadItem]
267             fdata['filename'] = fdata['url'].split('/')[-1]
268             if 'extract-dir' not in fdata:
269                 for ext in ('.tar.gz', '.tar.bz2', '.zip'):
270                     if fdata['filename'].endswith(ext):
271                         fdata['extract-dir'] = fdata['filename'][:-len(ext)]
272                         break
273             replaceables = ('extract-dir', 'filename', 'url')
274             for replaceItem in fdata:
275                 if replaceItem in replaceables: continue
276                 if type(fdata[replaceItem]) != str: continue
277                 for replaceable in replaceables:
278                     if type(fdata[replaceable]) != str: continue
279                     if replaceable in fdata:
280                         fdata[replaceable] = \
281                             fdata[replaceable].replace(
282                                 '$' + replaceItem,
283                                 fdata[replaceItem]
284                                 )
285         source_files[arch] = mergedSourceFiles
286     #print 'source_files:', source_files
287
288     def GetAll(self):
289
290         def progress(received, blockSize, fileSize):
291             if fileSize < 0: return
292             wDots = (100 * received * blockSize) / fileSize / 10
293             if wDots > self.dots:
294                 for i in range(wDots - self.dots):
295                     print '.',
296                     sys.stdout.flush()
297                     self.dots += 1
298
299         maxRetries = 1
300         for (fname, fdata) in self.source_files.items():
301             for retries in range(maxRetries):
302                 try:
303                     self.dots = 0
304                     local_file = os.path.join(self.config.src_dir, fdata['filename'])
305                     url = fdata['url']
306                     print 'Downloading %s:' % fname, url
307                     if retries > 0:
308                         print '(retry)',
309                     sys.stdout.flush()
310
311                     completed = False
312                     if os.path.exists(local_file):
313                         md5_pass = self.checkHash(fdata)
314                         if md5_pass:
315                             print '[md5 match]',
316                         else:
317                             print '[md5 mismatch]',
318                         sys.stdout.flush()
319                         completed = md5_pass
320
321                     if not completed:
322                         urllib.urlretrieve(url, local_file, progress)
323
324                     #
325                     # BUGBUG: Suggest proxy to user if download fails.
326                     #
327                     # export http_proxy=http://proxyservername.mycompany.com:911
328                     # export ftp_proxy=http://proxyservername.mycompany.com:911
329
330                     if not completed and os.path.exists(local_file):
331                         md5_pass = self.checkHash(fdata)
332                         if md5_pass:
333                             print '[md5 match]',
334                         else:
335                             print '[md5 mismatch]',
336                         sys.stdout.flush()
337                         completed = md5_pass
338
339                     if completed:
340                         print '[done]'
341                         break
342                     else:
343                         print '[failed]'
344                         print '  Tried to retrieve', url
345                         print '  to', local_file
346                         print 'Possible fixes:'
347                         print '* If you are behind a web-proxy, try setting the',
348                         print 'http_proxy environment variable'
349                         print '* You can try to download this file separately',
350                         print 'and rerun this script'
351                         raise Exception()
352                 
353                 except KeyboardInterrupt:
354                     print '[KeyboardInterrupt]'
355                     return False
356
357                 except Exception, e:
358                     print e
359
360             if not completed: return False
361
362         return True
363
364     def checkHash(self, fdata):
365         local_file = os.path.join(self.config.src_dir, fdata['filename'])
366         expect_md5 = fdata['md5']
367         data = open(local_file).read()
368         md5sum = md5()
369         md5sum.update(data)
370         return md5sum.hexdigest().lower() == expect_md5.lower()
371
372     def GetModules(self):
373         return self.source_files.keys()
374
375     def GetFilenameOf(self, module):
376         return self.source_files[module]['filename']
377
378     def GetMd5Of(self, module):
379         return self.source_files[module]['md5']
380
381     def GetExtractDirOf(self, module):
382         return self.source_files[module]['extract-dir']
383
384     def GetAdditionalParameters(self, module, step):
385         key = step + '-params'
386         if key in self.source_files[module]:
387             return self.source_files[module][key]
388         else:
389             return tuple()
390
391 class Extracter:
392     """class Extracter
393
394     Handles the extraction of the source files from their downloaded
395     archive files.
396     """
397
398     def __init__(self, source_files, config):
399         self.source_files = source_files
400         self.config = config
401
402     def Extract(self, module):
403         src = self.config.src_dir
404         extractDst = os.path.join(src, self.config.arch)
405         local_file = os.path.join(src, self.source_files.GetFilenameOf(module))
406         moduleMd5 = self.source_files.GetMd5Of(module)
407         extracted = os.path.join(extractDst, os.path.split(local_file)[1] + '.extracted')
408         if not os.path.exists(extractDst):
409             os.mkdir(extractDst)
410
411         extractedMd5 = None
412         if os.path.exists(extracted):
413             extractedMd5 = open(extracted).read()
414
415         if extractedMd5 != moduleMd5:
416             print 'Extracting %s:' % self.config.Relative(local_file)
417             tar = tarfile.open(local_file)
418             tar.extractall(extractDst)
419             open(extracted, 'w').write(moduleMd5)
420         else:
421             pass
422             #print 'Previously extracted', self.config.Relative(local_file)
423
424     def ExtractAll(self):
425         for module in self.source_files.GetModules():
426             self.Extract(module)
427
428 class Builder:
429     """class Builder
430
431     Builds and installs the GCC tool suite.
432     """
433
434     def __init__(self, source_files, config):
435         self.source_files = source_files
436         self.config = config
437
438     def Build(self):
439         if not self.config.options.skip_binutils:
440             self.BuildModule('binutils')
441         if not self.config.options.skip_gcc:
442             self.CopyIncludeDirectory()
443             self.BuildModule('gcc')
444             self.MakeSymLinks()
445
446     def IsBuildStepComplete(self, step):
447         return \
448             os.path.exists(
449                 os.path.join(
450                     self.config.build_dir, self.config.arch, step + '.completed'
451                     )
452                 )
453
454     def MarkBuildStepComplete(self, step):
455         open(
456             os.path.join(
457                 self.config.build_dir, self.config.arch, step + '.completed'
458                 ),
459             "w"
460             ).close()
461
462     def CopyIncludeDirectory(self):
463         linkdst = os.path.join(self.config.prefix, 'mingw')
464         src = os.path.join(
465             self.config.src_dir,
466             self.config.arch,
467             self.source_files.GetExtractDirOf('mingw_hdr'),
468             'include'
469             )
470         dst_parent = os.path.join(self.config.prefix, self.config.target_combo)
471         dst = os.path.join(dst_parent, 'include')
472         if not os.path.exists(dst):
473             if not os.path.exists(dst_parent):
474                 os.makedirs(dst_parent)
475             print 'Copying headers to', self.config.Relative(dst)
476             shutil.copytree(src, dst, True)
477         if not os.path.lexists(linkdst):
478             print 'Making symlink at', self.config.Relative(linkdst)
479             os.symlink(self.config.target_combo, linkdst)
480
481     def BuildModule(self, module):
482         base_dir = os.getcwd()
483         build_dir = os.path.join(self.config.build_dir, self.config.arch, module)
484         module_dir = self.source_files.GetExtractDirOf(module)
485         module_dir = os.path.realpath(os.path.join('src', self.config.arch, module_dir))
486         configure = os.path.join(module_dir, 'configure')
487         prefix = self.config.prefix
488         if not os.path.exists(build_dir):
489             os.makedirs(build_dir)
490         os.chdir(build_dir)
491
492         cmd = (
493             configure,
494             '--target=%s' % self.config.target_combo,
495             '--prefix=' + prefix,
496             '--with-sysroot=' + prefix,
497             '--disable-werror',
498             )
499         if os.path.exists('/opt/local/include/gmp.h'):
500             cmd += ('--with-gmp=/opt/local',)
501         if module == 'gcc': cmd += ('--oldincludedir=/opt/local/include',)
502         cmd += self.source_files.GetAdditionalParameters(module, 'configure')
503         self.RunCommand(cmd, module, 'config', skipable=True)
504
505         cmd = ('make',)
506         if module == 'gcc':
507             cmd += ('all-gcc',)
508         self.RunCommand(cmd, module, 'build')
509
510         cmd = ('make',)
511         if module == 'gcc':
512             cmd += ('install-gcc',)
513         else:
514             cmd += ('install',)
515         self.RunCommand(cmd, module, 'install')
516
517         os.chdir(base_dir)
518
519         print '%s module is now built and installed' % module
520
521     def RunCommand(self, cmd, module, stage, skipable=False):
522         if skipable:
523             if self.IsBuildStepComplete('%s.%s' % (module, stage)):
524                 return
525
526         popen = lambda cmd: \
527             subprocess.Popen(
528                 cmd,
529                 stdin=subprocess.PIPE,
530                 stdout=subprocess.PIPE,
531                 stderr=subprocess.STDOUT
532                 )
533
534         print '%s [%s] ...' % (module, stage),
535         sys.stdout.flush()
536         p = popen(cmd)
537         output = p.stdout.read()
538         p.wait()
539         if p.returncode != 0:
540             print '[failed!]'
541             logFile = os.path.join(self.config.build_dir, 'log.txt')
542             f = open(logFile, "w")
543             f.write(output)
544             f.close()
545             raise Exception, 'Failed to %s %s\n' % (stage, module) + \
546                 'See output log at %s' % self.config.Relative(logFile)
547         else:
548             print '[done]'
549
550         if skipable:
551             self.MarkBuildStepComplete('%s.%s' % (module, stage))
552
553     def MakeSymLinks(self):
554         links_dir = os.path.join(self.config.symlinks, self.config.arch)
555         if not os.path.exists(links_dir):
556             os.makedirs(links_dir)
557         startPrinted = False
558         for link in ('ar', 'ld', 'gcc'):
559             src = os.path.join(
560                 self.config.prefix, 'bin', self.config.target_combo + '-' + link
561                 )
562             linkdst = os.path.join(links_dir, link)
563             if not os.path.lexists(linkdst):
564                 if not startPrinted:
565                     print 'Making symlinks in %s:' % self.config.Relative(links_dir),
566                     startPrinted = True
567                 print link,
568                 os.symlink(src, linkdst)
569
570         if startPrinted:
571             print '[done]'
572
573 class App:
574     """class App
575
576     The main body of the application.
577     """
578
579     def __init__(self):
580         config = Config()
581
582         if not config.IsConfigOk():
583             return
584
585         config.MakeDirs()
586
587         sources = SourceFiles(config)
588         result = sources.GetAll()
589         if result:
590             print 'All files have been downloaded & verified'
591         else:
592             print 'An error occured while downloading a file'
593             return
594
595         Extracter(sources, config).ExtractAll()
596
597         Builder(sources, config).Build()
598
599 App()
600