36079a1718644ed0b0cf73f716b6506e67f63cb0
[mirror/efi/basetools/.git] / gcc / mingw-gcc-build.py
1 #!/usr/bin/env python
2
3 ## @file
4 #
5 # Automation of instructions from:
6 #   http://mingw-w64.svn.sourceforge.net/viewvc/mingw-w64/trunk/mingw-w64-doc/
7 #     howto-build/mingw-w64-howto-build.txt?revision=216&view=markup
8 #
9 # Copyright (c) 2008 - 2010, Intel Corporation
10 # All rights reserved. This program and the accompanying materials
11 # are licensed and made available under the terms and conditions of the BSD License
12 # which accompanies this distribution.    The full text of the license may be found at
13 # http://opensource.org/licenses/bsd-license.php
14 #
15 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
16 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
17 #
18
19
20 from optparse import OptionParser
21 import os
22 import shutil
23 import subprocess
24 import sys
25 import tarfile
26 import urllib
27 import urlparse
28 try:
29     from hashlib import md5
30 except Exception:
31     from md5 import md5
32
33 if sys.version_info < (2, 5):
34     #
35     # This script (and edk2 BaseTools) require Python 2.5 or newer
36     #
37     print 'Python version 2.5 or later is required.'
38     sys.exit(-1)
39
40 #
41 # Version and Copyright
42 #
43 VersionNumber = "0.01"
44 __version__ = "%prog Version " + VersionNumber
45 __copyright__ = "Copyright (c) 2008 - 2010, Intel Corporation.  All rights reserved."
46
47 class Config:
48     """class Config
49
50     Stores the configuration options for the rest of the script.
51
52     Handles the command line options, and allows the code within
53     the script to easily interact with the 'config' requested by
54     the user.
55     """
56
57     def __init__(self):
58         self.base_dir = os.getcwd()
59         (self.options, self.args) = self.CheckOptions()
60         self.__init_dirs__()
61
62     def CheckOptions(self):
63         Parser = \
64             OptionParser(
65                 description=__copyright__,
66                 version=__version__,
67                 prog="mingw-gcc-build",
68                 usage="%prog [options] [target]"
69                 )
70         Parser.add_option(
71             "--arch",
72             action = "store", type = "string",
73             default = '',
74             dest = "arch",
75             help = "Processor architecture to build gcc for."
76             )
77         Parser.add_option(
78             "--src-dir",
79             action = "store", type = "string", dest = "src_dir",
80             default = os.path.join(self.base_dir, 'src'),
81             help = "Directory to download/extract binutils/gcc sources"
82             )
83         Parser.add_option(
84             "--build-dir",
85             action = "store", type = "string", dest = "build_dir",
86             default = os.path.join(self.base_dir, 'build'),
87             help = "Directory to download/extract binutils/gcc sources"
88             )
89         Parser.add_option(
90             "--prefix",
91             action = "store", type = "string", dest = "prefix",
92             default = os.path.join(self.base_dir, 'install'),
93             help = "Prefix to install binutils/gcc into"
94             )
95         Parser.add_option(
96             "--symlinks",
97             action = "store", type = "string", dest = "symlinks",
98             default = os.path.join(self.base_dir, 'symlinks'),
99             help = "Directory to create binutils/gcc symbolic links into."
100             )
101         Parser.add_option(
102             "-v", "--verbose",
103             action="store_true",
104             type=None, help="Print verbose messages"
105             )
106
107         (Opt, Args) = Parser.parse_args()
108
109         self.arch = Opt.arch.lower()
110         allowedArchs = ('ia32', 'x64', 'ipf')
111         if self.arch not in allowedArchs:
112             Parser.error(
113                 'Please use --arch to specify one of: %s' %
114                     ', '.join(allowedArchs)
115                 )
116         self.target_arch = {'ia32': 'i686', 'x64': 'x86_64', 'ipf': 'ia64'}[self.arch]
117         self.target_sys = {'ia32': 'pc', 'x64': 'pc', 'ipf': 'pc'}[self.arch]
118         self.target_bin = {'ia32': 'mingw32', 'x64': 'mingw32', 'ipf': 'elf'}[self.arch]
119         self.target_combo = '-'.join((self.target_arch, self.target_sys, self.target_bin))
120
121         return (Opt, Args)
122
123     def __init_dirs__(self):
124         self.src_dir = os.path.realpath(os.path.expanduser(self.options.src_dir))
125         self.build_dir = os.path.realpath(os.path.expanduser(self.options.build_dir))
126         self.prefix = os.path.realpath(os.path.expanduser(self.options.prefix))
127         self.symlinks = os.path.realpath(os.path.expanduser(self.options.symlinks))
128
129     def IsConfigOk(self):
130                 
131         print "Current directory:"
132         print "   ", self.base_dir
133         print "Sources download/extraction:", self.Relative(self.src_dir)
134         print "Build directory            :", self.Relative(self.build_dir)
135         print "Prefix (install) directory :", self.Relative(self.prefix)
136         print "Create symlinks directory  :", self.Relative(self.symlinks)
137         print
138         answer = raw_input("Is this configuration ok? (default = no): ")
139         if (answer.lower() not in ('y', 'yes')):
140             print
141             print "Please try using --help and then change the configuration."
142             return False
143
144         if self.arch.lower() == 'ipf':
145             print
146             print 'Please note that the IPF compiler built by this script has'
147             print 'not yet been validated!'
148             print
149             answer = raw_input("Are you sure you want to build it? (default = no): ")
150             if (answer.lower() not in ('y', 'yes')):
151                 print
152                 print "Please try using --help and then change the configuration."
153                 return False
154
155         print
156         return True
157
158     def Relative(self, path):
159         if path.startswith(self.base_dir):
160             return '.' + path[len(self.base_dir):]
161         return path
162
163     def MakeDirs(self):
164         for path in (self.src_dir, self.build_dir,self.prefix, self.symlinks):
165             if not os.path.exists(path):
166                 os.makedirs(path)
167
168 class SourceFiles:
169     """class SourceFiles
170
171     Handles the downloading of source files used by the script.
172     """
173
174     def __init__(self, config):
175         self.config = config
176         self.source_files = self.source_files[config.arch]
177
178     source_files_common = {
179         'binutils': {
180             'url': 'http://www.kernel.org/pub/linux/devel/binutils/' + \
181                    'binutils-$version.tar.bz2',
182             'version': '2.20.51.0.5',
183             'md5': '6d2de7cdf7a8389e70b124e3d73b4d37',
184             },
185         }
186
187     source_files_x64 = {
188         'gcc': {
189             'url': 'http://ftpmirror.gnu.org/gcc/' + \
190                    'gcc-$version/gcc-$version.tar.bz2',
191             'version': '4.3.0',
192             'md5': '197ed8468b38db1d3481c3111691d85b',
193             },
194         'mingw_hdr': {
195             'url': 'http://sourceforge.net/projects/' + \
196                    'mingw-w64/files/mingw-w64/mingw-w64-snapshot/' + \
197                    'mingw-w64-trunk-snapshot-$version.tar.bz2/download',
198             'extract-dir': os.path.join('trunk', 'mingw-w64-headers'),
199             'version': '20091222',
200             'md5': 'fbcf282d1a05df121088d775e02095d6',
201             },
202         }
203
204     source_files_ia32 = {
205         'gcc': source_files_x64['gcc'],
206         'mingw_hdr': {
207             'url': 'http://sourceforge.net/projects/' + \
208                    'mingw/files/MinGW%20Runtime/' + \
209                    'mingwrt-$version/' + \
210                    'mingwrt-$version-mingw32-src.tar.gz/download',
211             'extract-dir': 'mingwrt-$version-mingw32',
212             'version': '3.15.2',
213             'md5': '7bf0525f158213f3ac990ea68a5ec34d',
214             },
215         }
216
217     source_files_ipf = source_files_x64.copy()
218     source_files_ipf['gcc']['configure-params'] = (
219         '--with-gnu-as', '--with-gnu-ld', '--with-newlib',
220         '--verbose', '--disable-libssp', '--disable-nls',
221         '--enable-languages=c,c++'
222         )
223
224     source_files = {
225         'ia32': [source_files_common, source_files_ia32],
226         'x64': [source_files_common, source_files_x64],
227         'ipf': [source_files_common, source_files_ipf],
228         }
229
230     for arch in source_files:
231         mergedSourceFiles = {}
232         for source_files_dict in source_files[arch]:
233             mergedSourceFiles.update(source_files_dict)
234         for downloadItem in mergedSourceFiles:
235             fdata = mergedSourceFiles[downloadItem]
236             fdata['filename'] = fdata['url'].split('/')[-1]
237             if 'extract-dir' not in fdata:
238                 for ext in ('.tar.gz', '.tar.bz2', '.zip'):
239                     if fdata['filename'].endswith(ext):
240                         fdata['extract-dir'] = fdata['filename'][:-len(ext)]
241                         break
242             replaceables = ('extract-dir', 'filename', 'url')
243             for replaceItem in fdata:
244                 if replaceItem in replaceables: continue
245                 if type(fdata[replaceItem]) != str: continue
246                 for replaceable in replaceables:
247                     if type(fdata[replaceable]) != str: continue
248                     if replaceable in fdata:
249                         fdata[replaceable] = \
250                             fdata[replaceable].replace(
251                                 '$' + replaceItem,
252                                 fdata[replaceItem]
253                                 )
254         source_files[arch] = mergedSourceFiles
255     #print 'source_files:', source_files
256
257     def GetAll(self):
258
259         def progress(received, blockSize, fileSize):
260             if fileSize < 0: return
261             wDots = (100 * received * blockSize) / fileSize / 10
262             if wDots > self.dots:
263                 for i in range(wDots - self.dots):
264                     print '.',
265                     sys.stdout.flush()
266                     self.dots += 1
267
268         maxRetries = 1
269         for (fname, fdata) in self.source_files.items():
270             for retries in range(maxRetries):
271                 try:
272                     self.dots = 0
273                     local_file = os.path.join(self.config.src_dir, fdata['filename'])
274                     url = fdata['url']
275                     print 'Downloading %s:' % fname, url
276                     if retries > 0:
277                         print '(retry)',
278                     sys.stdout.flush()
279
280                     completed = False
281                     if os.path.exists(local_file):
282                         md5_pass = self.checkHash(fdata)
283                         if md5_pass:
284                             print '[md5 match]',
285                         else:
286                             print '[md5 mismatch]',
287                         sys.stdout.flush()
288                         completed = md5_pass
289
290                     if not completed:
291                         urllib.urlretrieve(url, local_file, progress)
292
293                     #
294                     # BUGBUG: Suggest proxy to user if download fails.
295                     #
296                     # export http_proxy=http://proxyservername.mycompany.com:911
297                     # export ftp_proxy=http://proxyservername.mycompany.com:911
298
299                     if not completed and os.path.exists(local_file):
300                         md5_pass = self.checkHash(fdata)
301                         if md5_pass:
302                             print '[md5 match]',
303                         else:
304                             print '[md5 mismatch]',
305                         sys.stdout.flush()
306                         completed = md5_pass
307
308                     if completed:
309                         print '[done]'
310                         break
311                     else:
312                         print '[failed]'
313                         print '  Tried to retrieve', url
314                         print '  to', local_file
315                         print 'Possible fixes:'
316                         print '* If you are behind a web-proxy, try setting the',
317                         print 'http_proxy environment variable'
318                         print '* You can try to download this file separately',
319                         print 'and rerun this script'
320                         raise Exception()
321                 
322                 except KeyboardInterrupt:
323                     print '[KeyboardInterrupt]'
324                     return False
325
326                 except Exception, e:
327                     print e
328
329             if not completed: return False
330
331         return True
332
333     def checkHash(self, fdata):
334         local_file = os.path.join(self.config.src_dir, fdata['filename'])
335         expect_md5 = fdata['md5']
336         data = open(local_file).read()
337         md5sum = md5()
338         md5sum.update(data)
339         return md5sum.hexdigest().lower() == expect_md5.lower()
340
341     def GetModules(self):
342         return self.source_files.keys()
343
344     def GetFilenameOf(self, module):
345         return self.source_files[module]['filename']
346
347     def GetMd5Of(self, module):
348         return self.source_files[module]['md5']
349
350     def GetExtractDirOf(self, module):
351         return self.source_files[module]['extract-dir']
352
353     def GetAdditionalParameters(self, module, step):
354         key = step + '-params'
355         if key in self.source_files[module]:
356             return self.source_files[module][key]
357         else:
358             return tuple()
359
360 class Extracter:
361     """class Extracter
362
363     Handles the extraction of the source files from their downloaded
364     archive files.
365     """
366
367     def __init__(self, source_files, config):
368         self.source_files = source_files
369         self.config = config
370
371     def Extract(self, module):
372         src = self.config.src_dir
373         extractDst = os.path.join(src, self.config.arch)
374         local_file = os.path.join(src, self.source_files.GetFilenameOf(module))
375         moduleMd5 = self.source_files.GetMd5Of(module)
376         extracted = os.path.join(extractDst, os.path.split(local_file)[1] + '.extracted')
377         if not os.path.exists(extractDst):
378             os.mkdir(extractDst)
379
380         extractedMd5 = None
381         if os.path.exists(extracted):
382             extractedMd5 = open(extracted).read()
383
384         if extractedMd5 != moduleMd5:
385             print 'Extracting %s:' % self.config.Relative(local_file)
386             tar = tarfile.open(local_file)
387             tar.extractall(extractDst)
388             open(extracted, 'w').write(moduleMd5)
389         else:
390             pass
391             #print 'Previously extracted', self.config.Relative(local_file)
392
393     def ExtractAll(self):
394         for module in self.source_files.GetModules():
395             self.Extract(module)
396
397 class Builder:
398     """class Builder
399
400     Builds and installs the GCC tool suite.
401     """
402
403     def __init__(self, source_files, config):
404         self.source_files = source_files
405         self.config = config
406
407     def Build(self):
408         self.BuildModule('binutils')
409         self.CopyIncludeDirectory()
410         self.BuildModule('gcc')
411         self.MakeSymLinks()
412
413     def IsBuildStepComplete(self, step):
414         return \
415             os.path.exists(
416                 os.path.join(
417                     self.config.build_dir, self.config.arch, step + '.completed'
418                     )
419                 )
420
421     def MarkBuildStepComplete(self, step):
422         open(
423             os.path.join(
424                 self.config.build_dir, self.config.arch, step + '.completed'
425                 ),
426             "w"
427             ).close()
428
429     def CopyIncludeDirectory(self):
430         linkdst = os.path.join(self.config.prefix, 'mingw')
431         src = os.path.join(
432             self.config.src_dir,
433             self.config.arch,
434             self.source_files.GetExtractDirOf('mingw_hdr'),
435             'include'
436             )
437         dst_parent = os.path.join(self.config.prefix, self.config.target_combo)
438         dst = os.path.join(dst_parent, 'include')
439         if not os.path.exists(dst):
440             if not os.path.exists(dst_parent):
441                 os.makedirs(dst_parent)
442             print 'Copying headers to', self.config.Relative(dst)
443             shutil.copytree(src, dst, True)
444         if not os.path.lexists(linkdst):
445             print 'Making symlink at', self.config.Relative(linkdst)
446             os.symlink(self.config.target_combo, linkdst)
447
448     def BuildModule(self, module):
449         base_dir = os.getcwd()
450         build_dir = os.path.join(self.config.build_dir, self.config.arch, module)
451         module_dir = self.source_files.GetExtractDirOf(module)
452         module_dir = os.path.realpath(os.path.join('src', self.config.arch, module_dir))
453         configure = os.path.join(module_dir, 'configure')
454         prefix = self.config.prefix
455         if not os.path.exists(build_dir):
456             os.makedirs(build_dir)
457         os.chdir(build_dir)
458
459         cmd = (
460             configure,
461             '--target=%s' % self.config.target_combo,
462             '--prefix=' + prefix,
463             '--with-sysroot=' + prefix,
464             '--disable-werror',
465             )
466         if os.path.exists('/opt/local/include/gmp.h'):
467             cmd += ('--with-gmp=/opt/local',)
468         if module == 'gcc': cmd += ('--oldincludedir=/opt/local/include',)
469         cmd += self.source_files.GetAdditionalParameters(module, 'configure')
470         self.RunCommand(cmd, module, 'config', skipable=True)
471
472         cmd = ('make',)
473         if module == 'gcc':
474             cmd += ('all-gcc',)
475         self.RunCommand(cmd, module, 'build')
476
477         cmd = ('make',)
478         if module == 'gcc':
479             cmd += ('install-gcc',)
480         else:
481             cmd += ('install',)
482         self.RunCommand(cmd, module, 'install')
483
484         os.chdir(base_dir)
485
486         print '%s module is now built and installed' % module
487
488     def RunCommand(self, cmd, module, stage, skipable=False):
489         if skipable:
490             if self.IsBuildStepComplete('%s.%s' % (module, stage)):
491                 return
492
493         popen = lambda cmd: \
494             subprocess.Popen(
495                 cmd,
496                 stdin=subprocess.PIPE,
497                 stdout=subprocess.PIPE,
498                 stderr=subprocess.STDOUT
499                 )
500
501         print '%s [%s] ...' % (module, stage),
502         sys.stdout.flush()
503         p = popen(cmd)
504         output = p.stdout.read()
505         p.wait()
506         if p.returncode != 0:
507             print '[failed!]'
508             logFile = os.path.join(self.config.build_dir, 'log.txt')
509             f = open(logFile, "w")
510             f.write(output)
511             f.close()
512             raise Exception, 'Failed to %s %s\n' % (stage, module) + \
513                 'See output log at %s' % self.config.Relative(logFile)
514         else:
515             print '[done]'
516
517         if skipable:
518             self.MarkBuildStepComplete('%s.%s' % (module, stage))
519
520     def MakeSymLinks(self):
521         links_dir = os.path.join(self.config.symlinks, self.config.arch)
522         if not os.path.exists(links_dir):
523             os.makedirs(links_dir)
524         startPrinted = False
525         for link in ('ar', 'ld', 'gcc'):
526             src = os.path.join(
527                 self.config.prefix, 'bin', self.config.target_combo + '-' + link
528                 )
529             linkdst = os.path.join(links_dir, link)
530             if not os.path.lexists(linkdst):
531                 if not startPrinted:
532                     print 'Making symlinks in %s:' % self.config.Relative(links_dir),
533                     startPrinted = True
534                 print link,
535                 os.symlink(src, linkdst)
536
537         if startPrinted:
538             print '[done]'
539
540 class App:
541     """class App
542
543     The main body of the application.
544     """
545
546     def __init__(self):
547         config = Config()
548
549         if not config.IsConfigOk():
550             return
551
552         config.MakeDirs()
553
554         sources = SourceFiles(config)
555         result = sources.GetAll()
556         if result:
557             print 'All files have been downloaded & verified'
558         else:
559             print 'An error occured while downloading a file'
560             return
561
562         Extracter(sources, config).ExtractAll()
563
564         Builder(sources, config).Build()
565
566 App()
567