Update links for broken urls in BaseTools/gcc/mingw-gcc-build.py script.
[efi/edk2/.git] / edk2 / BaseTools / gcc / mingw-gcc-build.py
1 #!/usr/bin/env python
2
3 #
4 # Automation of instructions from:
5 #   http://mingw-w64.svn.sourceforge.net/viewvc/mingw-w64/trunk/mingw-w64-doc/
6 #     howto-build/mingw-w64-howto-build.txt?revision=216&view=markup
7 #
8
9 from optparse import OptionParser
10 import os
11 import shutil
12 import subprocess
13 import sys
14 import tarfile
15 import urllib
16 import urlparse
17 try:
18     from hashlib import md5
19 except Exception:
20     from md5 import md5
21
22 if sys.version_info < (2, 5):
23     #
24     # This script (and edk2 BaseTools) require Python 2.5 or newer
25     #
26     print 'Python version 2.5 or later is required.'
27     sys.exit(-1)
28
29 #
30 # Version and Copyright
31 #
32 VersionNumber = "0.01"
33 __version__ = "%prog Version " + VersionNumber
34 __copyright__ = "Copyright (c) 2008, Intel Corporation.  All rights reserved."
35
36 class Config:
37     """class Config
38
39     Stores the configuration options for the rest of the script.
40
41     Handles the command line options, and allows the code within
42     the script to easily interact with the 'config' requested by
43     the user.
44     """
45
46     def __init__(self):
47         self.base_dir = os.getcwd()
48         (self.options, self.args) = self.CheckOptions()
49         self.__init_dirs__()
50
51     def CheckOptions(self):
52         Parser = \
53             OptionParser(
54                 description=__copyright__,
55                 version=__version__,
56                 prog="mingw-gcc-build",
57                 usage="%prog [options] [target]"
58                 )
59         Parser.add_option(
60             "--arch",
61             action = "store", type = "string",
62             default = '',
63             dest = "arch",
64             help = "Processor architecture to build gcc for."
65             )
66         Parser.add_option(
67             "--src-dir",
68             action = "store", type = "string", dest = "src_dir",
69             default = os.path.join(self.base_dir, 'src'),
70             help = "Directory to download/extract binutils/gcc sources"
71             )
72         Parser.add_option(
73             "--build-dir",
74             action = "store", type = "string", dest = "build_dir",
75             default = os.path.join(self.base_dir, 'build'),
76             help = "Directory to download/extract binutils/gcc sources"
77             )
78         Parser.add_option(
79             "--prefix",
80             action = "store", type = "string", dest = "prefix",
81             default = os.path.join(self.base_dir, 'install'),
82             help = "Prefix to install binutils/gcc into"
83             )
84         Parser.add_option(
85             "--symlinks",
86             action = "store", type = "string", dest = "symlinks",
87             default = os.path.join(self.base_dir, 'symlinks'),
88             help = "Directory to create binutils/gcc symbolic links into."
89             )
90         Parser.add_option(
91             "-v", "--verbose",
92             action="store_true",
93             type=None, help="Print verbose messages"
94             )
95
96         (Opt, Args) = Parser.parse_args()
97
98         self.arch = Opt.arch.lower()
99         allowedArchs = ('ia32', 'x64', 'ipf')
100         if self.arch not in allowedArchs:
101             Parser.error(
102                 'Please use --arch to specify one of: %s' %
103                     ', '.join(allowedArchs)
104                 )
105         self.target_arch = {'ia32': 'i686', 'x64': 'x86_64', 'ipf': 'ia64'}[self.arch]
106         self.target_sys = {'ia32': 'pc', 'x64': 'pc', 'ipf': 'pc'}[self.arch]
107         self.target_bin = {'ia32': 'mingw32', 'x64': 'mingw32', 'ipf': 'elf'}[self.arch]
108         self.target_combo = '-'.join((self.target_arch, self.target_sys, self.target_bin))
109
110         return (Opt, Args)
111
112     def __init_dirs__(self):
113         self.src_dir = os.path.realpath(os.path.expanduser(self.options.src_dir))
114         self.build_dir = os.path.realpath(os.path.expanduser(self.options.build_dir))
115         self.prefix = os.path.realpath(os.path.expanduser(self.options.prefix))
116         self.symlinks = os.path.realpath(os.path.expanduser(self.options.symlinks))
117
118     def IsConfigOk(self):
119                 
120         print "Current directory:"
121         print "   ", self.base_dir
122         print "Sources download/extraction:", self.Relative(self.src_dir)
123         print "Build directory            :", self.Relative(self.build_dir)
124         print "Prefix (install) directory :", self.Relative(self.prefix)
125         print "Create symlinks directory  :", self.Relative(self.symlinks)
126         print
127         answer = raw_input("Is this configuration ok? (default = no): ")
128         if (answer.lower() not in ('y', 'yes')):
129             print
130             print "Please try using --help and then change the configuration."
131             return False
132
133         if self.arch.lower() == 'ipf':
134             print
135             print 'Please note that the IPF compiler built by this script has'
136             print 'not yet been validated!'
137             print
138             answer = raw_input("Are you sure you want to build it? (default = no): ")
139             if (answer.lower() not in ('y', 'yes')):
140                 print
141                 print "Please try using --help and then change the configuration."
142                 return False
143
144         print
145         return True
146
147     def Relative(self, path):
148         if path.startswith(self.base_dir):
149             return '.' + path[len(self.base_dir):]
150         return path
151
152     def MakeDirs(self):
153         for path in (self.src_dir, self.build_dir,self.prefix, self.symlinks):
154             if not os.path.exists(path):
155                 os.makedirs(path)
156
157 class SourceFiles:
158     """class SourceFiles
159
160     Handles the downloading of source files used by the script.
161     """
162
163     def __init__(self, config):
164         self.config = config
165         self.source_files = self.source_files[config.arch]
166
167     source_files_common = {
168         'binutils': {
169             'url': 'http://www.kernel.org/pub/linux/devel/binutils/' + \
170                    'binutils-$version.tar.bz2',
171             'version': '2.18.50.0.5',
172             'md5': 'daee18dbbf0a6ccfc186141bee18bf62',
173             },
174         }
175
176     source_files_x64 = {
177         'gcc': {
178             'url': 'http://gcc-ca.internet.bs/releases/' + \
179                    'gcc-$version/gcc-$version.tar.bz2',
180             'version': '4.3.0',
181             'md5': '197ed8468b38db1d3481c3111691d85b',
182             },
183         'mingw_hdr': {
184             'url': 'http://downloads.sourceforge.net/project/' + \
185                    'mingw-w64/mingw-w64/mingw-w64-snapshot/' + \
186                    'mingw-w64-snapshot-$version.tar.bz2',
187             'extract-dir': os.path.join('trunk', 'mingw-w64-headers'),
188             'version': '20090419',
189             'md5': '9146ecfabaf172e4cc427b88e8d218c1',
190             },
191         }
192
193     source_files_ia32 = {
194         'gcc': source_files_x64['gcc'],
195         'mingw_hdr': {
196             'url': 'http://downloads.sourceforge.net/project/' + \
197                    'mingw/MinGW%20Runtime/' + \
198                    'Current%20Release_%20mingwrt-$version/' + \
199                    'mingwrt-$version-mingw32-src.tar.gz',
200             'extract-dir': 'mingwrt-$version-mingw32',
201             'version': '3.15.2',
202             'md5': '7bf0525f158213f3ac990ea68a5ec34d',
203             },
204         }
205
206     source_files_ipf = source_files_x64.copy()
207     source_files_ipf['gcc']['configure-params'] = (
208         '--with-gnu-as', '--with-gnu-ld', '--with-newlib',
209         '--verbose', '--disable-libssp', '--disable-nls',
210         '--enable-languages=c,c++'
211         )
212
213     source_files = {
214         'ia32': [source_files_common, source_files_ia32],
215         'x64': [source_files_common, source_files_x64],
216         'ipf': [source_files_common, source_files_ipf],
217         }
218
219     for arch in source_files:
220         mergedSourceFiles = {}
221         for source_files_dict in source_files[arch]:
222             mergedSourceFiles.update(source_files_dict)
223         for downloadItem in mergedSourceFiles:
224             fdata = mergedSourceFiles[downloadItem]
225             fdata['filename'] = fdata['url'].split('/')[-1]
226             if 'extract-dir' not in fdata:
227                 for ext in ('.tar.gz', '.tar.bz2', '.zip'):
228                     if fdata['filename'].endswith(ext):
229                         fdata['extract-dir'] = fdata['filename'][:-len(ext)]
230                         break
231             replaceables = ('extract-dir', 'filename', 'url')
232             for replaceItem in fdata:
233                 if replaceItem in replaceables: continue
234                 if type(fdata[replaceItem]) != str: continue
235                 for replaceable in replaceables:
236                     if type(fdata[replaceable]) != str: continue
237                     if replaceable in fdata:
238                         fdata[replaceable] = \
239                             fdata[replaceable].replace(
240                                 '$' + replaceItem,
241                                 fdata[replaceItem]
242                                 )
243         source_files[arch] = mergedSourceFiles
244     #print 'source_files:', source_files
245
246     def GetAll(self):
247
248         def progress(received, blockSize, fileSize):
249             if fileSize < 0: return
250             wDots = (100 * received * blockSize) / fileSize / 10
251             if wDots > self.dots:
252                 for i in range(wDots - self.dots):
253                     print '.',
254                     sys.stdout.flush()
255                     self.dots += 1
256
257         maxRetries = 1
258         for (fname, fdata) in self.source_files.items():
259             for retries in range(maxRetries):
260                 try:
261                     self.dots = 0
262                     local_file = os.path.join(self.config.src_dir, fdata['filename'])
263                     url = fdata['url']
264                     print 'Downloading %s:' % fname,
265                     if retries > 0:
266                         print '(retry)',
267                     sys.stdout.flush()
268
269                     completed = False
270                     if os.path.exists(local_file):
271                         md5_pass = self.checkHash(fdata)
272                         if md5_pass:
273                             print '[md5 match]',
274                         else:
275                             print '[md5 mismatch]',
276                         sys.stdout.flush()
277                         completed = md5_pass
278
279                     if not completed:
280                         urllib.urlretrieve(url, local_file, progress)
281
282                     #
283                     # BUGBUG: Suggest proxy to user if download fails.
284                     #
285                     # export http_proxy=http://proxyservername.mycompany.com:911
286                     # export ftp_proxy=http://proxyservername.mycompany.com:911
287
288                     if not completed and os.path.exists(local_file):
289                         md5_pass = self.checkHash(fdata)
290                         if md5_pass:
291                             print '[md5 match]',
292                         else:
293                             print '[md5 mismatch]',
294                         sys.stdout.flush()
295                         completed = md5_pass
296
297                     if completed:
298                         print '[done]'
299                         break
300                     else:
301                         print '[failed]'
302                         print '  Tried to retrieve', url
303                         print '  to', local_file
304                         print 'Possible fixes:'
305                         print '* If you are behind a web-proxy, try setting the',
306                         print 'http_proxy environment variable'
307                         print '* You can try to download this file separately',
308                         print 'and rerun this script'
309                         raise Exception()
310                 
311                 except KeyboardInterrupt:
312                     print '[KeyboardInterrupt]'
313                     return False
314
315                 except:
316                     pass
317
318             if not completed: return False
319
320         return True
321
322     def checkHash(self, fdata):
323         local_file = os.path.join(self.config.src_dir, fdata['filename'])
324         expect_md5 = fdata['md5']
325         data = open(local_file).read()
326         md5sum = md5()
327         md5sum.update(data)
328         return md5sum.hexdigest().lower() == expect_md5.lower()
329
330     def GetModules(self):
331         return self.source_files.keys()
332
333     def GetFilenameOf(self, module):
334         return self.source_files[module]['filename']
335
336     def GetMd5Of(self, module):
337         return self.source_files[module]['md5']
338
339     def GetExtractDirOf(self, module):
340         return self.source_files[module]['extract-dir']
341
342     def GetAdditionalParameters(self, module, step):
343         key = step + '-params'
344         if key in self.source_files[module]:
345             return self.source_files[module][key]
346         else:
347             return tuple()
348
349 class Extracter:
350     """class Extracter
351
352     Handles the extraction of the source files from their downloaded
353     archive files.
354     """
355
356     def __init__(self, source_files, config):
357         self.source_files = source_files
358         self.config = config
359
360     def Extract(self, module):
361         src = self.config.src_dir
362         extractDst = os.path.join(src, self.config.arch)
363         local_file = os.path.join(src, self.source_files.GetFilenameOf(module))
364         moduleMd5 = self.source_files.GetMd5Of(module)
365         extracted = os.path.join(extractDst, os.path.split(local_file)[1] + '.extracted')
366         if not os.path.exists(extractDst):
367             os.mkdir(extractDst)
368
369         extractedMd5 = None
370         if os.path.exists(extracted):
371             extractedMd5 = open(extracted).read()
372
373         if extractedMd5 != moduleMd5:
374             print 'Extracting %s:' % self.config.Relative(local_file)
375             tar = tarfile.open(local_file)
376             tar.extractall(extractDst)
377             open(extracted, 'w').write(moduleMd5)
378         else:
379             pass
380             #print 'Previously extracted', self.config.Relative(local_file)
381
382     def ExtractAll(self):
383         for module in self.source_files.GetModules():
384             self.Extract(module)
385
386 class Builder:
387     """class Builder
388
389     Builds and installs the GCC tool suite.
390     """
391
392     def __init__(self, source_files, config):
393         self.source_files = source_files
394         self.config = config
395
396     def Build(self):
397         self.BuildModule('binutils')
398         self.CopyIncludeDirectory()
399         self.BuildModule('gcc')
400         self.MakeSymLinks()
401
402     def IsBuildStepComplete(self, step):
403         return \
404             os.path.exists(
405                 os.path.join(
406                     self.config.build_dir, self.config.arch, step + '.completed'
407                     )
408                 )
409
410     def MarkBuildStepComplete(self, step):
411         open(
412             os.path.join(
413                 self.config.build_dir, self.config.arch, step + '.completed'
414                 ),
415             "w"
416             ).close()
417
418     def CopyIncludeDirectory(self):
419         linkdst = os.path.join(self.config.prefix, 'mingw')
420         src = os.path.join(
421             self.config.src_dir,
422             self.config.arch,
423             self.source_files.GetExtractDirOf('mingw_hdr'),
424             'include'
425             )
426         dst_parent = os.path.join(self.config.prefix, self.config.target_combo)
427         dst = os.path.join(dst_parent, 'include')
428         if not os.path.exists(dst):
429             if not os.path.exists(dst_parent):
430                 os.makedirs(dst_parent)
431             print 'Copying headers to', self.config.Relative(dst)
432             shutil.copytree(src, dst, True)
433         if not os.path.lexists(linkdst):
434             print 'Making symlink at', self.config.Relative(linkdst)
435             os.symlink(self.config.target_combo, linkdst)
436
437     def BuildModule(self, module):
438         base_dir = os.getcwd()
439         build_dir = os.path.join(self.config.build_dir, self.config.arch, module)
440         module_dir = self.source_files.GetExtractDirOf(module)
441         module_dir = os.path.realpath(os.path.join('src', self.config.arch, module_dir))
442         configure = os.path.join(module_dir, 'configure')
443         prefix = self.config.prefix
444         if not os.path.exists(build_dir):
445             os.makedirs(build_dir)
446         os.chdir(build_dir)
447
448         cmd = (
449             configure,
450             '--target=%s' % self.config.target_combo,
451             '--prefix=' + prefix,
452             '--with-sysroot=' + prefix,
453             '--disable-werror',
454             )
455         if os.path.exists('/opt/local/include/gmp.h'):
456             cmd += ('--with-gmp=/opt/local',)
457         if module == 'gcc': cmd += ('--oldincludedir=/opt/local/include',)
458         cmd += self.source_files.GetAdditionalParameters(module, 'configure')
459         self.RunCommand(cmd, module, 'config', skipable=True)
460
461         cmd = ('make',)
462         if module == 'gcc':
463             cmd += ('all-gcc',)
464         self.RunCommand(cmd, module, 'build')
465
466         cmd = ('make',)
467         if module == 'gcc':
468             cmd += ('install-gcc',)
469         else:
470             cmd += ('install',)
471         self.RunCommand(cmd, module, 'install')
472
473         os.chdir(base_dir)
474
475         print '%s module is now built and installed' % module
476
477     def RunCommand(self, cmd, module, stage, skipable=False):
478         if skipable:
479             if self.IsBuildStepComplete('%s.%s' % (module, stage)):
480                 return
481
482         popen = lambda cmd: \
483             subprocess.Popen(
484                 cmd,
485                 stdin=subprocess.PIPE,
486                 stdout=subprocess.PIPE,
487                 stderr=subprocess.STDOUT
488                 )
489
490         print '%s [%s] ...' % (module, stage),
491         sys.stdout.flush()
492         p = popen(cmd)
493         output = p.stdout.read()
494         p.wait()
495         if p.returncode != 0:
496             print '[failed!]'
497             logFile = os.path.join(self.config.build_dir, 'log.txt')
498             f = open(logFile, "w")
499             f.write(output)
500             f.close()
501             raise Exception, 'Failed to %s %s\n' % (stage, module) + \
502                 'See output log at %s' % self.config.Relative(logFile)
503         else:
504             print '[done]'
505
506         if skipable:
507             self.MarkBuildStepComplete('%s.%s' % (module, stage))
508
509     def MakeSymLinks(self):
510         links_dir = os.path.join(self.config.symlinks, self.config.arch)
511         if not os.path.exists(links_dir):
512             os.makedirs(links_dir)
513         startPrinted = False
514         for link in ('ar', 'ld', 'gcc'):
515             src = os.path.join(
516                 self.config.prefix, 'bin', self.config.target_combo + '-' + link
517                 )
518             linkdst = os.path.join(links_dir, link)
519             if not os.path.lexists(linkdst):
520                 if not startPrinted:
521                     print 'Making symlinks in %s:' % self.config.Relative(links_dir),
522                     startPrinted = True
523                 print link,
524                 os.symlink(src, linkdst)
525
526         if startPrinted:
527             print '[done]'
528
529 class App:
530     """class App
531
532     The main body of the application.
533     """
534
535     def __init__(self):
536         config = Config()
537
538         if not config.IsConfigOk():
539             return
540
541         config.MakeDirs()
542
543         sources = SourceFiles(config)
544         result = sources.GetAll()
545         if result:
546             print 'All files have been downloaded & verified'
547         else:
548             print 'An error occured while downloading a file'
549             return
550
551         Extracter(sources, config).ExtractAll()
552
553         Builder(sources, config).Build()
554
555 App()
556