Fixed two issues, XmlSpecification default is 1.1 for UEFI and removed required heade...
[people/mcb30/basetools.git] / gcc / mingw-gcc-build.py
1 #!/usr/bin/env python
2
3 #
4 # Automation of instructions from:
5 #   http://mingw-w64.svn.sourceforge.net/viewvc/mingw-w64/trunk/mingw-w64-doc/
6 #     howto-build/mingw-w64-howto-build.txt?revision=216&view=markup
7 #
8
9 from optparse import OptionParser
10 import os
11 import shutil
12 import subprocess
13 import sys
14 import tarfile
15 import urllib
16 import urlparse
17 try:
18     from hashlib import md5
19 except Exception:
20     from md5 import md5
21
22 #
23 # Version and Copyright
24 #
25 VersionNumber = "0.01"
26 __version__ = "%prog Version " + VersionNumber
27 __copyright__ = "Copyright (c) 2008, Intel Corporation.  All rights reserved."
28
29 class Config:
30     """class Config
31
32     Stores the configuration options for the rest of the script.
33
34     Handles the command line options, and allows the code within
35     the script to easily interact with the 'config' requested by
36     the user.
37     """
38
39     def __init__(self):
40         self.base_dir = os.getcwd()
41         (self.options, self.args) = self.CheckOptions()
42         self.__init_dirs__()
43
44     def CheckOptions(self):
45         Parser = \
46             OptionParser(
47                 description=__copyright__,
48                 version=__version__,
49                 prog="mingw-gcc-build",
50                 usage="%prog [options] [target]"
51                 )
52         Parser.add_option(
53             "--arch",
54             action = "store", type = "string",
55             default = '',
56             dest = "arch",
57             help = "Processor architecture to build gcc for."
58             )
59         Parser.add_option(
60             "--src-dir",
61             action = "store", type = "string", dest = "src_dir",
62             default = os.path.join(self.base_dir, 'src'),
63             help = "Directory to download/extract binutils/gcc sources"
64             )
65         Parser.add_option(
66             "--build-dir",
67             action = "store", type = "string", dest = "build_dir",
68             default = os.path.join(self.base_dir, 'build'),
69             help = "Directory to download/extract binutils/gcc sources"
70             )
71         Parser.add_option(
72             "--prefix",
73             action = "store", type = "string", dest = "prefix",
74             default = os.path.join(self.base_dir, 'install'),
75             help = "Prefix to install binutils/gcc into"
76             )
77         Parser.add_option(
78             "--symlinks",
79             action = "store", type = "string", dest = "symlinks",
80             default = os.path.join(self.base_dir, 'symlinks'),
81             help = "Directory to create binutils/gcc symbolic links into."
82             )
83         Parser.add_option(
84             "-v", "--verbose",
85             action="store_true",
86             type=None, help="Print verbose messages"
87             )
88
89         (Opt, Args) = Parser.parse_args()
90
91         self.arch = Opt.arch.lower()
92         allowedArchs = ('ia32', 'x64')
93         if self.arch not in allowedArchs:
94             Parser.error(
95                 'Please use --arch to specify one of: %s' %
96                     ', '.join(allowedArchs)
97                 )
98         self.target_arch = {'ia32': 'i686', 'x64': 'x86_64'}[self.arch]
99
100         return (Opt, Args)
101
102     def __init_dirs__(self):
103         self.src_dir = os.path.realpath(os.path.expanduser(self.options.src_dir))
104         self.build_dir = os.path.realpath(os.path.expanduser(self.options.build_dir))
105         self.prefix = os.path.realpath(os.path.expanduser(self.options.prefix))
106         self.symlinks = os.path.realpath(os.path.expanduser(self.options.symlinks))
107
108     def IsConfigOk(self):
109                 
110         print "Current directory:"
111         print "   ", self.base_dir
112         print "Sources download/extraction:", self.Relative(self.src_dir)
113         print "Build directory            :", self.Relative(self.build_dir)
114         print "Prefix (install) directory :", self.Relative(self.prefix)
115         print "Create symlinks directory  :", self.Relative(self.symlinks)
116         print
117         answer = raw_input("Is this configuration ok? (default = no): ")
118         if (answer.lower() not in ('y', 'yes')):
119             print
120             print "Please try using --help and then change the configuration."
121             return False
122         
123         print
124         return True
125
126     def Relative(self, path):
127         if path.startswith(self.base_dir):
128             return '.' + path[len(self.base_dir):]
129         return path
130
131     def MakeDirs(self):
132         for path in (self.src_dir, self.build_dir,self.prefix, self.symlinks):
133             if not os.path.exists(path):
134                 os.makedirs(path)
135
136 class SourceFiles:
137     """class SourceFiles
138
139     Handles the downloading of source files used by the script.
140     """
141
142     def __init__(self, config):
143         self.config = config
144         self.source_files = self.source_files[config.arch]
145
146     source_files_common = {
147         'binutils': {
148             'url': 'http://www.kernel.org/pub/linux/devel/binutils/' + \
149                    'binutils-$version.tar.bz2',
150             'version': '2.18.50.0.5',
151             'md5': 'daee18dbbf0a6ccfc186141bee18bf62',
152             },
153         }
154
155     source_files_x64 = {
156         'gcc': {
157             'url': 'http://gcc-ca.internet.bs/releases/' + \
158                    'gcc-$version/gcc-$version.tar.bz2',
159             'version': '4.3.0',
160             'md5': '197ed8468b38db1d3481c3111691d85b',
161             },
162         'mingw_hdr': {
163             'url': 'http://superb-west.dl.sourceforge.net/sourceforge/' + \
164                    'mingw-w64/mingw-w64-snapshot-$version.tar.bz2',
165             'extract-dir': os.path.join('trunk', 'mingw-w64-headers'),
166             'version': '20080310',
167             'md5': '235b2d15c2411f7d213c0c0977b2162f',
168             },
169         }
170
171     source_files_ia32 = {
172         'gcc': {
173             'url': 'http://superb-east.dl.sourceforge.net/sourceforge/' + \
174                    'mingw/gcc-$version-mingw-$minor_version-src.tar.gz',
175             'version': '4.3.0',
176             'minor_version': 'alpha-20080403',
177             'extract-dir': 'gcc-$version',
178             'md5': '27961d80e304f4ef32c980833c6e8e44',
179             'configure-params': ('--with-gnu-as', '--with-gnu-ld', '--with-newlib',
180                                  '--verbose', '--disable-libssp', '--disable-nls',
181                                  '--enable-languages=c,c++'
182                                 )
183             },
184         'mingw_hdr': {
185             'url': 'http://superb-west.dl.sourceforge.net/sourceforge/' + \
186                    'mingw/mingw-runtime-$version-src.tar.gz',
187             'extract-dir': 'mingw-runtime-$version',
188             'version': '3.14',
189             'md5': '7d049a8331efcfe34600c0cda6934ac6',
190             },
191         }
192
193     source_files = {
194         'ia32': [source_files_common, source_files_ia32],
195         'x64': [source_files_common, source_files_x64],
196         }
197
198     for arch in source_files:
199         mergedSourceFiles = {}
200         for source_files_dict in source_files[arch]:
201             mergedSourceFiles.update(source_files_dict)
202         for downloadItem in mergedSourceFiles:
203             fdata = mergedSourceFiles[downloadItem]
204             fdata['filename'] = fdata['url'].split('/')[-1]
205             if 'extract-dir' not in fdata:
206                 for ext in ('.tar.gz', '.tar.bz2', '.zip'):
207                     if fdata['filename'].endswith(ext):
208                         fdata['extract-dir'] = fdata['filename'][:-len(ext)]
209                         break
210             replaceables = ('extract-dir', 'filename', 'url')
211             for replaceItem in fdata:
212                 if replaceItem in replaceables: continue
213                 if type(fdata[replaceItem]) != str: continue
214                 for replaceable in replaceables:
215                     if type(fdata[replaceable]) != str: continue
216                     if replaceable in fdata:
217                         fdata[replaceable] = \
218                             fdata[replaceable].replace(
219                                 '$' + replaceItem,
220                                 fdata[replaceItem]
221                                 )
222         source_files[arch] = mergedSourceFiles
223     #print 'source_files:', source_files
224
225     def GetAll(self):
226
227         def progress(received, blockSize, fileSize):
228             if fileSize < 0: return
229             wDots = (100 * received * blockSize) / fileSize / 10
230             if wDots > self.dots:
231                 for i in range(wDots - self.dots):
232                     print '.',
233                     sys.stdout.flush()
234                     self.dots += 1
235
236         for (fname, fdata) in self.source_files.items():
237             self.dots = 0
238             local_file = os.path.join(self.config.src_dir, fdata['filename'])
239             url = fdata['url']
240             print 'Downloading %s:' % fname,
241             sys.stdout.flush()
242
243             completed = False
244             if os.path.exists(local_file):
245                 md5_pass = self.checkHash(fdata)
246                 if md5_pass:
247                     print '[md5 match]',
248                 else:
249                     print '[md5 mismatch]',
250                 sys.stdout.flush()
251                 completed = md5_pass
252
253             if not completed:
254                 urllib.urlretrieve(url, local_file, progress)
255
256             #
257             # BUGBUG: Suggest proxy to user if download fails.
258             #
259             # export http_proxy=http://proxyservername.mycompany.com:911
260             # export ftp_proxy=http://proxyservername.mycompany.com:911
261
262             if not completed and os.path.exists(local_file):
263                 md5_pass = self.checkHash(fdata)
264                 if md5_pass:
265                     print '[md5 match]',
266                 else:
267                     print '[md5 mismatch]',
268                 sys.stdout.flush()
269                 completed = md5_pass
270
271             if completed:
272                 print '[done]'
273             else:
274                 print '[failed]'
275                 return False
276
277         return True
278
279     def checkHash(self, fdata):
280         local_file = os.path.join(self.config.src_dir, fdata['filename'])
281         expect_md5 = fdata['md5']
282         data = open(local_file).read()
283         md5sum = md5()
284         md5sum.update(data)
285         return md5sum.hexdigest().lower() == expect_md5.lower()
286
287     def GetModules(self):
288         return self.source_files.keys()
289
290     def GetFilenameOf(self, module):
291         return self.source_files[module]['filename']
292
293     def GetMd5Of(self, module):
294         return self.source_files[module]['md5']
295
296     def GetExtractDirOf(self, module):
297         return self.source_files[module]['extract-dir']
298
299     def GetAdditionalParameters(self, module, step):
300         key = step + '-params'
301         if key in self.source_files[module]:
302             return self.source_files[module][key]
303         else:
304             return tuple()
305
306 class Extracter:
307     """class Extracter
308
309     Handles the extraction of the source files from their downloaded
310     archive files.
311     """
312
313     def __init__(self, source_files, config):
314         self.source_files = source_files
315         self.config = config
316
317     def Extract(self, module):
318         src = self.config.src_dir
319         extractDst = os.path.join(src, self.config.arch)
320         local_file = os.path.join(src, self.source_files.GetFilenameOf(module))
321         moduleMd5 = self.source_files.GetMd5Of(module)
322         extracted = os.path.join(extractDst, os.path.split(local_file)[1] + '.extracted')
323         if not os.path.exists(extractDst):
324             os.mkdir(extractDst)
325
326         extractedMd5 = None
327         if os.path.exists(extracted):
328             extractedMd5 = open(extracted).read()
329
330         if extractedMd5 != moduleMd5:
331             print 'Extracting %s:' % self.config.Relative(local_file)
332             tar = tarfile.open(local_file)
333             tar.extractall(extractDst)
334             open(extracted, 'w').write(moduleMd5)
335         else:
336             pass
337             #print 'Previously extracted', self.config.Relative(local_file)
338
339     def ExtractAll(self):
340         for module in self.source_files.GetModules():
341             self.Extract(module)
342
343 class Builder:
344     """class Builder
345
346     Builds and installs the GCC tool suite.
347     """
348
349     def __init__(self, source_files, config):
350         self.source_files = source_files
351         self.config = config
352
353     def Build(self):
354         self.BuildModule('binutils')
355         self.CopyIncludeDirectory()
356         self.BuildModule('gcc')
357         self.MakeSymLinks()
358
359     def IsBuildStepComplete(self, step):
360         return \
361             os.path.exists(
362                 os.path.join(
363                     self.config.build_dir, self.config.arch, step + '.completed'
364                     )
365                 )
366
367     def MarkBuildStepComplete(self, step):
368         open(
369             os.path.join(
370                 self.config.build_dir, self.config.arch, step + '.completed'
371                 ),
372             "w"
373             ).close()
374
375     def CopyIncludeDirectory(self):
376         linkdst = os.path.join(self.config.prefix, 'mingw')
377         src = os.path.join(
378             self.config.src_dir,
379             self.config.arch,
380             self.source_files.GetExtractDirOf('mingw_hdr'),
381             'include'
382             )
383         dst_parent = os.path.join(self.config.prefix, self.config.target_arch + '-pc-mingw32')
384         dst = os.path.join(dst_parent, 'include')
385         if not os.path.exists(dst):
386             if not os.path.exists(dst_parent):
387                 os.makedirs(dst_parent)
388             print 'Copying headers to', self.config.Relative(dst)
389             shutil.copytree(src, dst, True)
390         if not os.path.lexists(linkdst):
391             print 'Making symlink at', self.config.Relative(linkdst)
392             os.symlink(self.config.target_arch + '-pc-mingw32', linkdst)
393
394     def BuildModule(self, module):
395         base_dir = os.getcwd()
396         build_dir = os.path.join(self.config.build_dir, self.config.arch, module)
397         module_dir = self.source_files.GetExtractDirOf(module)
398         module_dir = os.path.realpath(os.path.join('src', self.config.arch, module_dir))
399         configure = os.path.join(module_dir, 'configure')
400         prefix = self.config.prefix
401         if not os.path.exists(build_dir):
402             os.makedirs(build_dir)
403         os.chdir(build_dir)
404
405         cmd = (
406             configure,
407             '--target=%s-pc-mingw32' % self.config.target_arch,
408             '--prefix=' + prefix,
409             '--with-sysroot=' + prefix,
410             )
411         if os.path.exists('/opt/local/include/gmp.h'):
412             cmd += ('--with-gmp=/opt/local',)
413         cmd += self.source_files.GetAdditionalParameters(module, 'configure')
414         self.RunCommand(cmd, module, 'config', skipable=True)
415
416         cmd = ('make',)
417         if module == 'gcc':
418             cmd += ('all-gcc',)
419         self.RunCommand(cmd, module, 'build')
420
421         cmd = ('make',)
422         if module == 'gcc':
423             cmd += ('install-gcc',)
424         else:
425             cmd += ('install',)
426         self.RunCommand(cmd, module, 'install')
427
428         os.chdir(base_dir)
429
430         print '%s module is now built and installed' % module
431
432     def RunCommand(self, cmd, module, stage, skipable=False):
433         if skipable:
434             if self.IsBuildStepComplete('%s.%s' % (module, stage)):
435                 return
436
437         popen = lambda cmd: \
438             subprocess.Popen(
439                 cmd,
440                 stdin=subprocess.PIPE,
441                 stdout=subprocess.PIPE,
442                 stderr=subprocess.STDOUT
443                 )
444
445         print '%s [%s] ...' % (module, stage),
446         sys.stdout.flush()
447         p = popen(cmd)
448         output = p.stdout.read()
449         p.wait()
450         if p.returncode != 0:
451             print '[failed!]'
452             logFile = os.path.join(self.config.build_dir, 'log.txt')
453             f = open(logFile, "w")
454             f.write(output)
455             f.close()
456             raise Exception, 'Failed to %s %s\n' % (stage, module) + \
457                 'See output log at %s' % self.config.Relative(logFile)
458         else:
459             print '[done]'
460
461         if skipable:
462             self.MarkBuildStepComplete('%s.%s' % (module, stage))
463
464     def MakeSymLinks(self):
465         links_dir = os.path.join(self.config.symlinks, self.config.arch)
466         if not os.path.exists(links_dir):
467             os.makedirs(links_dir)
468         startPrinted = False
469         for link in ('ar', 'ld', 'gcc'):
470             src = os.path.join(
471                 self.config.prefix, 'bin', self.config.target_arch + '-pc-mingw32-' + link
472                 )
473             linkdst = os.path.join(links_dir, link)
474             if not os.path.lexists(linkdst):
475                 if not startPrinted:
476                     print 'Making symlinks in %s:' % self.config.Relative(links_dir),
477                     startPrinted = True
478                 print link,
479                 os.symlink(src, linkdst)
480
481         if startPrinted:
482             print '[done]'
483
484 class App:
485     """class App
486
487     The main body of the application.
488     """
489
490     def __init__(self):
491         config = Config()
492
493         if not config.IsConfigOk():
494             return
495
496         config.MakeDirs()
497
498         sources = SourceFiles(config)
499         result = sources.GetAll()
500         if result:
501             print 'All files have been downloaded & verified'
502         else:
503             print 'An error occured while downloading a file'
504             return
505
506         Extracter(sources, config).ExtractAll()
507
508         Builder(sources, config).Build()
509
510 App()
511