BaseTools/gcc/mingw-gcc-build.py:
[people/mcb30/basetools.git] / gcc / mingw-gcc-build.py
1 #!/usr/bin/env python
2
3 #
4 # Automation of instructions from:
5 #   http://mingw-w64.svn.sourceforge.net/viewvc/mingw-w64/trunk/mingw-w64-doc/
6 #     howto-build/mingw-w64-howto-build.txt?revision=216&view=markup
7 #
8
9 from optparse import OptionParser
10 import os
11 import shutil
12 import subprocess
13 import sys
14 import tarfile
15 import urllib
16 import urlparse
17 try:
18     from hashlib import md5
19 except Exception:
20     from md5 import md5
21
22 #
23 # Version and Copyright
24 #
25 VersionNumber = "0.01"
26 __version__ = "%prog Version " + VersionNumber
27 __copyright__ = "Copyright (c) 2008, Intel Corporation.  All rights reserved."
28
29 class Config:
30     """class Config
31
32     Stores the configuration options for the rest of the script.
33
34     Handles the command line options, and allows the code within
35     the script to easily interact with the 'config' requested by
36     the user.
37     """
38
39     def __init__(self):
40         self.base_dir = os.getcwd()
41         (self.options, self.args) = self.CheckOptions()
42         self.__init_dirs__()
43
44     def CheckOptions(self):
45         Parser = \
46             OptionParser(
47                 description=__copyright__,
48                 version=__version__,
49                 prog="mingw-gcc-build",
50                 usage="%prog [options] [target]"
51                 )
52         Parser.add_option(
53             "--arch",
54             action = "store", type = "string",
55             default = None,
56             dest = "arch",
57             help = "Processor architecture to build gcc for."
58             )
59         Parser.add_option(
60             "--src-dir",
61             action = "store", type = "string", dest = "src_dir",
62             default = os.path.join(self.base_dir, 'src'),
63             help = "Directory to download/extract binutils/gcc sources"
64             )
65         Parser.add_option(
66             "--build-dir",
67             action = "store", type = "string", dest = "build_dir",
68             default = os.path.join(self.base_dir, 'build'),
69             help = "Directory to download/extract binutils/gcc sources"
70             )
71         Parser.add_option(
72             "--prefix",
73             action = "store", type = "string", dest = "prefix",
74             default = os.path.join(self.base_dir, 'install'),
75             help = "Prefix to install binutils/gcc into"
76             )
77         Parser.add_option(
78             "--symlinks",
79             action = "store", type = "string", dest = "symlinks",
80             default = os.path.join(self.base_dir, 'symlinks'),
81             help = "Directory to create binutils/gcc symbolic links into."
82             )
83         Parser.add_option(
84             "-v", "--verbose",
85             action="store_true",
86             type=None, help="Print verbose messages"
87             )
88
89         (Opt, Args) = Parser.parse_args()
90
91         self.arch = Opt.arch.lower()
92         if self.arch not in ('ia32', 'x64'):
93             Parser.error('Please use --arch to specify the architecture')
94         self.target_arch = {'ia32': 'i686', 'x64': 'x86_64'}[self.arch]
95
96         return (Opt, Args)
97
98     def __init_dirs__(self):
99         self.src_dir = os.path.realpath(os.path.expanduser(self.options.src_dir))
100         self.build_dir = os.path.realpath(os.path.expanduser(self.options.build_dir))
101         self.prefix = os.path.realpath(os.path.expanduser(self.options.prefix))
102         self.symlinks = os.path.realpath(os.path.expanduser(self.options.symlinks))
103
104     def IsConfigOk(self):
105                 
106         print "Current directory:"
107         print "   ", self.base_dir
108         print "Sources download/extraction:", self.Relative(self.src_dir)
109         print "Build directory            :", self.Relative(self.build_dir)
110         print "Prefix (install) directory :", self.Relative(self.prefix)
111         print "Create symlinks directory  :", self.Relative(self.symlinks)
112         print
113         answer = raw_input("Is this configuration ok? (default = no): ")
114         if (answer.lower() not in ('y', 'yes')):
115             print
116             print "Please try using --help and then change the configuration."
117             return False
118         
119         print
120         return True
121
122     def Relative(self, path):
123         if path.startswith(self.base_dir):
124             return '.' + path[len(self.base_dir):]
125         return path
126
127     def MakeDirs(self):
128         for path in (self.src_dir, self.build_dir,self.prefix, self.symlinks):
129             if not os.path.exists(path):
130                 os.makedirs(path)
131
132 class SourceFiles:
133     """class SourceFiles
134
135     Handles the downloading of source files used by the script.
136     """
137
138     def __init__(self, config):
139         self.config = config
140         self.source_files = self.source_files[config.arch]
141
142     source_files_common = {
143         'binutils': {
144             'url': 'http://www.kernel.org/pub/linux/devel/binutils/' + \
145                    'binutils-$version.tar.bz2',
146             'version': '2.18.50.0.5',
147             'md5': 'daee18dbbf0a6ccfc186141bee18bf62',
148             },
149         }
150
151     source_files_x64 = {
152         'gcc': {
153             'url': 'http://gcc-ca.internet.bs/releases/' + \
154                    'gcc-$version/gcc-$version.tar.bz2',
155             'version': '4.3.0',
156             'md5': '197ed8468b38db1d3481c3111691d85b',
157             },
158         'mingw_hdr': {
159             'url': 'http://superb-west.dl.sourceforge.net/sourceforge/' + \
160                    'mingw-w64/mingw-w64-snapshot-$version.tar.bz2',
161             'version': '20080310',
162             'md5': '235b2d15c2411f7d213c0c0977b2162f',
163             },
164         }
165
166     source_files_ia32 = {
167         'gcc': {
168             'url': 'http://superb-east.dl.sourceforge.net/sourceforge/' + \
169                    'mingw/gcc-$version-mingw-$minor_version-src.tar.gz',
170             'version': '4.3.0',
171             'minor_version': 'alpha-20080403',
172             'extract-dir': 'gcc-$version',
173             'md5': '27961d80e304f4ef32c980833c6e8e44',
174             'configure-params': ('--with-gnu-as', '--with-gnu-ld', '--with-newlib',
175                                  '--verbose', '--disable-libssp', '--disable-nls',
176                                  '--enable-languages=c'
177                                 )
178             },
179         }
180
181     source_files = {
182         'ia32': [source_files_common, source_files_ia32],
183         'x64': [source_files_common, source_files_x64],
184         }
185
186     for arch in source_files:
187         mergedSourceFiles = {}
188         for source_files_dict in source_files[arch]:
189             mergedSourceFiles.update(source_files_dict)
190         for downloadItem in mergedSourceFiles:
191             fdata = mergedSourceFiles[downloadItem]
192             fdata['filename'] = fdata['url'].split('/')[-1]
193             if 'extract-dir' not in fdata:
194                 for ext in ('.tar.gz', '.tar.bz2', '.zip'):
195                     if fdata['filename'].endswith(ext):
196                         fdata['extract-dir'] = fdata['filename'][:-len(ext)]
197                         break
198             replaceables = ('extract-dir', 'filename', 'url')
199             for replaceItem in fdata:
200                 if replaceItem in replaceables: continue
201                 if type(fdata[replaceItem]) != str: continue
202                 for replaceable in replaceables:
203                     if type(fdata[replaceable]) != str: continue
204                     if replaceable in fdata:
205                         fdata[replaceable] = \
206                             fdata[replaceable].replace(
207                                 '$' + replaceItem,
208                                 fdata[replaceItem]
209                                 )
210         source_files[arch] = mergedSourceFiles
211     #print 'source_files:', source_files
212
213     def GetAll(self):
214
215         def progress(received, blockSize, fileSize):
216             if fileSize < 0: return
217             wDots = (100 * received * blockSize) / fileSize / 10
218             if wDots > self.dots:
219                 for i in range(wDots - self.dots):
220                     print '.',
221                     sys.stdout.flush()
222                     self.dots += 1
223
224         for (fname, fdata) in self.source_files.items():
225             self.dots = 0
226             local_file = os.path.join(self.config.src_dir, fdata['filename'])
227             url = fdata['url']
228             print 'Downloading %s:' % fname,
229             sys.stdout.flush()
230
231             completed = False
232             if os.path.exists(local_file):
233                 md5_pass = self.checkHash(fdata)
234                 if md5_pass:
235                     print '[md5 match]',
236                 else:
237                     print '[md5 mismatch]',
238                 sys.stdout.flush()
239                 completed = md5_pass
240
241             if not completed:
242                 urllib.urlretrieve(url, local_file, progress)
243
244             #
245             # BUGBUG: Suggest proxy to user if download fails.
246             #
247             # export http_proxy=http://proxyservername.mycompany.com:911
248             # export ftp_proxy=http://proxyservername.mycompany.com:911
249
250             if not completed and os.path.exists(local_file):
251                 md5_pass = self.checkHash(fdata)
252                 if md5_pass:
253                     print '[md5 match]',
254                 else:
255                     print '[md5 mismatch]',
256                 sys.stdout.flush()
257                 completed = md5_pass
258
259             if completed:
260                 print '[done]'
261             else:
262                 print '[failed]'
263                 return False
264
265         return True
266
267     def checkHash(self, fdata):
268         local_file = os.path.join(self.config.src_dir, fdata['filename'])
269         expect_md5 = fdata['md5']
270         data = open(local_file).read()
271         md5sum = md5()
272         md5sum.update(data)
273         return md5sum.hexdigest().lower() == expect_md5.lower()
274
275     def GetModules(self):
276         return self.source_files.keys()
277
278     def GetFilenameOf(self, module):
279         return self.source_files[module]['filename']
280
281     def GetMd5Of(self, module):
282         return self.source_files[module]['md5']
283
284     def GetExtractDirOf(self, module):
285         return self.source_files[module]['extract-dir']
286
287     def GetAdditionalParameters(self, module, step):
288         key = step + '-params'
289         if key in self.source_files[module]:
290             return self.source_files[module][key]
291         else:
292             return tuple()
293
294 class Extracter:
295     """class Extracter
296
297     Handles the extraction of the source files from their downloaded
298     archive files.
299     """
300
301     def __init__(self, source_files, config):
302         self.source_files = source_files
303         self.config = config
304
305     def Extract(self, module):
306         src = self.config.src_dir
307         extractDst = os.path.join(src, self.config.arch)
308         local_file = os.path.join(src, self.source_files.GetFilenameOf(module))
309         moduleMd5 = self.source_files.GetMd5Of(module)
310         extracted = os.path.join(extractDst, os.path.split(local_file)[1] + '.extracted')
311         if not os.path.exists(extractDst):
312             os.mkdir(extractDst)
313
314         extractedMd5 = None
315         if os.path.exists(extracted):
316             extractedMd5 = open(extracted).read()
317
318         if extractedMd5 != moduleMd5:
319             print 'Extracting %s:' % self.config.Relative(local_file)
320             tar = tarfile.open(local_file)
321             tar.extractall(extractDst)
322             open(extracted, 'w').write(moduleMd5)
323         else:
324             pass
325             #print 'Previously extracted', self.config.Relative(local_file)
326
327     def ExtractAll(self):
328         for module in self.source_files.GetModules():
329             self.Extract(module)
330
331 class Builder:
332     """class Builder
333
334     Builds and installs the GCC tool suite.
335     """
336
337     def __init__(self, source_files, config):
338         self.source_files = source_files
339         self.config = config
340
341     def Build(self):
342         self.BuildModule('binutils')
343         self.CopyIncludeDirectory()
344         self.BuildModule('gcc')
345         self.MakeSymLinks()
346
347     def IsBuildStepComplete(self, step):
348         return \
349             os.path.exists(
350                 os.path.join(
351                     self.config.build_dir, self.config.arch, step + '.completed'
352                     )
353                 )
354
355     def MarkBuildStepComplete(self, step):
356         open(
357             os.path.join(
358                 self.config.build_dir, self.config.arch, step + '.completed'
359                 ),
360             "w"
361             ).close()
362
363     def CopyIncludeDirectory(self):
364         linkdst = os.path.join(self.config.prefix, 'mingw')
365         if self.config.arch == 'x64':
366             src = os.path.join(self.config.src_dir, 'trunk', 'mingw-w64-headers', 'include')
367             dst_parent = os.path.join(self.config.prefix, self.config.target_arch + '-pc-mingw32')
368             dst = os.path.join(dst_parent, 'include')
369             if not os.path.exists(dst):
370                 if not os.path.exists(dst_parent):
371                     os.makedirs(dst_parent)
372                 print 'Copying headers to', self.config.Relative(dst)
373                 shutil.copytree(src, dst, True)
374         if not os.path.lexists(linkdst):
375             print 'Making symlink at', self.config.Relative(linkdst)
376             os.symlink(self.config.target_arch + '-pc-mingw32', linkdst)
377
378     def BuildModule(self, module):
379         base_dir = os.getcwd()
380         build_dir = os.path.join(self.config.build_dir, self.config.arch, module)
381         module_dir = self.source_files.GetExtractDirOf(module)
382         module_dir = os.path.realpath(os.path.join('src', self.config.arch, module_dir))
383         configure = os.path.join(module_dir, 'configure')
384         prefix = self.config.prefix
385         if not os.path.exists(build_dir):
386             os.makedirs(build_dir)
387         os.chdir(build_dir)
388
389         cmd = (
390             configure,
391             '--target=%s-pc-mingw32' % self.config.target_arch,
392             '--prefix=' + prefix,
393             '--with-sysroot=' + prefix,
394             )
395         if os.path.exists('/opt/local/include/gmp.h'):
396             cmd += ('--with-gmp=/opt/local',)
397         cmd += self.source_files.GetAdditionalParameters(module, 'configure')
398         self.RunCommand(cmd, module, 'config', skipable=True)
399
400         cmd = ('make',)
401         if module == 'gcc':
402             cmd += ('all-gcc',)
403         self.RunCommand(cmd, module, 'build')
404
405         cmd = ('make',)
406         if module == 'gcc':
407             cmd += ('install-gcc',)
408         else:
409             cmd += ('install',)
410         self.RunCommand(cmd, module, 'install')
411
412         os.chdir(base_dir)
413
414         print '%s module is now built and installed' % module
415
416     def RunCommand(self, cmd, module, stage, skipable=False):
417         if skipable:
418             if self.IsBuildStepComplete('%s.%s' % (module, stage)):
419                 return
420
421         popen = lambda cmd: \
422             subprocess.Popen(
423                 cmd,
424                 stdin=subprocess.PIPE,
425                 stdout=subprocess.PIPE,
426                 stderr=subprocess.STDOUT
427                 )
428
429         print '%s [%s] ...' % (module, stage),
430         sys.stdout.flush()
431         p = popen(cmd)
432         output = p.stdout.read()
433         p.wait()
434         if p.returncode != 0:
435             print '[failed!]'
436             logFile = os.path.join(self.config.build_dir, 'log.txt')
437             f = open(logFile, "w")
438             f.write(output)
439             f.close()
440             raise Exception, 'Failed to %s %s\n' % (stage, module) + \
441                 'See output log at %s' % self.config.Relative(logFile)
442         else:
443             print '[done]'
444
445         if skipable:
446             self.MarkBuildStepComplete('%s.%s' % (module, stage))
447
448     def MakeSymLinks(self):
449         links_dir = os.path.join(self.config.symlinks, self.config.arch)
450         if not os.path.exists(links_dir):
451             os.makedirs(links_dir)
452         startPrinted = False
453         for link in ('ar', 'ld', 'gcc'):
454             src = os.path.join(
455                 self.config.prefix, 'bin', self.config.target_arch + '-pc-mingw32-' + link
456                 )
457             linkdst = os.path.join(links_dir, link)
458             if not os.path.lexists(linkdst):
459                 if not startPrinted:
460                     print 'Making symlinks in %s:' % self.config.Relative(links_dir),
461                     startPrinted = True
462                 print link,
463                 os.symlink(src, linkdst)
464
465         if startPrinted:
466             print '[done]'
467
468 class App:
469     """class App
470
471     The main body of the application.
472     """
473
474     def __init__(self):
475         config = Config()
476
477         if not config.IsConfigOk():
478             return
479
480         config.MakeDirs()
481
482         sources = SourceFiles(config)
483         result = sources.GetAll()
484         if result:
485             print 'All files have been downloaded & verified'
486         else:
487             print 'An error occured while downloading a file'
488             return
489
490         Extracter(sources, config).ExtractAll()
491
492         Builder(sources, config).Build()
493
494 App()
495