BaseTools/gcc/mingw-gcc-build.py:
[people/mcb30/basetools.git] / gcc / mingw-gcc-build.py
1 #!/usr/bin/env python
2
3 #
4 # Automation of instructions from:
5 #   http://mingw-w64.svn.sourceforge.net/viewvc/mingw-w64/trunk/mingw-w64-doc/
6 #     howto-build/mingw-w64-howto-build.txt?revision=216&view=markup
7 #
8
9 from optparse import OptionParser
10 import os
11 import shutil
12 import subprocess
13 import sys
14 import tarfile
15 import urllib
16 import urlparse
17 try:
18     from hashlib import md5
19 except Exception:
20     from md5 import md5
21
22 #
23 # Version and Copyright
24 #
25 VersionNumber = "0.01"
26 __version__ = "%prog Version " + VersionNumber
27 __copyright__ = "Copyright (c) 2008, Intel Corporation.  All rights reserved."
28
29 class Config:
30     """class Config
31
32     Stores the configuration options for the rest of the script.
33
34     Handles the command line options, and allows the code within
35     the script to easily interact with the 'config' requested by
36     the user.
37     """
38
39     def __init__(self):
40         self.base_dir = os.getcwd()
41         (self.options, self.args) = self.CheckOptions()
42         self.__init_dirs__()
43
44     def CheckOptions(self):
45         Parser = \
46             OptionParser(
47                 description=__copyright__,
48                 version=__version__,
49                 prog="mingw-gcc-build",
50                 usage="%prog [options] [target]"
51                 )
52         Parser.add_option(
53             "--arch",
54             action = "store", type = "string",
55             default = '',
56             dest = "arch",
57             help = "Processor architecture to build gcc for."
58             )
59         Parser.add_option(
60             "--src-dir",
61             action = "store", type = "string", dest = "src_dir",
62             default = os.path.join(self.base_dir, 'src'),
63             help = "Directory to download/extract binutils/gcc sources"
64             )
65         Parser.add_option(
66             "--build-dir",
67             action = "store", type = "string", dest = "build_dir",
68             default = os.path.join(self.base_dir, 'build'),
69             help = "Directory to download/extract binutils/gcc sources"
70             )
71         Parser.add_option(
72             "--prefix",
73             action = "store", type = "string", dest = "prefix",
74             default = os.path.join(self.base_dir, 'install'),
75             help = "Prefix to install binutils/gcc into"
76             )
77         Parser.add_option(
78             "--symlinks",
79             action = "store", type = "string", dest = "symlinks",
80             default = os.path.join(self.base_dir, 'symlinks'),
81             help = "Directory to create binutils/gcc symbolic links into."
82             )
83         Parser.add_option(
84             "-v", "--verbose",
85             action="store_true",
86             type=None, help="Print verbose messages"
87             )
88
89         (Opt, Args) = Parser.parse_args()
90
91         self.arch = Opt.arch.lower()
92         allowedArchs = ('ia32', 'x64')
93         if self.arch not in allowedArchs:
94             Parser.error(
95                 'Please use --arch to specify one of: %s' %
96                     ', '.join(allowedArchs)
97                 )
98         self.target_arch = {'ia32': 'i686', 'x64': 'x86_64'}[self.arch]
99
100         return (Opt, Args)
101
102     def __init_dirs__(self):
103         self.src_dir = os.path.realpath(os.path.expanduser(self.options.src_dir))
104         self.build_dir = os.path.realpath(os.path.expanduser(self.options.build_dir))
105         self.prefix = os.path.realpath(os.path.expanduser(self.options.prefix))
106         self.symlinks = os.path.realpath(os.path.expanduser(self.options.symlinks))
107
108     def IsConfigOk(self):
109                 
110         print "Current directory:"
111         print "   ", self.base_dir
112         print "Sources download/extraction:", self.Relative(self.src_dir)
113         print "Build directory            :", self.Relative(self.build_dir)
114         print "Prefix (install) directory :", self.Relative(self.prefix)
115         print "Create symlinks directory  :", self.Relative(self.symlinks)
116         print
117         answer = raw_input("Is this configuration ok? (default = no): ")
118         if (answer.lower() not in ('y', 'yes')):
119             print
120             print "Please try using --help and then change the configuration."
121             return False
122         
123         print
124         return True
125
126     def Relative(self, path):
127         if path.startswith(self.base_dir):
128             return '.' + path[len(self.base_dir):]
129         return path
130
131     def MakeDirs(self):
132         for path in (self.src_dir, self.build_dir,self.prefix, self.symlinks):
133             if not os.path.exists(path):
134                 os.makedirs(path)
135
136 class SourceFiles:
137     """class SourceFiles
138
139     Handles the downloading of source files used by the script.
140     """
141
142     def __init__(self, config):
143         self.config = config
144         self.source_files = self.source_files[config.arch]
145
146     source_files_common = {
147         'binutils': {
148             'url': 'http://www.kernel.org/pub/linux/devel/binutils/' + \
149                    'binutils-$version.tar.bz2',
150             'version': '2.18.50.0.5',
151             'md5': 'daee18dbbf0a6ccfc186141bee18bf62',
152             },
153         }
154
155     source_files_x64 = {
156         'gcc': {
157             'url': 'http://gcc-ca.internet.bs/releases/' + \
158                    'gcc-$version/gcc-$version.tar.bz2',
159             'version': '4.3.0',
160             'md5': '197ed8468b38db1d3481c3111691d85b',
161             },
162         'mingw_hdr': {
163             'url': 'http://superb-west.dl.sourceforge.net/sourceforge/' + \
164                    'mingw-w64/mingw-w64-snapshot-$version.tar.bz2',
165             'version': '20080310',
166             'md5': '235b2d15c2411f7d213c0c0977b2162f',
167             },
168         }
169
170     source_files_ia32 = {
171         'gcc': {
172             'url': 'http://superb-east.dl.sourceforge.net/sourceforge/' + \
173                    'mingw/gcc-$version-mingw-$minor_version-src.tar.gz',
174             'version': '4.3.0',
175             'minor_version': 'alpha-20080403',
176             'extract-dir': 'gcc-$version',
177             'md5': '27961d80e304f4ef32c980833c6e8e44',
178             'configure-params': ('--with-gnu-as', '--with-gnu-ld', '--with-newlib',
179                                  '--verbose', '--disable-libssp', '--disable-nls',
180                                  '--enable-languages=c'
181                                 )
182             },
183         }
184
185     source_files = {
186         'ia32': [source_files_common, source_files_ia32],
187         'x64': [source_files_common, source_files_x64],
188         }
189
190     for arch in source_files:
191         mergedSourceFiles = {}
192         for source_files_dict in source_files[arch]:
193             mergedSourceFiles.update(source_files_dict)
194         for downloadItem in mergedSourceFiles:
195             fdata = mergedSourceFiles[downloadItem]
196             fdata['filename'] = fdata['url'].split('/')[-1]
197             if 'extract-dir' not in fdata:
198                 for ext in ('.tar.gz', '.tar.bz2', '.zip'):
199                     if fdata['filename'].endswith(ext):
200                         fdata['extract-dir'] = fdata['filename'][:-len(ext)]
201                         break
202             replaceables = ('extract-dir', 'filename', 'url')
203             for replaceItem in fdata:
204                 if replaceItem in replaceables: continue
205                 if type(fdata[replaceItem]) != str: continue
206                 for replaceable in replaceables:
207                     if type(fdata[replaceable]) != str: continue
208                     if replaceable in fdata:
209                         fdata[replaceable] = \
210                             fdata[replaceable].replace(
211                                 '$' + replaceItem,
212                                 fdata[replaceItem]
213                                 )
214         source_files[arch] = mergedSourceFiles
215     #print 'source_files:', source_files
216
217     def GetAll(self):
218
219         def progress(received, blockSize, fileSize):
220             if fileSize < 0: return
221             wDots = (100 * received * blockSize) / fileSize / 10
222             if wDots > self.dots:
223                 for i in range(wDots - self.dots):
224                     print '.',
225                     sys.stdout.flush()
226                     self.dots += 1
227
228         for (fname, fdata) in self.source_files.items():
229             self.dots = 0
230             local_file = os.path.join(self.config.src_dir, fdata['filename'])
231             url = fdata['url']
232             print 'Downloading %s:' % fname,
233             sys.stdout.flush()
234
235             completed = False
236             if os.path.exists(local_file):
237                 md5_pass = self.checkHash(fdata)
238                 if md5_pass:
239                     print '[md5 match]',
240                 else:
241                     print '[md5 mismatch]',
242                 sys.stdout.flush()
243                 completed = md5_pass
244
245             if not completed:
246                 urllib.urlretrieve(url, local_file, progress)
247
248             #
249             # BUGBUG: Suggest proxy to user if download fails.
250             #
251             # export http_proxy=http://proxyservername.mycompany.com:911
252             # export ftp_proxy=http://proxyservername.mycompany.com:911
253
254             if not completed and os.path.exists(local_file):
255                 md5_pass = self.checkHash(fdata)
256                 if md5_pass:
257                     print '[md5 match]',
258                 else:
259                     print '[md5 mismatch]',
260                 sys.stdout.flush()
261                 completed = md5_pass
262
263             if completed:
264                 print '[done]'
265             else:
266                 print '[failed]'
267                 return False
268
269         return True
270
271     def checkHash(self, fdata):
272         local_file = os.path.join(self.config.src_dir, fdata['filename'])
273         expect_md5 = fdata['md5']
274         data = open(local_file).read()
275         md5sum = md5()
276         md5sum.update(data)
277         return md5sum.hexdigest().lower() == expect_md5.lower()
278
279     def GetModules(self):
280         return self.source_files.keys()
281
282     def GetFilenameOf(self, module):
283         return self.source_files[module]['filename']
284
285     def GetMd5Of(self, module):
286         return self.source_files[module]['md5']
287
288     def GetExtractDirOf(self, module):
289         return self.source_files[module]['extract-dir']
290
291     def GetAdditionalParameters(self, module, step):
292         key = step + '-params'
293         if key in self.source_files[module]:
294             return self.source_files[module][key]
295         else:
296             return tuple()
297
298 class Extracter:
299     """class Extracter
300
301     Handles the extraction of the source files from their downloaded
302     archive files.
303     """
304
305     def __init__(self, source_files, config):
306         self.source_files = source_files
307         self.config = config
308
309     def Extract(self, module):
310         src = self.config.src_dir
311         extractDst = os.path.join(src, self.config.arch)
312         local_file = os.path.join(src, self.source_files.GetFilenameOf(module))
313         moduleMd5 = self.source_files.GetMd5Of(module)
314         extracted = os.path.join(extractDst, os.path.split(local_file)[1] + '.extracted')
315         if not os.path.exists(extractDst):
316             os.mkdir(extractDst)
317
318         extractedMd5 = None
319         if os.path.exists(extracted):
320             extractedMd5 = open(extracted).read()
321
322         if extractedMd5 != moduleMd5:
323             print 'Extracting %s:' % self.config.Relative(local_file)
324             tar = tarfile.open(local_file)
325             tar.extractall(extractDst)
326             open(extracted, 'w').write(moduleMd5)
327         else:
328             pass
329             #print 'Previously extracted', self.config.Relative(local_file)
330
331     def ExtractAll(self):
332         for module in self.source_files.GetModules():
333             self.Extract(module)
334
335 class Builder:
336     """class Builder
337
338     Builds and installs the GCC tool suite.
339     """
340
341     def __init__(self, source_files, config):
342         self.source_files = source_files
343         self.config = config
344
345     def Build(self):
346         self.BuildModule('binutils')
347         self.CopyIncludeDirectory()
348         self.BuildModule('gcc')
349         self.MakeSymLinks()
350
351     def IsBuildStepComplete(self, step):
352         return \
353             os.path.exists(
354                 os.path.join(
355                     self.config.build_dir, self.config.arch, step + '.completed'
356                     )
357                 )
358
359     def MarkBuildStepComplete(self, step):
360         open(
361             os.path.join(
362                 self.config.build_dir, self.config.arch, step + '.completed'
363                 ),
364             "w"
365             ).close()
366
367     def CopyIncludeDirectory(self):
368         linkdst = os.path.join(self.config.prefix, 'mingw')
369         if self.config.arch == 'x64':
370             src = os.path.join(self.config.src_dir, 'trunk', 'mingw-w64-headers', 'include')
371             dst_parent = os.path.join(self.config.prefix, self.config.target_arch + '-pc-mingw32')
372             dst = os.path.join(dst_parent, 'include')
373             if not os.path.exists(dst):
374                 if not os.path.exists(dst_parent):
375                     os.makedirs(dst_parent)
376                 print 'Copying headers to', self.config.Relative(dst)
377                 shutil.copytree(src, dst, True)
378         if not os.path.lexists(linkdst):
379             print 'Making symlink at', self.config.Relative(linkdst)
380             os.symlink(self.config.target_arch + '-pc-mingw32', linkdst)
381
382     def BuildModule(self, module):
383         base_dir = os.getcwd()
384         build_dir = os.path.join(self.config.build_dir, self.config.arch, module)
385         module_dir = self.source_files.GetExtractDirOf(module)
386         module_dir = os.path.realpath(os.path.join('src', self.config.arch, module_dir))
387         configure = os.path.join(module_dir, 'configure')
388         prefix = self.config.prefix
389         if not os.path.exists(build_dir):
390             os.makedirs(build_dir)
391         os.chdir(build_dir)
392
393         cmd = (
394             configure,
395             '--target=%s-pc-mingw32' % self.config.target_arch,
396             '--prefix=' + prefix,
397             '--with-sysroot=' + prefix,
398             )
399         if os.path.exists('/opt/local/include/gmp.h'):
400             cmd += ('--with-gmp=/opt/local',)
401         cmd += self.source_files.GetAdditionalParameters(module, 'configure')
402         self.RunCommand(cmd, module, 'config', skipable=True)
403
404         cmd = ('make',)
405         if module == 'gcc':
406             cmd += ('all-gcc',)
407         self.RunCommand(cmd, module, 'build')
408
409         cmd = ('make',)
410         if module == 'gcc':
411             cmd += ('install-gcc',)
412         else:
413             cmd += ('install',)
414         self.RunCommand(cmd, module, 'install')
415
416         os.chdir(base_dir)
417
418         print '%s module is now built and installed' % module
419
420     def RunCommand(self, cmd, module, stage, skipable=False):
421         if skipable:
422             if self.IsBuildStepComplete('%s.%s' % (module, stage)):
423                 return
424
425         popen = lambda cmd: \
426             subprocess.Popen(
427                 cmd,
428                 stdin=subprocess.PIPE,
429                 stdout=subprocess.PIPE,
430                 stderr=subprocess.STDOUT
431                 )
432
433         print '%s [%s] ...' % (module, stage),
434         sys.stdout.flush()
435         p = popen(cmd)
436         output = p.stdout.read()
437         p.wait()
438         if p.returncode != 0:
439             print '[failed!]'
440             logFile = os.path.join(self.config.build_dir, 'log.txt')
441             f = open(logFile, "w")
442             f.write(output)
443             f.close()
444             raise Exception, 'Failed to %s %s\n' % (stage, module) + \
445                 'See output log at %s' % self.config.Relative(logFile)
446         else:
447             print '[done]'
448
449         if skipable:
450             self.MarkBuildStepComplete('%s.%s' % (module, stage))
451
452     def MakeSymLinks(self):
453         links_dir = os.path.join(self.config.symlinks, self.config.arch)
454         if not os.path.exists(links_dir):
455             os.makedirs(links_dir)
456         startPrinted = False
457         for link in ('ar', 'ld', 'gcc'):
458             src = os.path.join(
459                 self.config.prefix, 'bin', self.config.target_arch + '-pc-mingw32-' + link
460                 )
461             linkdst = os.path.join(links_dir, link)
462             if not os.path.lexists(linkdst):
463                 if not startPrinted:
464                     print 'Making symlinks in %s:' % self.config.Relative(links_dir),
465                     startPrinted = True
466                 print link,
467                 os.symlink(src, linkdst)
468
469         if startPrinted:
470             print '[done]'
471
472 class App:
473     """class App
474
475     The main body of the application.
476     """
477
478     def __init__(self):
479         config = Config()
480
481         if not config.IsConfigOk():
482             return
483
484         config.MakeDirs()
485
486         sources = SourceFiles(config)
487         result = sources.GetAll()
488         if result:
489             print 'All files have been downloaded & verified'
490         else:
491             print 'An error occured while downloading a file'
492             return
493
494         Extracter(sources, config).ExtractAll()
495
496         Builder(sources, config).Build()
497
498 App()
499