Back to index

enigmail  1.4.3
expandlibs_exec.py
Go to the documentation of this file.
00001 # ***** BEGIN LICENSE BLOCK *****
00002 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
00003 #
00004 # The contents of this file are subject to the Mozilla Public License Version
00005 # 1.1 (the "License"); you may not use this file except in compliance with
00006 # the License. You may obtain a copy of the License at
00007 # http://www.mozilla.org/MPL/
00008 #
00009 # Software distributed under the License is distributed on an "AS IS" basis,
00010 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
00011 # for the specific language governing rights and limitations under the
00012 # License.
00013 #
00014 # The Original Code is a build helper for libraries
00015 #
00016 # The Initial Developer of the Original Code is
00017 # the Mozilla Foundation
00018 # Portions created by the Initial Developer are Copyright (C) 2011
00019 # the Initial Developer. All Rights Reserved.
00020 #
00021 # Contributor(s):
00022 # Mike Hommey <mh@glandium.org>
00023 #
00024 # Alternatively, the contents of this file may be used under the terms of
00025 # either the GNU General Public License Version 2 or later (the "GPL"), or
00026 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
00027 # in which case the provisions of the GPL or the LGPL are applicable instead
00028 # of those above. If you wish to allow use of your version of this file only
00029 # under the terms of either the GPL or the LGPL, and not to allow others to
00030 # use your version of this file under the terms of the MPL, indicate your
00031 # decision by deleting the provisions above and replace them with the notice
00032 # and other provisions required by the GPL or the LGPL. If you do not delete
00033 # the provisions above, a recipient may use your version of this file under
00034 # the terms of any one of the MPL, the GPL or the LGPL.
00035 #
00036 # ***** END LICENSE BLOCK *****
00037 
00038 '''expandlibs-exec.py applies expandlibs rules, and some more (see below) to
00039 a given command line, and executes that command line with the expanded
00040 arguments.
00041 
00042 With the --extract argument (useful for e.g. $(AR)), it extracts object files
00043 from static libraries (or use those listed in library descriptors directly).
00044 
00045 With the --uselist argument (useful for e.g. $(CC)), it replaces all object
00046 files with a list file. This can be used to avoid limitations in the length
00047 of a command line. The kind of list file format used depends on the
00048 EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list)
00049 or 'linkerscript' for GNU ld linker scripts.
00050 See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details.
00051 
00052 With the --symbol-order argument, followed by a file name, it will add the
00053 relevant linker options to change the order in which the linker puts the
00054 symbols appear in the resulting binary. Only works for ELF targets.
00055 '''
00056 from __future__ import with_statement
00057 import sys
00058 import os
00059 from expandlibs import ExpandArgs, relativize, isObject
00060 import expandlibs_config as conf
00061 from optparse import OptionParser
00062 import subprocess
00063 import tempfile
00064 import shutil
00065 import subprocess
00066 import re
00067 
00068 # The are the insert points for a GNU ld linker script, assuming a more
00069 # or less "standard" default linker script. This is not a dict because
00070 # order is important.
00071 SECTION_INSERT_BEFORE = [
00072   ('.text', '.fini'),
00073   ('.rodata', '.rodata1'),
00074   ('.data.rel.ro', '.dynamic'),
00075   ('.data', '.data1'),
00076 ]
00077 
00078 class ExpandArgsMore(ExpandArgs):
00079     ''' Meant to be used as 'with ExpandArgsMore(args) as ...: '''
00080     def __enter__(self):
00081         self.tmp = []
00082         return self
00083         
00084     def __exit__(self, type, value, tb):
00085         '''Automatically remove temporary files'''
00086         for tmp in self.tmp:
00087             if os.path.isdir(tmp):
00088                 shutil.rmtree(tmp, True)
00089             else:
00090                 os.remove(tmp)
00091 
00092     def extract(self):
00093         self[0:] = self._extract(self)
00094 
00095     def _extract(self, args):
00096         '''When a static library name is found, either extract its contents
00097         in a temporary directory or use the information found in the
00098         corresponding lib descriptor.
00099         '''
00100         ar_extract = conf.AR_EXTRACT.split()
00101         newlist = []
00102         for arg in args:
00103             if os.path.splitext(arg)[1] == conf.LIB_SUFFIX:
00104                 if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
00105                     newlist += self._extract(self._expand_desc(arg))
00106                 elif os.path.exists(arg) and len(ar_extract):
00107                     tmp = tempfile.mkdtemp(dir=os.curdir)
00108                     self.tmp.append(tmp)
00109                     subprocess.call(ar_extract + [os.path.abspath(arg)], cwd=tmp)
00110                     objs = []
00111                     for root, dirs, files in os.walk(tmp):
00112                         objs += [relativize(os.path.join(root, f)) for f in files if isObject(f)]
00113                     newlist += objs
00114                 else:
00115                     newlist += [arg]
00116             else:
00117                 newlist += [arg]
00118         return newlist
00119 
00120     def makelist(self):
00121         '''Replaces object file names with a temporary list file, using a
00122         list format depending on the EXPAND_LIBS_LIST_STYLE variable
00123         '''
00124         objs = [o for o in self if isObject(o)]
00125         if not len(objs): return
00126         fd, tmp = tempfile.mkstemp(suffix=".list",dir=os.curdir)
00127         if conf.EXPAND_LIBS_LIST_STYLE == "linkerscript":
00128             content = ["INPUT(%s)\n" % obj for obj in objs]
00129             ref = tmp
00130         elif conf.EXPAND_LIBS_LIST_STYLE == "list":
00131             content = ["%s\n" % obj for obj in objs]
00132             ref = "@" + tmp
00133         else:
00134             os.close(fd)
00135             os.remove(tmp)
00136             return
00137         self.tmp.append(tmp)
00138         f = os.fdopen(fd, "w")
00139         f.writelines(content)
00140         f.close()
00141         idx = self.index(objs[0])
00142         newlist = self[0:idx] + [ref] + [item for item in self[idx:] if item not in objs]
00143         self[0:] = newlist
00144 
00145     def _getFoldedSections(self):
00146         '''Returns a dict about folded sections.
00147         When section A and B are folded into section C, the dict contains:
00148         { 'A': 'C',
00149           'B': 'C',
00150           'C': ['A', 'B'] }'''
00151         if not conf.LD_PRINT_ICF_SECTIONS:
00152             return {}
00153 
00154         proc = subprocess.Popen(self + [conf.LD_PRINT_ICF_SECTIONS], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
00155         (stdout, stderr) = proc.communicate()
00156         result = {}
00157         # gold's --print-icf-sections output looks like the following:
00158         # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o'
00159         # In terms of words, chances are this will change in the future,
00160         # especially considering "into" is misplaced. Splitting on quotes
00161         # seems safer.
00162         for l in stderr.split('\n'):
00163             quoted = l.split("'")
00164             if len(quoted) > 5 and quoted[1] != quoted[5]:
00165                 result[quoted[1]] = quoted[5]
00166                 if quoted[5] in result:
00167                     result[quoted[5]].append(quoted[1])
00168                 else:
00169                     result[quoted[5]] = [quoted[1]]
00170         return result
00171 
00172     def _getOrderedSections(self, ordered_symbols):
00173         '''Given an ordered list of symbols, returns the corresponding list
00174         of sections following the order.'''
00175         if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
00176             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
00177         finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX])
00178         folded = self._getFoldedSections()
00179         sections = set()
00180         ordered_sections = []
00181         for symbol in ordered_symbols:
00182             symbol_sections = finder.getSections(symbol)
00183             all_symbol_sections = []
00184             for section in symbol_sections:
00185                 if section in folded:
00186                     if isinstance(folded[section], str):
00187                         section = folded[section]
00188                     all_symbol_sections.append(section)
00189                     all_symbol_sections.extend(folded[section])
00190                 else:
00191                     all_symbol_sections.append(section)
00192             for section in all_symbol_sections:
00193                 if not section in sections:
00194                     ordered_sections.append(section)
00195                     sections.add(section)
00196         return ordered_sections
00197 
00198     def orderSymbols(self, order):
00199         '''Given a file containing a list of symbols, adds the appropriate
00200         argument to make the linker put the symbols in that order.'''
00201         with open(order) as file:
00202             sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()])
00203         split_sections = {}
00204         linked_sections = [s[0] for s in SECTION_INSERT_BEFORE]
00205         for s in sections:
00206             for linked_section in linked_sections:
00207                 if s.startswith(linked_section):
00208                     if linked_section in split_sections:
00209                         split_sections[linked_section].append(s)
00210                     else:
00211                         split_sections[linked_section] = [s]
00212                     break
00213         content = []
00214         # Order is important
00215         linked_sections = [s for s in linked_sections if s in split_sections]
00216 
00217         if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file':
00218             option = '-Wl,--section-ordering-file,%s'
00219             content = sections
00220             for linked_section in linked_sections:
00221                 content.extend(split_sections[linked_section])
00222                 content.append('%s.*' % linked_section)
00223                 content.append(linked_section)
00224 
00225         elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript':
00226             option = '-Wl,-T,%s'
00227             section_insert_before = dict(SECTION_INSERT_BEFORE)
00228             for linked_section in linked_sections:
00229                 content.append('SECTIONS {')
00230                 content.append('  %s : {' % linked_section)
00231                 content.extend('    *(%s)' % s for s in split_sections[linked_section])
00232                 content.append('  }')
00233                 content.append('}')
00234                 content.append('INSERT BEFORE %s' % section_insert_before[linked_section])
00235         else:
00236             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
00237 
00238         fd, tmp = tempfile.mkstemp(dir=os.curdir)
00239         f = os.fdopen(fd, "w")
00240         f.write('\n'.join(content)+'\n')
00241         f.close()
00242         self.tmp.append(tmp)
00243         self.append(option % tmp)
00244 
00245 class SectionFinder(object):
00246     '''Instances of this class allow to map symbol names to sections in
00247     object files.'''
00248 
00249     def __init__(self, objs):
00250         '''Creates an instance, given a list of object files.'''
00251         if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
00252             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
00253         self.mapping = {}
00254         for obj in objs:
00255             if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX:
00256                 raise Exception('%s is not an object nor a static library' % obj)
00257             for symbol, section in SectionFinder._getSymbols(obj):
00258                 sym = SectionFinder._normalize(symbol)
00259                 if sym in self.mapping:
00260                     if not section in self.mapping[sym]:
00261                         self.mapping[sym].append(section)
00262                 else:
00263                     self.mapping[sym] = [section]
00264 
00265     def getSections(self, symbol):
00266         '''Given a symbol, returns a list of sections containing it or the
00267         corresponding thunks. When the given symbol is a thunk, returns the
00268         list of sections containing its corresponding normal symbol and the
00269         other thunks for that symbol.'''
00270         sym = SectionFinder._normalize(symbol)
00271         if sym in self.mapping:
00272             return self.mapping[sym]
00273         return []
00274 
00275     @staticmethod
00276     def _normalize(symbol):
00277         '''For normal symbols, return the given symbol. For thunks, return
00278         the corresponding normal symbol.'''
00279         if re.match('^_ZThn[0-9]+_', symbol):
00280             return re.sub('^_ZThn[0-9]+_', '_Z', symbol)
00281         return symbol
00282 
00283     @staticmethod
00284     def _getSymbols(obj):
00285         '''Returns a list of (symbol, section) contained in the given object
00286         file.'''
00287         proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
00288         (stdout, stderr) = proc.communicate()
00289         syms = []
00290         for line in stdout.splitlines():
00291             # Each line has the following format:
00292             # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>
00293             tmp = line.split(' ',1)
00294             # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"]
00295             # We only need to consider cases where "<section>\t<length> <symbol>" is present,
00296             # and where the [FfO] flag is either F (function) or O (object).
00297             if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']:
00298                 tmp = tmp[1][8:].split()
00299                 # That gives us ["<section>","<length>", "<symbol>"]
00300                 syms.append((tmp[-1], tmp[0]))
00301         return syms
00302 
00303 def main():
00304     parser = OptionParser()
00305     parser.add_option("--extract", action="store_true", dest="extract",
00306         help="when a library has no descriptor file, extract it first, when possible")
00307     parser.add_option("--uselist", action="store_true", dest="uselist",
00308         help="use a list file for objects when executing a command")
00309     parser.add_option("--verbose", action="store_true", dest="verbose",
00310         help="display executed command and temporary files content")
00311     parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE",
00312         help="use the given list of symbols to order symbols in the resulting binary when using with a linker")
00313 
00314     (options, args) = parser.parse_args()
00315 
00316     with ExpandArgsMore(args) as args:
00317         if options.extract:
00318             args.extract()
00319         if options.symbol_order:
00320             args.orderSymbols(options.symbol_order)
00321         if options.uselist:
00322             args.makelist()
00323 
00324         if options.verbose:
00325             print >>sys.stderr, "Executing: " + " ".join(args)
00326             for tmp in [f for f in args.tmp if os.path.isfile(f)]:
00327                 print >>sys.stderr, tmp + ":"
00328                 with open(tmp) as file:
00329                     print >>sys.stderr, "".join(["    " + l for l in file.readlines()])
00330             sys.stderr.flush()
00331         exit(subprocess.call(args))
00332 
00333 if __name__ == '__main__':
00334     main()