Back to index

enigmail  1.4.3
JarMaker.py
Go to the documentation of this file.
00001 # ***** BEGIN LICENSE BLOCK *****
00002 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
00003 #
00004 # The contents of this file are subject to the Mozilla Public License Version
00005 # 1.1 (the "License"); you may not use this file except in compliance with
00006 # the License. You may obtain a copy of the License at
00007 # http://www.mozilla.org/MPL/
00008 #
00009 # Software distributed under the License is distributed on an "AS IS" basis,
00010 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
00011 # for the specific language governing rights and limitations under the
00012 # License.
00013 #
00014 # The Original Code is Mozilla build system.
00015 #
00016 # The Initial Developer of the Original Code is
00017 # Mozilla Foundation.
00018 # Portions created by the Initial Developer are Copyright (C) 2008
00019 # the Initial Developer. All Rights Reserved.
00020 #
00021 # Contributor(s):
00022 #  Axel Hecht <l10n@mozilla.com>
00023 #
00024 # Alternatively, the contents of this file may be used under the terms of
00025 # either the GNU General Public License Version 2 or later (the "GPL"), or
00026 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
00027 # in which case the provisions of the GPL or the LGPL are applicable instead
00028 # of those above. If you wish to allow use of your version of this file only
00029 # under the terms of either the GPL or the LGPL, and not to allow others to
00030 # use your version of this file under the terms of the MPL, indicate your
00031 # decision by deleting the provisions above and replace them with the notice
00032 # and other provisions required by the GPL or the LGPL. If you do not delete
00033 # the provisions above, a recipient may use your version of this file under
00034 # the terms of any one of the MPL, the GPL or the LGPL.
00035 #
00036 # ***** END LICENSE BLOCK *****
00037 
00038 '''jarmaker.py provides a python class to package up chrome content by
00039 processing jar.mn files.
00040 
00041 See the documentation for jar.mn on MDC for further details on the format.
00042 '''
00043 
00044 import sys
00045 import os
00046 import os.path
00047 import errno
00048 import re
00049 import logging
00050 from time import localtime
00051 from optparse import OptionParser
00052 from MozZipFile import ZipFile
00053 from cStringIO import StringIO
00054 from datetime import datetime
00055 
00056 from utils import pushback_iter, lockFile
00057 from Preprocessor import Preprocessor
00058 from buildlist import addEntriesToListFile
00059 if sys.platform == "win32":
00060   from ctypes import windll, WinError
00061   CreateHardLink = windll.kernel32.CreateHardLinkA
00062 
00063 __all__ = ['JarMaker']
00064 
00065 class ZipEntry:
00066   '''Helper class for jar output.
00067 
00068   This class defines a simple file-like object for a zipfile.ZipEntry
00069   so that we can consecutively write to it and then close it.
00070   This methods hooks into ZipFile.writestr on close().
00071   '''
00072   def __init__(self, name, zipfile):
00073     self._zipfile = zipfile
00074     self._name = name
00075     self._inner = StringIO()
00076 
00077   def write(self, content):
00078     'Append the given content to this zip entry'
00079     self._inner.write(content)
00080     return
00081 
00082   def close(self):
00083     'The close method writes the content back to the zip file.'
00084     self._zipfile.writestr(self._name, self._inner.getvalue())
00085 
00086 def getModTime(aPath):
00087   if not os.path.isfile(aPath):
00088     return 0
00089   mtime = os.stat(aPath).st_mtime
00090   return localtime(mtime)
00091 
00092 
00093 class JarMaker(object):
00094   '''JarMaker reads jar.mn files and process those into jar files or
00095   flat directories, along with chrome.manifest files.
00096   '''
00097 
00098   ignore = re.compile('\s*(\#.*)?$')
00099   jarline = re.compile('(?:(?P<jarfile>[\w\d.\-\_\\\/]+).jar\:)|(?:\s*(\#.*)?)\s*$')
00100   regline = re.compile('\%\s+(.*)$')
00101   entryre = '(?P<optPreprocess>\*)?(?P<optOverwrite>\+?)\s+'
00102   entryline = re.compile(entryre + '(?P<output>[\w\d.\-\_\\\/\+]+)\s*(\((?P<locale>\%?)(?P<source>[\w\d.\-\_\\\/]+)\))?\s*$')
00103 
00104   def __init__(self, outputFormat = 'flat', useJarfileManifest = True,
00105                useChromeManifest = False):
00106     self.outputFormat = outputFormat
00107     self.useJarfileManifest = useJarfileManifest
00108     self.useChromeManifest = useChromeManifest
00109     self.pp = Preprocessor()
00110 
00111   def getCommandLineParser(self):
00112     '''Get a optparse.OptionParser for jarmaker.
00113 
00114     This OptionParser has the options for jarmaker as well as
00115     the options for the inner PreProcessor.
00116     '''
00117     # HACK, we need to unescape the string variables we get,
00118     # the perl versions didn't grok strings right
00119     p = self.pp.getCommandLineParser(unescapeDefines = True)
00120     p.add_option('-f', type="choice", default="jar",
00121                  choices=('jar', 'flat', 'symlink'),
00122                  help="fileformat used for output", metavar="[jar, flat, symlink]")
00123     p.add_option('-v', action="store_true", dest="verbose",
00124                  help="verbose output")
00125     p.add_option('-q', action="store_false", dest="verbose",
00126                  help="verbose output")
00127     p.add_option('-e', action="store_true",
00128                  help="create chrome.manifest instead of jarfile.manifest")
00129     p.add_option('--both-manifests', action="store_true",
00130                  dest="bothManifests",
00131                  help="create chrome.manifest and jarfile.manifest")
00132     p.add_option('-s', type="string", action="append", default=[],
00133                  help="source directory")
00134     p.add_option('-t', type="string",
00135                  help="top source directory")
00136     p.add_option('-c', '--l10n-src', type="string", action="append",
00137                  help="localization directory")
00138     p.add_option('--l10n-base', type="string", action="append", default=[],
00139                  help="base directory to be used for localization (multiple)")
00140     p.add_option('-j', type="string",
00141                  help="jarfile directory")
00142     # backwards compat, not needed
00143     p.add_option('-a', action="store_false", default=True,
00144                  help="NOT SUPPORTED, turn auto-registration of chrome off (installed-chrome.txt)")
00145     p.add_option('-d', type="string",
00146                  help="UNUSED, chrome directory")
00147     p.add_option('-o', help="cross compile for auto-registration, ignored")
00148     p.add_option('-l', action="store_true",
00149                  help="ignored (used to switch off locks)")
00150     p.add_option('-x', action="store_true",
00151                  help="force Unix")
00152     p.add_option('-z', help="backwards compat, ignored")
00153     p.add_option('-p', help="backwards compat, ignored")
00154     return p
00155 
00156   def processIncludes(self, includes):
00157     '''Process given includes with the inner PreProcessor.
00158 
00159     Only use this for #defines, the includes shouldn't generate
00160     content.
00161     '''
00162     self.pp.out = StringIO()
00163     for inc in includes:
00164       self.pp.do_include(inc)
00165     includesvalue = self.pp.out.getvalue()
00166     if includesvalue:
00167       logging.info("WARNING: Includes produce non-empty output")
00168     self.pp.out = None
00169     pass
00170 
00171   def finalizeJar(self, jarPath, chromebasepath, register,
00172                   doZip=True):
00173     '''Helper method to write out the chrome registration entries to
00174     jarfile.manifest or chrome.manifest, or both.
00175 
00176     The actual file processing is done in updateManifest.
00177     '''
00178     # rewrite the manifest, if entries given
00179     if not register:
00180       return
00181 
00182     chromeManifest = os.path.join(os.path.dirname(jarPath),
00183                                   '..', 'chrome.manifest')
00184 
00185     if self.useJarfileManifest:
00186       self.updateManifest(jarPath + '.manifest', chromebasepath % '',
00187                           register)
00188       addEntriesToListFile(chromeManifest, ['manifest chrome/%s.manifest' % (os.path.basename(jarPath),)])
00189     if self.useChromeManifest:
00190       self.updateManifest(chromeManifest, chromebasepath % 'chrome/',
00191                           register)
00192 
00193   def updateManifest(self, manifestPath, chromebasepath, register):
00194     '''updateManifest replaces the % in the chrome registration entries
00195     with the given chrome base path, and updates the given manifest file.
00196     '''
00197     lock = lockFile(manifestPath + '.lck')
00198     try:
00199       myregister = dict.fromkeys(map(lambda s: s.replace('%', chromebasepath),
00200                                      register.iterkeys()))
00201       manifestExists = os.path.isfile(manifestPath)
00202       mode = (manifestExists and 'r+b') or 'wb'
00203       mf = open(manifestPath, mode)
00204       if manifestExists:
00205         # import previous content into hash, ignoring empty ones and comments
00206         imf = re.compile('(#.*)?$')
00207         for l in re.split('[\r\n]+', mf.read()):
00208           if imf.match(l):
00209             continue
00210           myregister[l] = None
00211         mf.seek(0)
00212       for k in myregister.iterkeys():
00213         mf.write(k + os.linesep)
00214       mf.close()
00215     finally:
00216       lock = None
00217   
00218   def makeJar(self, infile=None,
00219                jardir='',
00220                sourcedirs=[], topsourcedir='', localedirs=None):
00221     '''makeJar is the main entry point to JarMaker.
00222 
00223     It takes the input file, the output directory, the source dirs and the
00224     top source dir as argument, and optionally the l10n dirs.
00225     '''
00226     if isinstance(infile, basestring):
00227       logging.info("processing " + infile)
00228     pp = self.pp.clone()
00229     pp.out = StringIO()
00230     pp.do_include(infile)
00231     lines = pushback_iter(pp.out.getvalue().splitlines())
00232     try:
00233       while True:
00234         l = lines.next()
00235         m = self.jarline.match(l)
00236         if not m:
00237           raise RuntimeError(l)
00238         if m.group('jarfile') is None:
00239           # comment
00240           continue
00241         self.processJarSection(m.group('jarfile'), lines,
00242                                jardir, sourcedirs, topsourcedir,
00243                                localedirs)
00244     except StopIteration:
00245       # we read the file
00246       pass
00247     return
00248 
00249   def makeJars(self, infiles, l10nbases,
00250                jardir='',
00251                sourcedirs=[], topsourcedir='', localedirs=None):
00252     '''makeJars is the second main entry point to JarMaker.
00253 
00254     It takes an iterable sequence of input file names, the l10nbases,
00255     the output directory, the source dirs and the
00256     top source dir as argument, and optionally the l10n dirs.
00257 
00258     It iterates over all inputs, guesses srcdir and l10ndir from the
00259     path and topsourcedir and calls into makeJar.
00260 
00261     The l10ndirs are created by guessing the relativesrcdir, and resolving
00262     that against the l10nbases. l10nbases can either be path strings, or 
00263     callables. In the latter case, that will be called with the 
00264     relativesrcdir as argument, and is expected to return a path string.
00265     This logic is disabled if the jar.mn path is not inside the topsrcdir.
00266     '''
00267     topsourcedir = os.path.normpath(os.path.abspath(topsourcedir))
00268     def resolveL10nBase(relpath):
00269       def _resolve(base):
00270         if isinstance(base, basestring):
00271           return os.path.join(base, relpath)
00272         if callable(base):
00273           return base(relpath)
00274         return base
00275       return _resolve
00276     for infile in infiles:
00277       srcdir = os.path.normpath(os.path.abspath(os.path.dirname(infile)))
00278       l10ndir = srcdir
00279       if os.path.basename(srcdir) == 'locales':
00280         l10ndir = os.path.dirname(l10ndir)
00281 
00282       l10ndirs = None
00283       # srcdir may not be a child of topsourcedir, in which case
00284       # we assume that the caller passed in suitable sourcedirs,
00285       # and just skip passing in localedirs
00286       if srcdir.startswith(topsourcedir):
00287         rell10ndir = l10ndir[len(topsourcedir):].lstrip(os.sep)
00288 
00289         l10ndirs = map(resolveL10nBase(rell10ndir), l10nbases)
00290         if localedirs is not None:
00291           l10ndirs += [os.path.normpath(os.path.abspath(s))
00292                        for s in localedirs]
00293       srcdirs = [os.path.normpath(os.path.abspath(s))
00294                  for s in sourcedirs] + [srcdir]
00295       self.makeJar(infile=infile,
00296                    sourcedirs=srcdirs, topsourcedir=topsourcedir,
00297                    localedirs=l10ndirs,
00298                    jardir=jardir)
00299 
00300 
00301   def processJarSection(self, jarfile, lines,
00302                         jardir, sourcedirs, topsourcedir, localedirs):
00303     '''Internal method called by makeJar to actually process a section
00304     of a jar.mn file.
00305 
00306     jarfile is the basename of the jarfile or the directory name for 
00307     flat output, lines is a pushback_iterator of the lines of jar.mn,
00308     the remaining options are carried over from makeJar.
00309     '''
00310 
00311     # chromebasepath is used for chrome registration manifests
00312     # %s is getting replaced with chrome/ for chrome.manifest, and with
00313     # an empty string for jarfile.manifest
00314     chromebasepath = '%s' + jarfile
00315     if self.outputFormat == 'jar':
00316       chromebasepath = 'jar:' + chromebasepath + '.jar!'
00317     chromebasepath += '/'
00318 
00319     jarfile = os.path.join(jardir, jarfile)
00320     jf = None
00321     if self.outputFormat == 'jar':
00322       #jar
00323       jarfilepath = jarfile + '.jar'
00324       try:
00325         os.makedirs(os.path.dirname(jarfilepath))
00326       except OSError:
00327         pass
00328       jf = ZipFile(jarfilepath, 'a', lock = True)
00329       outHelper = self.OutputHelper_jar(jf)
00330     else:
00331       outHelper = getattr(self, 'OutputHelper_' + self.outputFormat)(jarfile)
00332     register = {}
00333     # This loop exits on either
00334     # - the end of the jar.mn file
00335     # - an line in the jar.mn file that's not part of a jar section
00336     # - on an exception raised, close the jf in that case in a finally
00337     try:
00338       while True:
00339         try:
00340           l = lines.next()
00341         except StopIteration:
00342           # we're done with this jar.mn, and this jar section
00343           self.finalizeJar(jarfile, chromebasepath, register)
00344           if jf is not None:
00345             jf.close()
00346           # reraise the StopIteration for makeJar
00347           raise
00348         if self.ignore.match(l):
00349           continue
00350         m = self.regline.match(l)
00351         if  m:
00352           rline = m.group(1)
00353           register[rline] = 1
00354           continue
00355         m = self.entryline.match(l)
00356         if not m:
00357           # neither an entry line nor chrome reg, this jar section is done
00358           self.finalizeJar(jarfile, chromebasepath, register)
00359           if jf is not None:
00360             jf.close()
00361           lines.pushback(l)
00362           return
00363         self._processEntryLine(m, sourcedirs, topsourcedir, localedirs,
00364                               outHelper, jf)
00365     finally:
00366       if jf is not None:
00367         jf.close()
00368     return
00369 
00370   def _processEntryLine(self, m, 
00371                         sourcedirs, topsourcedir, localedirs,
00372                         outHelper, jf):
00373       out = m.group('output')
00374       src = m.group('source') or os.path.basename(out)
00375       # pick the right sourcedir -- l10n, topsrc or src
00376       if m.group('locale'):
00377         src_base = localedirs
00378       elif src.startswith('/'):
00379         # path/in/jar/file_name.xul     (/path/in/sourcetree/file_name.xul)
00380         # refers to a path relative to topsourcedir, use that as base
00381         # and strip the leading '/'
00382         src_base = [topsourcedir]
00383         src = src[1:]
00384       else:
00385         # use srcdirs and the objdir (current working dir) for relative paths
00386         src_base = sourcedirs + [os.getcwd()]
00387       # check if the source file exists
00388       realsrc = None
00389       for _srcdir in src_base:
00390         if os.path.isfile(os.path.join(_srcdir, src)):
00391           realsrc = os.path.join(_srcdir, src)
00392           break
00393       if realsrc is None:
00394         if jf is not None:
00395           jf.close()
00396         raise RuntimeError('File "%s" not found in %s' % (src, ', '.join(src_base)))
00397       if m.group('optPreprocess'):
00398         outf = outHelper.getOutput(out)
00399         inf = open(realsrc)
00400         pp = self.pp.clone()
00401         if src[-4:] == '.css':
00402           pp.setMarker('%')
00403         pp.out = outf
00404         pp.do_include(inf)
00405         outf.close()
00406         inf.close()
00407         return
00408       # copy or symlink if newer or overwrite
00409       if (m.group('optOverwrite')
00410           or (getModTime(realsrc) >
00411               outHelper.getDestModTime(m.group('output')))):
00412         if self.outputFormat == 'symlink':
00413           outHelper.symlink(realsrc, out)
00414           return
00415         outf = outHelper.getOutput(out)
00416         # open in binary mode, this can be images etc
00417         inf = open(realsrc, 'rb')
00418         outf.write(inf.read())
00419         outf.close()
00420         inf.close()
00421     
00422 
00423   class OutputHelper_jar(object):
00424     '''Provide getDestModTime and getOutput for a given jarfile.
00425     '''
00426     def __init__(self, jarfile):
00427       self.jarfile = jarfile
00428     def getDestModTime(self, aPath):
00429       try :
00430         info = self.jarfile.getinfo(aPath)
00431         return info.date_time
00432       except:
00433         return 0
00434     def getOutput(self, name):
00435       return ZipEntry(name, self.jarfile)
00436 
00437   class OutputHelper_flat(object):
00438     '''Provide getDestModTime and getOutput for a given flat
00439     output directory. The helper method ensureDirFor is used by
00440     the symlink subclass.
00441     '''
00442     def __init__(self, basepath):
00443       self.basepath = basepath
00444     def getDestModTime(self, aPath):
00445       return getModTime(os.path.join(self.basepath, aPath))
00446     def getOutput(self, name):
00447       out = self.ensureDirFor(name)
00448       # remove previous link or file
00449       try:
00450         os.remove(out)
00451       except OSError, e:
00452         if e.errno != errno.ENOENT:
00453           raise
00454       return open(out, 'wb')
00455     def ensureDirFor(self, name):
00456       out = os.path.join(self.basepath, name)
00457       outdir = os.path.dirname(out)
00458       if not os.path.isdir(outdir):
00459         os.makedirs(outdir)
00460       return out
00461 
00462   class OutputHelper_symlink(OutputHelper_flat):
00463     '''Subclass of OutputHelper_flat that provides a helper for
00464     creating a symlink including creating the parent directories.
00465     '''
00466     def symlink(self, src, dest):
00467       out = self.ensureDirFor(dest)
00468       # remove previous link or file
00469       try:
00470         os.remove(out)
00471       except OSError, e:
00472         if e.errno != errno.ENOENT:
00473           raise
00474       if sys.platform != "win32":
00475         os.symlink(src, out)
00476       else:
00477         # On Win32, use ctypes to create a hardlink
00478         rv = CreateHardLink(out, src, None)
00479         if rv == 0:
00480           raise WinError()
00481 
00482 def main():
00483   jm = JarMaker()
00484   p = jm.getCommandLineParser()
00485   (options, args) = p.parse_args()
00486   jm.processIncludes(options.I)
00487   jm.outputFormat = options.f
00488   if options.e:
00489     jm.useChromeManifest = True
00490     jm.useJarfileManifest = False
00491   if options.bothManifests:
00492     jm.useChromeManifest = True
00493     jm.useJarfileManifest = True
00494   noise = logging.INFO
00495   if options.verbose is not None:
00496     noise = (options.verbose and logging.DEBUG) or logging.WARN
00497   if sys.version_info[:2] > (2,3):
00498     logging.basicConfig(format = "%(message)s")
00499   else:
00500     logging.basicConfig()
00501   logging.getLogger().setLevel(noise)
00502   topsrc = options.t
00503   topsrc = os.path.normpath(os.path.abspath(topsrc))
00504   if not args:
00505     jm.makeJar(infile=sys.stdin,
00506                sourcedirs=options.s, topsourcedir=topsrc,
00507                localedirs=options.l10n_src,
00508                jardir=options.j)
00509   else:
00510     jm.makeJars(args, options.l10n_base,
00511                 jardir=options.j,
00512                 sourcedirs=options.s, topsourcedir=topsrc,
00513                 localedirs=options.l10n_src)
00514 
00515 if __name__ == "__main__":
00516   main()