Back to index

enigmail  1.4.3
manifestparser.py
Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 
00003 # ***** BEGIN LICENSE BLOCK *****
00004 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
00005 # 
00006 # The contents of this file are subject to the Mozilla Public License Version
00007 # 1.1 (the "License"); you may not use this file except in compliance with
00008 # the License. You may obtain a copy of the License at
00009 # http://www.mozilla.org/MPL/
00010 # 
00011 # Software distributed under the License is distributed on an "AS IS" basis,
00012 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
00013 # for the specific language governing rights and limitations under the
00014 # License.
00015 # 
00016 # The Original Code is mozilla.org code.
00017 # 
00018 # The Initial Developer of the Original Code is
00019 # Mozilla.org.
00020 # Portions created by the Initial Developer are Copyright (C) 2010
00021 # the Initial Developer. All Rights Reserved.
00022 # 
00023 # Contributor(s):
00024 #     Jeff Hammel <jhammel@mozilla.com>     (Original author)
00025 # 
00026 # Alternatively, the contents of this file may be used under the terms of
00027 # either of the GNU General Public License Version 2 or later (the "GPL"),
00028 # or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
00029 # in which case the provisions of the GPL or the LGPL are applicable instead
00030 # of those above. If you wish to allow use of your version of this file only
00031 # under the terms of either the GPL or the LGPL, and not to allow others to
00032 # use your version of this file under the terms of the MPL, indicate your
00033 # decision by deleting the provisions above and replace them with the notice
00034 # and other provisions required by the GPL or the LGPL. If you do not delete
00035 # the provisions above, a recipient may use your version of this file under
00036 # the terms of any one of the MPL, the GPL or the LGPL.
00037 # 
00038 # ***** END LICENSE BLOCK *****
00039 
00040 """
00041 Mozilla universal manifest parser
00042 """
00043 
00044 # this file lives at
00045 # http://hg.mozilla.org/automation/ManifestDestiny/raw-file/tip/manifestparser.py
00046 
00047 __all__ = ['read_ini', # .ini reader
00048            'ManifestParser', 'TestManifest', 'convert', # manifest handling
00049            'parse', 'ParseError', 'ExpressionParser'] # conditional expression parser
00050 
00051 import os
00052 import re
00053 import shutil
00054 import sys
00055 from fnmatch import fnmatch
00056 from optparse import OptionParser
00057 
00058 version = '0.5.3' # package version
00059 try:
00060     from setuptools import setup
00061 except:
00062     setup = None
00063 
00064 # we need relpath, but it is introduced in python 2.6
00065 # http://docs.python.org/library/os.path.html
00066 try:
00067     relpath = os.path.relpath
00068 except AttributeError:
00069     def relpath(path, start):
00070         """
00071         Return a relative version of a path
00072         from /usr/lib/python2.6/posixpath.py
00073         """
00074 
00075         if not path:
00076             raise ValueError("no path specified")
00077 
00078         start_list = os.path.abspath(start).split(os.path.sep)
00079         path_list = os.path.abspath(path).split(os.path.sep)
00080 
00081         # Work out how much of the filepath is shared by start and path.
00082         i = len(os.path.commonprefix([start_list, path_list]))
00083 
00084         rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:]
00085         if not rel_list:
00086             return start
00087         return os.path.join(*rel_list)
00088 
00089 # expr.py
00090 # from:
00091 # http://k0s.org/mozilla/hg/expressionparser
00092 # http://hg.mozilla.org/users/tmielczarek_mozilla.com/expressionparser
00093 
00094 # Implements a top-down parser/evaluator for simple boolean expressions.
00095 # ideas taken from http://effbot.org/zone/simple-top-down-parsing.htm
00096 #
00097 # Rough grammar:
00098 # expr := literal
00099 #       | '(' expr ')'
00100 #       | expr '&&' expr
00101 #       | expr '||' expr
00102 #       | expr '==' expr
00103 #       | expr '!=' expr
00104 # literal := BOOL
00105 #          | INT
00106 #          | STRING
00107 #          | IDENT
00108 # BOOL   := true|false
00109 # INT    := [0-9]+
00110 # STRING := "[^"]*"
00111 # IDENT  := [A-Za-z_]\w*
00112 
00113 # Identifiers take their values from a mapping dictionary passed as the second
00114 # argument.
00115 
00116 # Glossary (see above URL for details):
00117 # - nud: null denotation
00118 # - led: left detonation
00119 # - lbp: left binding power
00120 # - rbp: right binding power
00121 
00122 class ident_token(object):
00123     def __init__(self, value):
00124         self.value = value
00125     def nud(self, parser):
00126         # identifiers take their value from the value mappings passed
00127         # to the parser
00128         return parser.value(self.value)
00129 
00130 class literal_token(object):
00131     def __init__(self, value):
00132         self.value = value
00133     def nud(self, parser):
00134         return self.value
00135 
00136 class eq_op_token(object):
00137     "=="
00138     def led(self, parser, left):
00139         return left == parser.expression(self.lbp)
00140     
00141 class neq_op_token(object):
00142     "!="
00143     def led(self, parser, left):
00144         return left != parser.expression(self.lbp)
00145 
00146 class not_op_token(object):
00147     "!"
00148     def nud(self, parser):
00149         return not parser.expression()
00150 
00151 class and_op_token(object):
00152     "&&"
00153     def led(self, parser, left):
00154         right = parser.expression(self.lbp)
00155         return left and right
00156     
00157 class or_op_token(object):
00158     "||"
00159     def led(self, parser, left):
00160         right = parser.expression(self.lbp)
00161         return left or right
00162 
00163 class lparen_token(object):
00164     "("
00165     def nud(self, parser):
00166         expr = parser.expression()
00167         parser.advance(rparen_token)
00168         return expr
00169 
00170 class rparen_token(object):
00171     ")"
00172 
00173 class end_token(object):
00174     """always ends parsing"""
00175 
00176 ### derived literal tokens
00177 
00178 class bool_token(literal_token):
00179     def __init__(self, value):
00180         value = {'true':True, 'false':False}[value]
00181         literal_token.__init__(self, value)
00182 
00183 class int_token(literal_token):
00184     def __init__(self, value):
00185         literal_token.__init__(self, int(value))
00186 
00187 class string_token(literal_token):
00188     def __init__(self, value):
00189         literal_token.__init__(self, value[1:-1])
00190 
00191 precedence = [(end_token, rparen_token),
00192               (or_op_token,),
00193               (and_op_token,),
00194               (eq_op_token, neq_op_token),
00195               (lparen_token,),
00196               ]
00197 for index, rank in enumerate(precedence):
00198     for token in rank:
00199         token.lbp = index # lbp = lowest left binding power
00200 
00201 class ParseError(Exception):
00202     """errror parsing conditional expression"""
00203 
00204 class ExpressionParser(object):
00205     def __init__(self, text, valuemapping, strict=False):
00206         """
00207         Initialize the parser with input |text|, and |valuemapping| as
00208         a dict mapping identifier names to values.
00209         """
00210         self.text = text
00211         self.valuemapping = valuemapping
00212         self.strict = strict
00213 
00214     def _tokenize(self):
00215         """
00216         Lex the input text into tokens and yield them in sequence.
00217         """
00218         # scanner callbacks
00219         def bool_(scanner, t): return bool_token(t)
00220         def identifier(scanner, t): return ident_token(t)
00221         def integer(scanner, t): return int_token(t)
00222         def eq(scanner, t): return eq_op_token()
00223         def neq(scanner, t): return neq_op_token()
00224         def or_(scanner, t): return or_op_token()
00225         def and_(scanner, t): return and_op_token()
00226         def lparen(scanner, t): return lparen_token()
00227         def rparen(scanner, t): return rparen_token()
00228         def string_(scanner, t): return string_token(t)
00229         def not_(scanner, t): return not_op_token()
00230 
00231         scanner = re.Scanner([
00232             (r"true|false", bool_),
00233             (r"[a-zA-Z_]\w*", identifier),
00234             (r"[0-9]+", integer),
00235             (r'("[^"]*")|(\'[^\']*\')', string_),
00236             (r"==", eq),
00237             (r"!=", neq),
00238             (r"\|\|", or_),
00239             (r"!", not_),
00240             (r"&&", and_),
00241             (r"\(", lparen),
00242             (r"\)", rparen),
00243             (r"\s+", None), # skip whitespace
00244             ])
00245         tokens, remainder = scanner.scan(self.text)
00246         for t in tokens:
00247             yield t
00248         yield end_token()
00249 
00250     def value(self, ident):
00251         """
00252         Look up the value of |ident| in the value mapping passed in the
00253         constructor.
00254         """
00255         if self.strict:
00256             return self.valuemapping[ident]
00257         else:
00258             return self.valuemapping.get(ident, None)
00259 
00260     def advance(self, expected):
00261         """
00262         Assert that the next token is an instance of |expected|, and advance
00263         to the next token.
00264         """
00265         if not isinstance(self.token, expected):
00266             raise Exception, "Unexpected token!"
00267         self.token = self.iter.next()
00268         
00269     def expression(self, rbp=0):
00270         """
00271         Parse and return the value of an expression until a token with
00272         right binding power greater than rbp is encountered.
00273         """
00274         t = self.token
00275         self.token = self.iter.next()
00276         left = t.nud(self)
00277         while rbp < self.token.lbp:
00278             t = self.token
00279             self.token = self.iter.next()
00280             left = t.led(self, left)
00281         return left
00282 
00283     def parse(self):
00284         """
00285         Parse and return the value of the expression in the text
00286         passed to the constructor. Raises a ParseError if the expression
00287         could not be parsed.
00288         """
00289         try:
00290             self.iter = self._tokenize()
00291             self.token = self.iter.next()
00292             return self.expression()
00293         except:
00294             raise ParseError("could not parse: %s; variables: %s" % (self.text, self.valuemapping))
00295 
00296     __call__ = parse
00297 
00298 def parse(text, **values):
00299     """
00300     Parse and evaluate a boolean expression in |text|. Use |values| to look
00301     up the value of identifiers referenced in the expression. Returns the final
00302     value of the expression. A ParseError will be raised if parsing fails.
00303     """
00304     return ExpressionParser(text, values).parse()
00305 
00306 def normalize_path(path):
00307     """normalize a relative path"""
00308     if sys.platform.startswith('win'):
00309         return path.replace('/', os.path.sep)
00310     return path
00311 
00312 def denormalize_path(path):
00313     """denormalize a relative path"""
00314     if sys.platform.startswith('win'):
00315         return path.replace(os.path.sep, '/')
00316     return path
00317     
00318 
00319 def read_ini(fp, variables=None, default='DEFAULT',
00320              comments=';#', separators=('=', ':'),
00321              strict=True):
00322     """
00323     read an .ini file and return a list of [(section, values)]
00324     - fp : file pointer or path to read
00325     - variables : default set of variables
00326     - default : name of the section for the default section
00327     - comments : characters that if they start a line denote a comment
00328     - separators : strings that denote key, value separation in order
00329     - strict : whether to be strict about parsing
00330     """
00331 
00332     if variables is None:
00333         variables = {}
00334 
00335     if isinstance(fp, basestring):
00336         fp = file(fp)
00337 
00338     sections = []
00339     key = value = None
00340     section_names = set([])
00341 
00342     # read the lines
00343     for line in fp.readlines():
00344 
00345         stripped = line.strip()
00346 
00347         # ignore blank lines
00348         if not stripped:
00349             # reset key and value to avoid continuation lines
00350             key = value = None
00351             continue
00352 
00353         # ignore comment lines
00354         if stripped[0] in comments:
00355             continue
00356 
00357         # check for a new section
00358         if len(stripped) > 2 and stripped[0] == '[' and stripped[-1] == ']':
00359             section = stripped[1:-1].strip()
00360             key = value = None
00361 
00362             # deal with DEFAULT section
00363             if section.lower() == default.lower():
00364                 if strict:
00365                     assert default not in section_names
00366                 section_names.add(default)
00367                 current_section = variables
00368                 continue
00369 
00370             if strict:
00371                 # make sure this section doesn't already exist
00372                 assert section not in section_names
00373 
00374             section_names.add(section)
00375             current_section = {}
00376             sections.append((section, current_section))
00377             continue
00378 
00379         # if there aren't any sections yet, something bad happen
00380         if not section_names:
00381             raise Exception('No sections found')
00382 
00383         # (key, value) pair
00384         for separator in separators:
00385             if separator in stripped:
00386                 key, value = stripped.split(separator, 1)
00387                 key = key.strip()
00388                 value = value.strip()
00389 
00390                 if strict:
00391                     # make sure this key isn't already in the section or empty
00392                     assert key
00393                     if current_section is not variables:
00394                         assert key not in current_section
00395 
00396                 current_section[key] = value
00397                 break
00398         else:
00399             # continuation line ?
00400             if line[0].isspace() and key:
00401                 value = '%s%s%s' % (value, os.linesep, stripped)
00402                 current_section[key] = value
00403             else:
00404                 # something bad happen!
00405                 raise Exception("Not sure what you're trying to do")
00406 
00407     # interpret the variables
00408     def interpret_variables(global_dict, local_dict):
00409         variables = global_dict.copy()
00410         variables.update(local_dict)
00411         return variables
00412 
00413     sections = [(i, interpret_variables(variables, j)) for i, j in sections]
00414     return sections
00415 
00416 
00417 ### objects for parsing manifests
00418 
00419 class ManifestParser(object):
00420     """read .ini manifests"""
00421 
00422     ### methods for reading manifests
00423 
00424     def __init__(self, manifests=(), defaults=None, strict=True):
00425         self._defaults = defaults or {}
00426         self.tests = []
00427         self.strict = strict
00428         self.rootdir = None
00429         self.relativeRoot = None
00430         if manifests:
00431             self.read(*manifests)
00432 
00433     def getRelativeRoot(self, root):
00434         return root
00435 
00436     def read(self, *filenames, **defaults):
00437 
00438         # ensure all files exist
00439         missing = [ filename for filename in filenames
00440                     if not os.path.exists(filename) ]
00441         if missing:
00442             raise IOError('Missing files: %s' % ', '.join(missing))
00443 
00444         # process each file
00445         for filename in filenames:
00446 
00447             # set the per file defaults
00448             defaults = defaults.copy() or self._defaults.copy()
00449             here = os.path.dirname(os.path.abspath(filename))
00450             defaults['here'] = here
00451 
00452             if self.rootdir is None:
00453                 # set the root directory
00454                 # == the directory of the first manifest given
00455                 self.rootdir = here
00456 
00457             # read the configuration
00458             sections = read_ini(fp=filename, variables=defaults, strict=self.strict)
00459 
00460             # get the tests
00461             for section, data in sections:
00462 
00463                 # a file to include
00464                 # TODO: keep track of included file structure:
00465                 # self.manifests = {'manifest.ini': 'relative/path.ini'}
00466                 if section.startswith('include:'):
00467                     include_file = section.split('include:', 1)[-1]
00468                     include_file = normalize_path(include_file)
00469                     if not os.path.isabs(include_file):
00470                         include_file = os.path.join(self.getRelativeRoot(here), include_file)
00471                     if not os.path.exists(include_file):
00472                         if self.strict:
00473                             raise IOError("File '%s' does not exist" % include_file)
00474                         else:
00475                             continue
00476                     include_defaults = data.copy()
00477                     self.read(include_file, **include_defaults)
00478                     continue
00479 
00480                 # otherwise an item
00481                 test = data
00482                 test['name'] = section
00483                 test['manifest'] = os.path.abspath(filename)
00484 
00485                 # determine the path
00486                 path = test.get('path', section)
00487                 if '://' not in path: # don't futz with URLs
00488                     path = normalize_path(path)
00489                     if not os.path.isabs(path):
00490                         path = os.path.join(here, path)
00491                 test['path'] = path
00492 
00493                 # append the item
00494                 self.tests.append(test)
00495 
00496     ### methods for querying manifests
00497 
00498     def query(self, *checks, **kw):
00499         """
00500         general query function for tests
00501         - checks : callable conditions to test if the test fulfills the query
00502         """
00503         tests = kw.get('tests', None)
00504         if tests is None:
00505             tests = self.tests
00506         retval = []
00507         for test in tests:
00508             for check in checks:
00509                 if not check(test):
00510                     break
00511             else:
00512                 retval.append(test)
00513         return retval
00514 
00515     def get(self, _key=None, inverse=False, tags=None, tests=None, **kwargs):
00516         # TODO: pass a dict instead of kwargs since you might hav
00517         # e.g. 'inverse' as a key in the dict
00518 
00519         # TODO: tags should just be part of kwargs with None values
00520         # (None == any is kinda weird, but probably still better)
00521 
00522         # fix up tags
00523         if tags:
00524             tags = set(tags)
00525         else:
00526             tags = set()
00527 
00528         # make some check functions
00529         if inverse:
00530             has_tags = lambda test: not tags.intersection(test.keys())
00531             def dict_query(test):
00532                 for key, value in kwargs.items():
00533                     if test.get(key) == value:
00534                         return False
00535                 return True
00536         else:
00537             has_tags = lambda test: tags.issubset(test.keys())
00538             def dict_query(test):
00539                 for key, value in kwargs.items():
00540                     if test.get(key) != value:
00541                         return False
00542                 return True
00543 
00544         # query the tests
00545         tests = self.query(has_tags, dict_query, tests=tests)
00546 
00547         # if a key is given, return only a list of that key
00548         # useful for keys like 'name' or 'path'
00549         if _key:
00550             return [test[_key] for test in tests]
00551 
00552         # return the tests
00553         return tests
00554 
00555     def missing(self, tests=None):
00556         """return list of tests that do not exist on the filesystem"""
00557         if tests is None:
00558             tests = self.tests
00559         return [test for test in tests
00560                 if not os.path.exists(test['path'])]
00561 
00562     def manifests(self, tests=None):
00563         """
00564         return manifests in order in which they appear in the tests
00565         """
00566         if tests is None:
00567             tests = self.tests
00568         manifests = []
00569         for test in tests:
00570             manifest = test.get('manifest')
00571             if not manifest:
00572                 continue
00573             if manifest not in manifests:
00574                 manifests.append(manifest)
00575         return manifests
00576 
00577     ### methods for outputting from manifests
00578 
00579     def write(self, fp=sys.stdout, rootdir=None,
00580               global_tags=None, global_kwargs=None,
00581               local_tags=None, local_kwargs=None):
00582         """
00583         write a manifest given a query
00584         global and local options will be munged to do the query
00585         globals will be written to the top of the file
00586         locals (if given) will be written per test
00587         """
00588 
00589         # root directory
00590         if rootdir is None:
00591             rootdir = self.rootdir
00592 
00593         # sanitize input
00594         global_tags = global_tags or set()
00595         local_tags = local_tags or set()
00596         global_kwargs = global_kwargs or {}
00597         local_kwargs = local_kwargs or {}
00598         
00599         # create the query
00600         tags = set([])
00601         tags.update(global_tags)
00602         tags.update(local_tags)
00603         kwargs = {}
00604         kwargs.update(global_kwargs)
00605         kwargs.update(local_kwargs)
00606 
00607         # get matching tests
00608         tests = self.get(tags=tags, **kwargs)
00609 
00610         # print the .ini manifest
00611         if global_tags or global_kwargs:
00612             print >> fp, '[DEFAULT]'
00613             for tag in global_tags:
00614                 print >> fp, '%s =' % tag
00615             for key, value in global_kwargs.items():
00616                 print >> fp, '%s = %s' % (key, value)
00617             print >> fp
00618 
00619         for test in tests:
00620             test = test.copy() # don't overwrite
00621 
00622             path = test['name']
00623             if not os.path.isabs(path):
00624                 path = denormalize_path(relpath(test['path'], self.rootdir))
00625             print >> fp, '[%s]' % path
00626           
00627             # reserved keywords:
00628             reserved = ['path', 'name', 'here', 'manifest']
00629             for key in sorted(test.keys()):
00630                 if key in reserved:
00631                     continue
00632                 if key in global_kwargs:
00633                     continue
00634                 if key in global_tags and not test[key]:
00635                     continue
00636                 print >> fp, '%s = %s' % (key, test[key])
00637             print >> fp
00638 
00639     def copy(self, directory, rootdir=None, *tags, **kwargs):
00640         """
00641         copy the manifests and associated tests
00642         - directory : directory to copy to
00643         - rootdir : root directory to copy to (if not given from manifests)
00644         - tags : keywords the tests must have
00645         - kwargs : key, values the tests must match
00646         """
00647         # XXX note that copy does *not* filter the tests out of the
00648         # resulting manifest; it just stupidly copies them over.
00649         # ideally, it would reread the manifests and filter out the
00650         # tests that don't match *tags and **kwargs
00651         
00652         # destination
00653         if not os.path.exists(directory):
00654             os.path.makedirs(directory)
00655         else:
00656             # sanity check
00657             assert os.path.isdir(directory)
00658 
00659         # tests to copy
00660         tests = self.get(tags=tags, **kwargs)
00661         if not tests:
00662             return # nothing to do!
00663 
00664         # root directory
00665         if rootdir is None:
00666             rootdir = self.rootdir
00667 
00668         # copy the manifests + tests
00669         manifests = [relpath(manifest, rootdir) for manifest in self.manifests()]
00670         for manifest in manifests:
00671             destination = os.path.join(directory, manifest)
00672             dirname = os.path.dirname(destination)
00673             if not os.path.exists(dirname):
00674                 os.makedirs(dirname)
00675             else:
00676                 # sanity check
00677                 assert os.path.isdir(dirname)
00678             shutil.copy(os.path.join(rootdir, manifest), destination)
00679         for test in tests:
00680             if os.path.isabs(test['name']):
00681                 continue
00682             source = test['path']
00683             if not os.path.exists(source):
00684                 print >> sys.stderr, "Missing test: '%s' does not exist!" % source
00685                 continue
00686                 # TODO: should err on strict
00687             destination = os.path.join(directory, relpath(test['path'], rootdir))
00688             shutil.copy(source, destination)
00689             # TODO: ensure that all of the tests are below the from_dir
00690 
00691     def update(self, from_dir, rootdir=None, *tags, **kwargs):
00692         """
00693         update the tests as listed in a manifest from a directory
00694         - from_dir : directory where the tests live
00695         - rootdir : root directory to copy to (if not given from manifests)
00696         - tags : keys the tests must have
00697         - kwargs : key, values the tests must match
00698         """
00699     
00700         # get the tests
00701         tests = self.get(tags=tags, **kwargs)
00702 
00703         # get the root directory
00704         if not rootdir:
00705             rootdir = self.rootdir
00706 
00707         # copy them!
00708         for test in tests:
00709             if not os.path.isabs(test['name']):
00710                 _relpath = relpath(test['path'], rootdir)
00711                 source = os.path.join(from_dir, _relpath)
00712                 if not os.path.exists(source):
00713                     # TODO err on strict
00714                     print >> sys.stderr, "Missing test: '%s'; skipping" % test['name']
00715                     continue
00716                 destination = os.path.join(rootdir, _relpath)
00717                 shutil.copy(source, destination)
00718 
00719 
00720 class TestManifest(ManifestParser):
00721     """
00722     apply logic to manifests;  this is your integration layer :)
00723     specific harnesses may subclass from this if they need more logic
00724     """
00725 
00726     def filter(self, values, tests):
00727         """
00728         filter on a specific list tag, e.g.:
00729         run-if.os = win linux
00730         skip-if.os = mac
00731         """
00732 
00733         # tags:
00734         run_tag = 'run-if'
00735         skip_tag = 'skip-if'
00736         fail_tag = 'fail-if'
00737 
00738         # loop over test
00739         for test in tests:
00740             reason = None # reason to disable
00741             
00742             # tagged-values to run
00743             if run_tag in test:
00744                 condition = test[run_tag]
00745                 if not parse(condition, **values):
00746                     reason = '%s: %s' % (run_tag, condition)
00747 
00748             # tagged-values to skip
00749             if skip_tag in test:
00750                 condition = test[skip_tag]
00751                 if parse(condition, **values):
00752                     reason = '%s: %s' % (skip_tag, condition)
00753 
00754             # mark test as disabled if there's a reason
00755             if reason:
00756                 test.setdefault('disabled', reason)        
00757 
00758             # mark test as a fail if so indicated
00759             if fail_tag in test:
00760                 condition = test[fail_tag]
00761                 if parse(condition, **values):
00762                     test['expected'] = 'fail'
00763 
00764     def active_tests(self, exists=True, disabled=True, **values):
00765         """
00766         - exists : return only existing tests
00767         - disabled : whether to return disabled tests
00768         - tags : keys and values to filter on (e.g. `os = linux mac`)
00769         """
00770 
00771         tests = [i.copy() for i in self.tests] # shallow copy
00772 
00773         # mark all tests as passing unless indicated otherwise
00774         for test in tests:
00775             test['expected'] = test.get('expected', 'pass')
00776         
00777         # ignore tests that do not exist
00778         if exists:
00779             tests = [test for test in tests if os.path.exists(test['path'])]
00780 
00781         # filter by tags
00782         self.filter(values, tests)
00783 
00784         # ignore disabled tests if specified
00785         if not disabled:
00786             tests = [test for test in tests
00787                      if not 'disabled' in test]
00788 
00789         # return active tests
00790         return tests
00791 
00792     def test_paths(self):
00793         return [test['path'] for test in self.active_tests()]
00794 
00795 
00796 ### utility function(s); probably belongs elsewhere
00797 
00798 def convert(directories, pattern=None, ignore=(), write=None):
00799     """
00800     convert directories to a simple manifest
00801     """
00802 
00803     retval = []
00804     include = []
00805     for directory in directories:
00806         for dirpath, dirnames, filenames in os.walk(directory):
00807 
00808             # filter out directory names
00809             dirnames = [ i for i in dirnames if i not in ignore ]
00810             dirnames.sort()
00811 
00812             # reference only the subdirectory
00813             _dirpath = dirpath
00814             dirpath = dirpath.split(directory, 1)[-1].strip(os.path.sep)
00815 
00816             if dirpath.split(os.path.sep)[0] in ignore:
00817                 continue
00818 
00819             # filter by glob
00820             if pattern:
00821                 filenames = [filename for filename in filenames
00822                              if fnmatch(filename, pattern)]
00823 
00824             filenames.sort()
00825 
00826             # write a manifest for each directory
00827             if write and (dirnames or filenames):
00828                 manifest = file(os.path.join(_dirpath, write), 'w')
00829                 for dirname in dirnames:
00830                     print >> manifest, '[include:%s]' % os.path.join(dirname, write)
00831                 for filename in filenames:
00832                     print >> manifest, '[%s]' % filename
00833                 manifest.close()
00834 
00835             # add to the list
00836             retval.extend([denormalize_path(os.path.join(dirpath, filename))
00837                            for filename in filenames])
00838 
00839     if write:
00840         return # the manifests have already been written!
00841   
00842     retval.sort()
00843     retval = ['[%s]' % filename for filename in retval]
00844     return '\n'.join(retval)
00845 
00846 ### command line attributes
00847 
00848 class ParserError(Exception):
00849   """error for exceptions while parsing the command line"""
00850 
00851 def parse_args(_args):
00852     """
00853     parse and return:
00854     --keys=value (or --key value)
00855     -tags
00856     args
00857     """
00858 
00859     # return values
00860     _dict = {}
00861     tags = []
00862     args = []
00863 
00864     # parse the arguments
00865     key = None
00866     for arg in _args:
00867         if arg.startswith('---'):
00868             raise ParserError("arguments should start with '-' or '--' only")
00869         elif arg.startswith('--'):
00870             if key:
00871                 raise ParserError("Key %s still open" % key)
00872             key = arg[2:]
00873             if '=' in key:
00874                 key, value = key.split('=', 1)
00875                 _dict[key] = value
00876                 key = None
00877                 continue
00878         elif arg.startswith('-'):
00879             if key:
00880                 raise ParserError("Key %s still open" % key)
00881             tags.append(arg[1:])
00882             continue
00883         else:
00884             if key:
00885                 _dict[key] = arg
00886                 continue
00887             args.append(arg)
00888 
00889     # return values
00890     return (_dict, tags, args)
00891 
00892 
00893 ### classes for subcommands
00894 
00895 class CLICommand(object):
00896     usage = '%prog [options] command'
00897     def __init__(self, parser):
00898       self._parser = parser # master parser
00899     def parser(self):
00900       return OptionParser(usage=self.usage, description=self.__doc__,
00901                           add_help_option=False)
00902 
00903 class Copy(CLICommand):
00904     usage = '%prog [options] copy manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
00905     def __call__(self, options, args):
00906       # parse the arguments
00907       try:
00908         kwargs, tags, args = parse_args(args)
00909       except ParserError, e:
00910         self._parser.error(e.message)
00911 
00912       # make sure we have some manifests, otherwise it will
00913       # be quite boring
00914       if not len(args) == 2:
00915         HelpCLI(self._parser)(options, ['copy'])
00916         return
00917 
00918       # read the manifests
00919       # TODO: should probably ensure these exist here
00920       manifests = ManifestParser()
00921       manifests.read(args[0])
00922 
00923       # print the resultant query
00924       manifests.copy(args[1], None, *tags, **kwargs)
00925 
00926 
00927 class CreateCLI(CLICommand):
00928     """
00929     create a manifest from a list of directories
00930     """
00931     usage = '%prog [options] create directory <directory> <...>'
00932 
00933     def parser(self):
00934         parser = CLICommand.parser(self)
00935         parser.add_option('-p', '--pattern', dest='pattern',
00936                           help="glob pattern for files")
00937         parser.add_option('-i', '--ignore', dest='ignore',
00938                           default=[], action='append',
00939                           help='directories to ignore')
00940         parser.add_option('-w', '--in-place', dest='in_place',
00941                           help='Write .ini files in place; filename to write to')
00942         return parser
00943 
00944     def __call__(self, _options, args):
00945         parser = self.parser()
00946         options, args = parser.parse_args(args)
00947 
00948         # need some directories
00949         if not len(args):
00950             parser.print_usage()
00951             return
00952 
00953         # add the directories to the manifest
00954         for arg in args:
00955             assert os.path.exists(arg)
00956             assert os.path.isdir(arg)
00957             manifest = convert(args, pattern=options.pattern, ignore=options.ignore,
00958                                write=options.in_place)
00959         if manifest:
00960             print manifest
00961 
00962 
00963 class WriteCLI(CLICommand):
00964     """
00965     write a manifest based on a query
00966     """
00967     usage = '%prog [options] write manifest <manifest> -tag1 -tag2 --key1=value1 --key2=value2 ...'
00968     def __call__(self, options, args):
00969 
00970         # parse the arguments
00971         try:
00972             kwargs, tags, args = parse_args(args)
00973         except ParserError, e:
00974             self._parser.error(e.message)
00975 
00976         # make sure we have some manifests, otherwise it will
00977         # be quite boring
00978         if not args:
00979             HelpCLI(self._parser)(options, ['write'])
00980             return
00981 
00982         # read the manifests
00983         # TODO: should probably ensure these exist here
00984         manifests = ManifestParser()
00985         manifests.read(*args)
00986 
00987         # print the resultant query
00988         manifests.write(global_tags=tags, global_kwargs=kwargs)
00989       
00990 
00991 class HelpCLI(CLICommand):
00992     """
00993     get help on a command
00994     """
00995     usage = '%prog [options] help [command]'
00996 
00997     def __call__(self, options, args):
00998         if len(args) == 1 and args[0] in commands:
00999             commands[args[0]](self._parser).parser().print_help()
01000         else:
01001             self._parser.print_help()
01002             print '\nCommands:'
01003             for command in sorted(commands):
01004                 print '  %s : %s' % (command, commands[command].__doc__.strip())
01005 
01006 class SetupCLI(CLICommand):
01007     """
01008     setup using setuptools
01009     """
01010     # use setup.py from the repo when you want to distribute to python!
01011     # otherwise setuptools will complain that it can't find setup.py
01012     # and result in a useless package
01013     
01014     usage = '%prog [options] setup [setuptools options]'
01015     
01016     def __call__(self, options, args):
01017         sys.argv = [sys.argv[0]] + args
01018         assert setup is not None, "You must have setuptools installed to use SetupCLI"
01019         here = os.path.dirname(os.path.abspath(__file__))
01020         try:
01021             filename = os.path.join(here, 'README.txt')
01022             description = file(filename).read()
01023         except:    
01024             description = ''
01025         os.chdir(here)
01026 
01027         setup(name='ManifestDestiny',
01028               version=version,
01029               description="Universal manifests for Mozilla test harnesses",
01030               long_description=description,
01031               classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
01032               keywords='mozilla manifests',
01033               author='Jeff Hammel',
01034               author_email='jhammel@mozilla.com',
01035               url='https://wiki.mozilla.org/Auto-tools/Projects/ManifestDestiny',
01036               license='MPL',
01037               zip_safe=False,
01038               py_modules=['manifestparser'],
01039               install_requires=[
01040                   # -*- Extra requirements: -*-
01041                   ],
01042               entry_points="""
01043               [console_scripts]
01044               manifestparser = manifestparser:main
01045               """,
01046               )
01047 
01048 
01049 class UpdateCLI(CLICommand):
01050     """
01051     update the tests as listed in a manifest from a directory
01052     """
01053     usage = '%prog [options] update manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
01054 
01055     def __call__(self, options, args):
01056         # parse the arguments
01057         try:
01058             kwargs, tags, args = parse_args(args)
01059         except ParserError, e:
01060             self._parser.error(e.message)
01061 
01062         # make sure we have some manifests, otherwise it will
01063         # be quite boring
01064         if not len(args) == 2:
01065             HelpCLI(self._parser)(options, ['update'])
01066             return
01067 
01068         # read the manifests
01069         # TODO: should probably ensure these exist here
01070         manifests = ManifestParser()
01071         manifests.read(args[0])
01072 
01073         # print the resultant query
01074         manifests.update(args[1], None, *tags, **kwargs)
01075 
01076 
01077 # command -> class mapping
01078 commands = { 'create': CreateCLI,
01079              'help': HelpCLI,
01080              'update': UpdateCLI,
01081              'write': WriteCLI }
01082 if setup is not None:
01083     commands['setup'] = SetupCLI
01084 
01085 def main(args=sys.argv[1:]):
01086     """console_script entry point"""
01087 
01088     # set up an option parser
01089     usage = '%prog [options] [command] ...'
01090     description = __doc__
01091     parser = OptionParser(usage=usage, description=description)
01092     parser.add_option('-s', '--strict', dest='strict',
01093                       action='store_true', default=False,
01094                       help='adhere strictly to errors')
01095     parser.disable_interspersed_args()
01096 
01097     options, args = parser.parse_args(args)
01098 
01099     if not args:
01100         HelpCLI(parser)(options, args)
01101         parser.exit()
01102 
01103     # get the command
01104     command = args[0]
01105     if command not in commands:
01106         parser.error("Command must be one of %s (you gave '%s')" % (', '.join(sorted(commands.keys())), command))
01107 
01108     handler = commands[command](parser)
01109     handler(options, args[1:])
01110 
01111 if __name__ == '__main__':
01112     main()