Back to index

python-cliapp  1.20120630
app.py
Go to the documentation of this file.
00001 # Copyright (C) 2011  Lars Wirzenius
00002 # 
00003 # This program is free software; you can redistribute it and/or modify
00004 # it under the terms of the GNU General Public License as published by
00005 # the Free Software Foundation; either version 2 of the License, or
00006 # (at your option) any later version.
00007 # 
00008 # This program is distributed in the hope that it will be useful,
00009 # but WITHOUT ANY WARRANTY; without even the implied warranty of
00010 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00011 # GNU General Public License for more details.
00012 # 
00013 # You should have received a copy of the GNU General Public License along
00014 # with this program; if not, write to the Free Software Foundation, Inc.,
00015 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
00016 
00017 
00018 import errno
00019 import gc
00020 import inspect
00021 import logging
00022 import logging.handlers
00023 import os
00024 import StringIO
00025 import sys
00026 import traceback
00027 
00028 import cliapp
00029 
00030 
00031 class AppException(Exception):
00032 
00033     '''Base class for application specific exceptions.
00034     
00035     Any exceptions that are subclasses of this one get printed as
00036     nice errors to the user. Any other exceptions cause a Python
00037     stack trace to be written to stderr.
00038     
00039     '''
00040     
00041     def __init__(self, msg):
00042         self.msg = msg
00043         
00044     def __str__(self):
00045         return self.msg
00046     
00047     
00048 class LogHandler(logging.handlers.RotatingFileHandler): # pragma: no cover
00049 
00050     '''Like RotatingFileHandler, but set permissions of new files.'''
00051     
00052     def __init__(self, filename, perms=0600, *args, **kwargs):
00053         self._perms = perms
00054         logging.handlers.RotatingFileHandler.__init__(self, filename, 
00055                                                       *args, **kwargs)
00056 
00057     def _open(self):
00058         if not os.path.exists(self.baseFilename):
00059             flags = os.O_CREAT | os.O_WRONLY
00060             fd = os.open(self.baseFilename, flags, self._perms)
00061             os.close(fd)
00062         return logging.handlers.RotatingFileHandler._open(self)
00063 
00064 
00065 class Application(object):
00066 
00067     '''A framework for Unix-like command line programs.
00068     
00069     The user should subclass this base class for each application.
00070     The subclass does not need code for the mundane, boilerplate
00071     parts that are the same in every utility, and can concentrate on the 
00072     interesting part that is unique to it.
00073 
00074     To start the application, call the `run` method.
00075     
00076     The ``progname`` argument sets tne name of the program, which is
00077     used for various purposes, such as determining the name of the
00078     configuration file.
00079     
00080     Similarly, ``version`` sets the version number of the program.
00081     
00082     ``description`` and ``epilog`` are included in the output of
00083     ``--help``. They are formatted to fit the screen. Unlike the
00084     default behavior of ``optparse``, empty lines separate
00085     paragraphs.
00086     
00087     '''
00088 
00089     def __init__(self, progname=None, version='0.0.0', description=None,
00090                  epilog=None):
00091         self.fileno = 0
00092         self.global_lineno = 0
00093         self.lineno = 0
00094         self._description = description
00095         if not hasattr(self, 'arg_synopsis'):
00096             self.arg_synopsis = '[FILE]...'
00097         if not hasattr(self, 'cmd_synopsis'):
00098             self.cmd_synopsis = {}
00099 
00100         self.subcommands = {}
00101         for method_name in self._subcommand_methodnames():
00102             cmd = self._unnormalize_cmd(method_name)
00103             self.subcommands[cmd] = getattr(self, method_name)
00104         
00105         self.settings = cliapp.Settings(progname, version, 
00106                                         usage=self._format_usage,
00107                                         description=self._format_description,
00108                                         epilog=epilog)
00109 
00110         self.plugin_subdir = 'plugins'
00111         
00112     def add_settings(self):
00113         '''Add application specific settings.'''
00114 
00115     def run(self, args=None, stderr=sys.stderr, sysargv=sys.argv, 
00116             log=logging.critical):
00117         '''Run the application.'''
00118         
00119         def run_it():
00120             self._run(args=args, stderr=stderr, log=log)
00121 
00122         if self.settings.progname is None and sysargv:
00123             self.settings.progname = os.path.basename(sysargv[0])
00124         envname = '%s_PROFILE' % self._envname(self.settings.progname)
00125         profname = os.environ.get(envname, '')
00126         if profname: # pragma: no cover
00127             import cProfile
00128             cProfile.runctx('run_it()', globals(), locals(), profname)
00129         else:
00130             run_it()
00131 
00132     def _envname(self, progname):
00133         '''Create an environment variable name of the name of a program.'''
00134         
00135         basename = os.path.basename(progname)
00136         if '.' in basename:
00137             basename = basename.split('.')[0]
00138         
00139         ok = 'abcdefghijklmnopqrstuvwxyz0123456789'
00140         ok += ok.upper()
00141         
00142         return ''.join(x.upper() if x in ok else '_' for x in basename)
00143 
00144     def _run(self, args=None, stderr=sys.stderr, log=logging.critical):
00145         try:
00146             self.add_settings()
00147             self.setup_plugin_manager()
00148             
00149             # A little bit of trickery here to make --no-default-configs and
00150             # --config=foo work right: we first parse the command line once,
00151             # and pick up any config files. Then we read configs. Finally,
00152             # we re-parse the command line to allow any options to override
00153             # config file settings.
00154             self.setup()
00155             self.enable_plugins()
00156             args = sys.argv[1:] if args is None else args
00157             self.parse_args(args, configs_only=True)
00158             self.settings.load_configs()
00159             args = self.parse_args(args)
00160 
00161             self.setup_logging()
00162             self.log_config()
00163             
00164             if self.settings['output']:
00165                 self.output = open(self.settings['output'], 'w')
00166             else:
00167                 self.output = sys.stdout
00168 
00169             self.process_args(args)
00170             self.cleanup()
00171             self.disable_plugins()
00172         except AppException, e:
00173             log(traceback.format_exc())
00174             stderr.write('ERROR: %s\n' % str(e))
00175             sys.exit(1)
00176         except SystemExit, e:
00177             sys.exit(e.code if type(e.code) == int else 1)
00178         except KeyboardInterrupt, e:
00179             sys.exit(255)
00180         except IOError, e: # pragma: no cover
00181             if e.errno == errno.EPIPE and e.filename is None:
00182                 # We're writing to stdout, and it broke. This almost always
00183                 # happens when we're being piped to less, and the user quits
00184                 # less before we finish writing everything out. So we ignore
00185                 # the error in that case.
00186                 sys.exit(1)
00187             log(traceback.format_exc())
00188             stderr.write('ERROR: %s\n' % str(e))
00189             sys.exit(1)
00190         except OSError, e: # pragma: no cover
00191             log(traceback.format_exc())
00192             if hasattr(e, 'filename') and e.filename:
00193                 stderr.write('ERROR: %s: %s\n' % (e.filename, e.strerror))
00194             else:
00195                 stderr.write('ERROR: %s\n' % e.strerror)
00196             sys.exit(1)
00197         except BaseException, e: # pragma: no cover
00198             log(traceback.format_exc())
00199             stderr.write(traceback.format_exc())
00200             sys.exit(1)
00201 
00202         logging.info('%s version %s ends normally' % 
00203                      (self.settings.progname, self.settings.version))
00204     
00205     def add_subcommand(self, name, func, arg_synopsis=None):
00206         '''Add a subcommand.
00207         
00208         Normally, subcommands are defined by add ``cmd_foo`` methods
00209         to the application class. However, sometimes it is more convenient
00210         to have them elsewhere (e.g., in plugins). This method allows
00211         doing that.
00212         
00213         The callback function must accept a list of command line
00214         non-option arguments.
00215         
00216         '''
00217         
00218         if name not in self.subcommands:
00219             self.subcommands[name] = func
00220             self.cmd_synopsis[name] = arg_synopsis
00221     
00222     def _subcommand_methodnames(self):
00223         return [x 
00224                  for x in dir(self) 
00225                  if x.startswith('cmd_') and 
00226                     inspect.ismethod(getattr(self, x))]
00227 
00228     def _normalize_cmd(self, cmd):
00229         return 'cmd_%s' % cmd.replace('-', '_')
00230 
00231     def _unnormalize_cmd(self, method):
00232         assert method.startswith('cmd_')
00233         return method[len('cmd_'):].replace('_', '-')
00234 
00235     def _format_usage(self):
00236         '''Format usage, possibly also subcommands, if any.'''
00237         if self.subcommands:
00238             lines = []
00239             prefix = 'Usage:'
00240             for cmd in sorted(self.subcommands.keys()):
00241                 args = self.cmd_synopsis.get(cmd, '')
00242                 lines.append('%s %%prog [options] %s %s' % (prefix, cmd, args))
00243                 prefix = ' ' * len(prefix)
00244             return '\n'.join(lines)
00245         else:
00246             return None
00247 
00248     def _format_description(self):
00249         '''Format OptionParser description, with subcommand support.'''
00250         if self.subcommands:
00251             paras = []
00252             for cmd in sorted(self.subcommands.keys()):
00253                 paras.append(self._format_subcommand_description(cmd))
00254             cmd_desc = '\n\n'.join(paras)
00255             return '%s\n\n%s' % (self._description or '', cmd_desc)
00256         else:
00257             return self._description
00258 
00259     def _format_subcommand_description(self, cmd): # pragma: no cover
00260 
00261         def remove_empties(lines):
00262             while lines and not lines[0].strip():
00263                 del lines[0]
00264 
00265         def split_para(lines):
00266             para = []
00267             while lines and lines[0].strip():
00268                 para.append(lines[0].strip())
00269                 del lines[0]
00270             return para
00271 
00272         indent = ' ' * 4
00273         method = self.subcommands[cmd]
00274         doc = method.__doc__ or ''
00275         lines = doc.splitlines()
00276         remove_empties(lines)
00277         if lines:
00278             heading = '* %s -- %s' % (cmd, lines[0])
00279             result = [heading]
00280             del lines[0]
00281             remove_empties(lines)
00282             while lines:
00283                 result.append('')
00284                 para_lines = split_para(lines)
00285                 para_text = ' '.join(para_lines)
00286                 result.append(para_text)
00287                 remove_empties(lines)
00288             return '\n'.join(result)
00289         else:
00290             return '* %s' % cmd
00291         
00292     def setup_logging(self): # pragma: no cover
00293         '''Set up logging.'''
00294         
00295         level_name = self.settings['log-level']
00296         levels = {
00297             'debug': logging.DEBUG,
00298             'info': logging.INFO,
00299             'warning': logging.WARNING,
00300             'error': logging.ERROR,
00301             'critical': logging.CRITICAL,
00302             'fatal': logging.FATAL,
00303         }
00304         level = levels.get(level_name, logging.INFO)
00305 
00306         if self.settings['log'] == 'syslog':
00307             handler = logging.handlers.SysLogHandler(address='/dev/log')
00308         elif self.settings['log'] and self.settings['log'] != 'none':
00309             handler = LogHandler(
00310                             self.settings['log'],
00311                             perms=int(self.settings['log-mode'], 8),
00312                             maxBytes=self.settings['log-max'], 
00313                             backupCount=self.settings['log-keep'],
00314                             delay=False)
00315         else:
00316             handler = logging.FileHandler('/dev/null')
00317             # reduce amount of pointless I/O
00318             level = logging.FATAL
00319 
00320         fmt = '%(asctime)s %(levelname)s %(message)s'
00321         datefmt = '%Y-%m-%d %H:%M:%S'
00322         formatter = logging.Formatter(fmt, datefmt)
00323         handler.setFormatter(formatter)
00324 
00325         logger = logging.getLogger()
00326         logger.addHandler(handler)
00327         logger.setLevel(level)
00328 
00329     def log_config(self):
00330         logging.info('%s version %s starts' % 
00331                      (self.settings.progname, self.settings.version))
00332         logging.debug('sys.argv: %s' % sys.argv)
00333         logging.debug('environment variables:')
00334         for name in os.environ:
00335             logging.debug('environment: %s=%s' % (name, os.environ[name]))
00336         cp = self.settings.as_cp()
00337         f = StringIO.StringIO()
00338         cp.write(f)
00339         logging.debug('Config:\n%s' % f.getvalue())
00340 
00341     def app_directory(self):
00342         '''Return the directory where the application class is defined.
00343         
00344         Plugins are searched relative to this directory, in the subdirectory
00345         specified by self.plugin_subdir.
00346         
00347         '''
00348         
00349         module_name = self.__class__.__module__
00350         module = sys.modules[module_name]
00351         dirname = os.path.dirname(module.__file__) or '.'
00352         return dirname
00353 
00354     def setup_plugin_manager(self):
00355         '''Create a plugin manager.'''
00356         self.pluginmgr = cliapp.PluginManager()
00357         dirname = os.path.join(self.app_directory(), self.plugin_subdir)
00358         self.pluginmgr.locations = [dirname]
00359 
00360     def enable_plugins(self): # pragma: no cover
00361         '''Load plugins.'''
00362         for plugin in self.pluginmgr.plugins:
00363             plugin.app = self
00364             plugin.setup()
00365         self.pluginmgr.enable_plugins()
00366 
00367     def disable_plugins(self):
00368         self.pluginmgr.disable_plugins()
00369 
00370     def parse_args(self, args, configs_only=False):
00371         '''Parse the command line.
00372         
00373         Return list of non-option arguments.
00374         
00375         '''
00376 
00377         return self.settings.parse_args(args, configs_only=configs_only,
00378                                          arg_synopsis=self.arg_synopsis,
00379                                          cmd_synopsis=self.cmd_synopsis)
00380 
00381     def setup(self):
00382         '''Prepare for process_args.
00383         
00384         This method is called just before process_args. By default it
00385         does nothing, but subclasses may override it with a suitable
00386         implementation. This is easier than overriding process_args
00387         itself.
00388         
00389         '''
00390 
00391     def cleanup(self):
00392         '''Clean up after process_args.
00393         
00394         This method is called just after process_args. By default it
00395         does nothing, but subclasses may override it with a suitable
00396         implementation. This is easier than overriding process_args
00397         itself.
00398         
00399         '''
00400 
00401     def process_args(self, args):
00402         '''Process command line non-option arguments.
00403         
00404         The default is to call process_inputs with the argument list,
00405         or to invoke the requested subcommand, if subcommands have
00406         been defined.
00407         
00408         '''
00409         
00410             
00411         if self.subcommands:
00412             if not args:
00413                 raise SystemExit('must give subcommand')
00414             if args[0] in self.subcommands:
00415                 method = self.subcommands[args[0]]
00416                 method(args[1:])
00417             else:
00418                 raise SystemExit('unknown subcommand %s' % args[0])
00419         else:
00420             self.process_inputs(args)
00421 
00422     def process_inputs(self, args):
00423         '''Process all arguments as input filenames.
00424         
00425         The default implementation calls process_input for each
00426         input filename. If no filenames were given, then 
00427         process_input is called with ``-`` as the argument name.
00428         This implements the usual Unix command line practice of
00429         reading from stdin if no inputs are named.
00430         
00431         The attributes ``fileno``, ``global_lineno``, and ``lineno`` are set,
00432         and count files and lines. The global line number is the
00433         line number as if all input files were one.
00434         
00435         '''
00436 
00437         for arg in args or ['-']:
00438             self.process_input(arg)
00439 
00440     def open_input(self, name, mode='r'):
00441         '''Open an input file for reading.
00442         
00443         The default behaviour is to open a file named on the local
00444         filesystem. A subclass might override this behavior for URLs,
00445         for example.
00446         
00447         The optional mode argument speficies the mode in which the file
00448         gets opened. It should allow reading. Some files should perhaps
00449         be opened in binary mode ('rb') instead of the default text mode.
00450         
00451         '''
00452         
00453         if name == '-':
00454             return sys.stdin
00455         else:
00456             return open(name, mode)
00457 
00458     def process_input(self, name, stdin=sys.stdin):
00459         '''Process a particular input file.
00460         
00461         The ``stdin`` argument is meant for unit test only.
00462         
00463         '''
00464 
00465         self.fileno += 1
00466         self.lineno = 0
00467         f = self.open_input(name)
00468         for line in f:
00469             self.global_lineno += 1
00470             self.lineno += 1
00471             self.process_input_line(name, line)
00472         if f != stdin:
00473             f.close()
00474 
00475     def process_input_line(self, filename, line):
00476         '''Process one line of the input file.
00477         
00478         Applications that are line-oriented can redefine only this method in
00479         a subclass, and should not need to care about the other methods.
00480         
00481         '''
00482         
00483     def runcmd(self, *args, **kwargs): # pragma: no cover
00484         return cliapp.runcmd(*args, **kwargs)
00485         
00486     def runcmd_unchecked(self, *args, **kwargs): # pragma: no cover
00487         return cliapp.runcmd_unchecked(*args, **kwargs)
00488 
00489     def _vmrss(self): # pragma: no cover
00490         '''Return current resident memory use, in KiB.'''
00491         f = open('/proc/self/status')
00492         rss = 0
00493         for line in f:
00494             if line.startswith('VmRSS'):
00495                 rss = line.split()[1]
00496         f.close()
00497         return rss
00498 
00499     def dump_memory_profile(self, msg): # pragma: no cover
00500         '''Log memory profiling information.
00501         
00502         Get the memory profiling method from the dump-memory-profile
00503         setting, and log the results at DEBUG level. ``msg`` is a
00504         message the caller provides to identify at what point the profiling
00505         happens.
00506         
00507         '''
00508 
00509         kind = self.settings['dump-memory-profile']
00510 
00511         if kind == 'none':
00512             return
00513 
00514         logging.debug('dumping memory profiling data: %s' % msg)
00515         logging.debug('VmRSS: %s KiB' % self._vmrss())
00516         
00517         if kind == 'simple':
00518             return
00519 
00520         # These are fairly expensive operations, so we only log them
00521         # if we're doing expensive stuff anyway.
00522         logging.debug('# objects: %d' % len(gc.get_objects()))
00523         logging.debug('# garbage: %d' % len(gc.garbage))
00524 
00525         if kind == 'heapy':
00526             from guppy import hpy
00527             h = hpy()
00528             logging.debug('memory profile:\n%s' % h.heap())
00529         elif kind == 'meliae':
00530             filename = 'obnam-%d.meliae' % self.memory_dump_counter
00531             logging.debug('memory profile: see %s' % filename)
00532             from meliae import scanner
00533             scanner.dump_all_objects(filename)
00534             self.memory_dump_counter += 1
00535