Back to index

obnam  1.1
verify_plugin.py
Go to the documentation of this file.
00001 # Copyright (C) 2010  Lars Wirzenius
00002 #
00003 # This program is free software: you can redistribute it and/or modify
00004 # it under the terms of the GNU General Public License as published by
00005 # the Free Software Foundation, either version 3 of the License, or
00006 # (at your option) any later version.
00007 #
00008 # This program is distributed in the hope that it will be useful,
00009 # but WITHOUT ANY WARRANTY; without even the implied warranty of
00010 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00011 # GNU General Public License for more details.
00012 #
00013 # You should have received a copy of the GNU General Public License
00014 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
00015 
00016 
00017 import logging
00018 import os
00019 import random
00020 import stat
00021 import sys
00022 import urlparse
00023 
00024 import obnamlib
00025 
00026 
00027 class Fail(obnamlib.Error):
00028 
00029     def __init__(self, filename, reason):
00030         self.filename = filename
00031         self.reason = reason
00032 
00033     def __str__(self):
00034         return '%s: %s' % (self.filename, self.reason)
00035 
00036 
00037 class VerifyPlugin(obnamlib.ObnamPlugin):
00038 
00039     def enable(self):
00040         self.app.add_subcommand('verify', self.verify,
00041                                 arg_synopsis='[FILE]...')
00042         self.app.settings.integer(['verify-randomly'],
00043                                   'verify N files randomly from the backup '
00044                                     '(default is zero, meaning everything)',
00045                                   metavar='N')
00046 
00047     def verify(self, args):
00048         '''Verify that live data and backed up data match.'''
00049         self.app.settings.require('repository')
00050         self.app.settings.require('client-name')
00051         self.app.settings.require('generation')
00052 
00053         logging.debug('verifying generation %s' % 
00054                         self.app.settings['generation'])
00055         if not args:
00056             self.app.settings.require('root')
00057             args = self.app.settings['root']
00058         if not args:
00059             logging.debug('no roots/args given, so verifying everything')
00060             args = ['/']
00061         logging.debug('verifying what: %s' % repr(args))
00062 
00063         self.repo = self.app.open_repository()
00064         self.repo.open_client(self.app.settings['client-name'])
00065         self.fs = self.app.fsf.new(args[0])
00066         self.fs.connect()
00067         t = urlparse.urlparse(args[0])
00068         root_url = urlparse.urlunparse((t[0], t[1], '/', t[3], t[4], t[5]))
00069         logging.debug('t: %s' % repr(t))
00070         logging.debug('root_url: %s' % repr(root_url))
00071         self.fs.reinit(root_url)
00072 
00073         self.failed = False
00074         gen = self.repo.genspec(self.app.settings['generation'])
00075 
00076         self.app.ts['done'] = 0
00077         self.app.ts['total'] = 0
00078         self.app.ts['filename'] = ''
00079         if not self.app.settings['quiet']:
00080             self.app.ts.format(
00081                 'verifying file %Counter(filename)/%Integer(total) '
00082                 '%PercentDone(done,total): '
00083                 '%Pathname(filename)')
00084 
00085         num_randomly = self.app.settings['verify-randomly']
00086         if num_randomly == 0:
00087             self.app.ts['total'] = \
00088                 self.repo.client.get_generation_file_count(gen)
00089             for filename, metadata in self.walk(gen, args):
00090                 self.app.ts['filename'] = filename
00091                 try:
00092                     self.verify_metadata(gen, filename, metadata)
00093                 except Fail, e:
00094                     self.log_fail(e)
00095                 else:
00096                     if metadata.isfile():
00097                         try:
00098                             self.verify_regular_file(gen, filename, metadata)
00099                         except Fail, e:
00100                             self.log_fail(e)
00101                 self.app.ts['done'] += 1
00102         else:
00103             logging.debug('verifying %d files randomly' % num_randomly)
00104             self.app.ts['total'] = num_randomly
00105             self.app.ts.notify('finding all files to choose randomly')
00106             filenames = [filename
00107                          for filename, metadata in self.walk(gen, args)
00108                          if metadata.isfile()]
00109             chosen = []
00110             for i in range(min(num_randomly, len(filenames))):
00111                 filename = random.choice(filenames)
00112                 filenames.remove(filename)
00113                 chosen.append(filename)
00114             for filename in chosen:
00115                 self.app.ts['filename'] = filename
00116                 metadata = self.repo.get_metadata(gen, filename)
00117                 try:
00118                     self.verify_metadata(gen, filename, metadata)
00119                     self.verify_regular_file(gen, filename, metadata)            
00120                 except Fail, e:
00121                     self.log_fail(e)
00122                 self.app.ts['done'] += 1
00123 
00124         self.fs.close()
00125         self.repo.fs.close()
00126         self.app.ts.finish()
00127 
00128         if self.failed:
00129             sys.exit(1)
00130         print "Verify did not find problems."
00131 
00132     def log_fail(self, e):
00133         msg = 'verify failure: %s: %s' % (e.filename, e.reason)
00134         logging.error(msg)
00135         if self.app.settings['quiet']:
00136             sys.stderr.write('%s\n' % msg)
00137         else:
00138             self.app.ts.notify(msg)
00139         self.failed = True
00140 
00141     def verify_metadata(self, gen, filename, backed_up):
00142         try:
00143             live_data = obnamlib.read_metadata(self.fs, filename)
00144         except OSError, e:
00145             raise Fail(filename, 'missing or inaccessible: %s' % e.strerror)
00146         for field in obnamlib.metadata_verify_fields:
00147             v1 = getattr(backed_up, field)
00148             v2 = getattr(live_data, field)
00149             if v1 != v2:
00150                 raise Fail(filename, 
00151                             'metadata change: %s (%s vs %s)' % (field, v1, v2))
00152 
00153     def verify_regular_file(self, gen, filename, metadata):
00154         logging.debug('verifying regular %s' % filename)
00155         f = self.fs.open(filename, 'r')
00156 
00157         chunkids = self.repo.get_file_chunks(gen, filename)
00158         if not self.verify_chunks(f, chunkids):
00159             raise Fail(filename, 'data changed')
00160 
00161         f.close()
00162 
00163     def verify_chunks(self, f, chunkids):
00164         for chunkid in chunkids:
00165             backed_up = self.repo.get_chunk(chunkid)
00166             live_data = f.read(len(backed_up))
00167             if backed_up != live_data:
00168                 return False
00169         return True
00170 
00171     def walk(self, gen, args):
00172         '''Iterate over each pathname specified by arguments.
00173         
00174         This is a generator.
00175         
00176         '''
00177         
00178         for arg in args:
00179             scheme, netloc, path, query, fragment = urlparse.urlsplit(arg)
00180             arg = os.path.normpath(path)
00181             for x in self.repo.walk(gen, arg):
00182                 yield x
00183