Back to index

obnam  1.1
restore_plugin.py
Go to the documentation of this file.
00001 # Copyright (C) 2009, 2010  Lars Wirzenius
00002 #
00003 # This program is free software: you can redistribute it and/or modify
00004 # it under the terms of the GNU General Public License as published by
00005 # the Free Software Foundation, either version 3 of the License, or
00006 # (at your option) any later version.
00007 #
00008 # This program is distributed in the hope that it will be useful,
00009 # but WITHOUT ANY WARRANTY; without even the implied warranty of
00010 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00011 # GNU General Public License for more details.
00012 #
00013 # You should have received a copy of the GNU General Public License
00014 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
00015 
00016 
00017 import logging
00018 import os
00019 import stat
00020 import ttystatus
00021 
00022 import obnamlib
00023 
00024 
00025 class Hardlinks(object):
00026 
00027     '''Keep track of inodes with unrestored hardlinks.'''
00028     
00029     def __init__(self):
00030         self.inodes = dict()
00031         
00032     def key(self, metadata):
00033         return '%s:%s' % (metadata.st_dev, metadata.st_ino)
00034         
00035     def add(self, filename, metadata):
00036         self.inodes[self.key(metadata)] = (filename, metadata.st_nlink)
00037         
00038     def filename(self, metadata):
00039         key = self.key(metadata)
00040         if key in self.inodes:
00041             return self.inodes[key][0]
00042         else:
00043             return None
00044         
00045     def forget(self, metadata):
00046         key = self.key(metadata)
00047         filename, nlinks = self.inodes[key]
00048         if nlinks <= 2:
00049             del self.inodes[key]
00050         else:
00051             self.inodes[key] = (filename, nlinks - 1)
00052 
00053 
00054 class RestorePlugin(obnamlib.ObnamPlugin):
00055 
00056     # A note about the implementation: we need to make sure all the
00057     # files we restore go into the target directory. We do this by
00058     # prefixing all filenames we write to with './', and then using
00059     # os.path.join to put the target directory name at the beginning.
00060     # The './' business is necessary because os.path.join(a,b) returns
00061     # just b if b is an absolute path.
00062 
00063     def enable(self):
00064         self.app.add_subcommand('restore', self.restore, 
00065                                 arg_synopsis='[FILE]...')
00066         self.app.settings.string(['to'], 'where to restore')
00067         self.app.settings.string(['generation'], 
00068                                 'which generation to restore',
00069                                  default='latest')
00070 
00071     @property
00072     def write_ok(self):
00073         return not self.app.settings['dry-run']
00074 
00075     def configure_ttystatus(self):
00076         self.app.ts['current'] = ''
00077         self.app.ts['total'] = 0
00078         self.app.ts['current-bytes'] = 0
00079         self.app.ts['total-bytes'] = 0
00080         
00081         self.app.ts.format('%RemainingTime(current-bytes,total-bytes) '
00082                            '%Counter(current) files '
00083                            '%ByteSize(current-bytes) '
00084                            '(%PercentDone(current-bytes,total-bytes)) '
00085                            '%ByteSpeed(current-bytes) '
00086                            '%Pathname(current)')
00087 
00088     def restore(self, args):
00089         '''Restore some or all files from a generation.'''
00090         self.app.settings.require('repository')
00091         self.app.settings.require('client-name')
00092         self.app.settings.require('generation')
00093         self.app.settings.require('to')
00094 
00095         logging.debug('restoring generation %s' % 
00096                         self.app.settings['generation'])
00097         logging.debug('restoring to %s' % self.app.settings['to'])
00098     
00099         logging.debug('restoring what: %s' % repr(args))
00100         if not args:
00101             logging.debug('no args given, so restoring everything')
00102             args = ['/']
00103     
00104         self.repo = self.app.open_repository()
00105         self.repo.open_client(self.app.settings['client-name'])
00106         if self.write_ok:
00107             self.fs = self.app.fsf.new(self.app.settings['to'], create=True)
00108             self.fs.connect()
00109         else:
00110             self.fs = None # this will trigger error if we try to really write
00111 
00112         self.hardlinks = Hardlinks()
00113         
00114         self.errors = False
00115         
00116         gen = self.repo.genspec(self.app.settings['generation'])
00117 
00118         self.configure_ttystatus()
00119         self.app.ts['total'] = self.repo.client.get_generation_file_count(gen)
00120         self.app.ts['total-bytes'] = self.repo.client.get_generation_data(gen)
00121 
00122         self.app.dump_memory_profile('at beginning after setup')
00123 
00124         for arg in args:
00125             self.restore_something(gen, arg)
00126             self.app.dump_memory_profile('at restoring %s' % repr(arg))
00127 
00128         self.repo.fs.close()
00129         if self.write_ok:
00130             self.fs.close()
00131         
00132         self.app.ts.finish()
00133                 
00134         if self.errors:
00135             raise obnamlib.Error('There were errors when restoring')
00136 
00137     def restore_something(self, gen, root):
00138         for pathname, metadata in self.repo.walk(gen, root, depth_first=True):
00139             self.app.ts['current'] = pathname
00140             dirname = os.path.dirname(pathname)
00141             if self.write_ok and not self.fs.exists('./' + dirname):
00142                 self.fs.makedirs('./' + dirname)
00143     
00144             set_metadata = True
00145             if metadata.isdir():
00146                 self.restore_dir(gen, pathname, metadata)
00147             elif metadata.islink():
00148                 self.restore_symlink(gen, pathname, metadata)
00149             elif metadata.st_nlink > 1:
00150                 link = self.hardlinks.filename(metadata)
00151                 if link:
00152                     self.restore_hardlink(pathname, link, metadata)
00153                     set_metadata = False
00154                 else:
00155                     self.hardlinks.add(pathname, metadata)
00156                     self.restore_first_link(gen, pathname, metadata)
00157             else:
00158                 self.restore_first_link(gen, pathname, metadata)
00159             if set_metadata and self.write_ok:
00160                 obnamlib.set_metadata(self.fs, './' + pathname, metadata)
00161 
00162     def restore_dir(self, gen, root, metadata):
00163         logging.debug('restoring dir %s' % root)
00164         if self.write_ok:
00165             if not self.fs.exists('./' + root):
00166                 self.fs.mkdir('./' + root)
00167         self.app.dump_memory_profile('after recursing through %s' % repr(root))
00168 
00169     def restore_hardlink(self, filename, link, metadata):
00170         logging.debug('restoring hardlink %s to %s' % (filename, link))
00171         if self.write_ok:
00172             self.fs.link('./' + link, './' + filename)
00173             self.hardlinks.forget(metadata)
00174         
00175     def restore_symlink(self, gen, filename, metadata):
00176         logging.debug('restoring symlink %s' % filename)
00177 
00178     def restore_first_link(self, gen, filename, metadata):
00179         if stat.S_ISREG(metadata.st_mode):
00180             self.restore_regular_file(gen, filename, metadata)
00181         elif stat.S_ISFIFO(metadata.st_mode):
00182             self.restore_fifo(gen, filename, metadata)
00183         elif stat.S_ISSOCK(metadata.st_mode):
00184             self.restore_socket(gen, filename, metadata)
00185         else:
00186             msg = ('Unknown file type: %s (%o)' % 
00187                    (filename, metadata.st_mode))
00188             logging.error(msg)
00189             self.app.ts.notify(msg)
00190         
00191     def restore_regular_file(self, gen, filename, metadata):
00192         logging.debug('restoring regular %s' % filename)
00193         if self.write_ok:
00194             f = self.fs.open('./' + filename, 'wb')
00195             summer = self.repo.new_checksummer()
00196 
00197             try:
00198                 contents = self.repo.get_file_data(gen, filename)
00199                 if contents is None:
00200                     chunkids = self.repo.get_file_chunks(gen, filename)
00201                     self.restore_chunks(f, chunkids, summer)
00202                 else:
00203                     f.write(contents)
00204                     summer.update(contents)
00205             except obnamlib.MissingFilterError, e:
00206                 msg = 'Missing filter error during restore: %s' % filename
00207                 logging.error(msg)
00208                 self.app.ts.notify(msg)
00209                 self.errors = True
00210             f.close()
00211 
00212             correct_checksum = metadata.md5
00213             if summer.digest() != correct_checksum:
00214                 msg = 'File checksum restore error: %s' % filename
00215                 logging.error(msg)
00216                 self.app.ts.notify(msg)
00217                 self.errors = True
00218 
00219     def restore_chunks(self, f, chunkids, checksummer):
00220         zeroes = ''
00221         hole_at_end = False
00222         for chunkid in chunkids:
00223             data = self.repo.get_chunk(chunkid)
00224             self.verify_chunk_checksum(data, chunkid)
00225             checksummer.update(data)
00226             if len(data) != len(zeroes):
00227                 zeroes = '\0' * len(data)
00228             if data == zeroes:
00229                 f.seek(len(data), 1)
00230                 hole_at_end = True
00231             else:
00232                 f.write(data)
00233                 hole_at_end = False
00234             self.app.ts['current-bytes'] += len(data)
00235         if hole_at_end:
00236             pos = f.tell()
00237             if pos > 0:
00238                 f.seek(-1, 1)
00239                 f.write('\0')
00240 
00241     def verify_chunk_checksum(self, data, chunkid):
00242         checksum = self.repo.checksum(data)
00243         try:
00244             wanted = self.repo.chunklist.get_checksum(chunkid)
00245         except KeyError:
00246             # Chunk might not be in the tree, but that does not
00247             # mean it is invalid. We'll assume it is valid.
00248             return
00249         if checksum != wanted:
00250             raise obnamlib.Error('chunk %s checksum error' % chunkid)
00251 
00252     def restore_fifo(self, gen, filename, metadata):
00253         logging.debug('restoring fifo %s' % filename)
00254         if self.write_ok:
00255             self.fs.mknod('./' + filename, metadata.st_mode)
00256 
00257     def restore_socket(self, gen, filename, metadata):
00258         logging.debug('restoring socket %s' % filename)
00259         if self.write_ok:
00260             self.fs.mknod('./' + filename, metadata.st_mode)
00261