Back to index

obnam  1.1
vfs_local.py
Go to the documentation of this file.
00001 # Copyright (C) 2008  Lars Wirzenius <liw@liw.fi>
00002 #
00003 # This program is free software; you can redistribute it and/or modify
00004 # it under the terms of the GNU General Public License as published by
00005 # the Free Software Foundation; either version 2 of the License, or
00006 # (at your option) any later version.
00007 #
00008 # This program is distributed in the hope that it will be useful,
00009 # but WITHOUT ANY WARRANTY; without even the implied warranty of
00010 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00011 # GNU General Public License for more details.
00012 #
00013 # You should have received a copy of the GNU General Public License along
00014 # with this program; if not, write to the Free Software Foundation, Inc.,
00015 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
00016 
00017 
00018 import errno
00019 import fcntl
00020 import grp
00021 import logging
00022 import math
00023 import os
00024 import pwd
00025 import tempfile
00026 import tracing
00027 
00028 import obnamlib
00029 
00030 
00031 class LocalFSFile(file):
00032 
00033     def read(self, amount=-1):
00034         offset = self.tell()
00035         data = file.read(self, amount)
00036         if data:
00037             fd = self.fileno()
00038             obnamlib._obnam.fadvise_dontneed(fd, offset, len(data))
00039         return data
00040 
00041     def write(self, data):
00042         offset = self.tell()
00043         file.write(self, data)
00044         fd = self.fileno()
00045         obnamlib._obnam.fadvise_dontneed(fd, offset, len(data))
00046 
00047 
00048 class LocalFS(obnamlib.VirtualFileSystem):
00049 
00050     """A VFS implementation for local filesystems."""
00051     
00052     chunk_size = 1024 * 1024
00053     
00054     def __init__(self, baseurl, create=False):
00055         tracing.trace('baseurl=%s', baseurl)
00056         tracing.trace('create=%s', create)
00057         obnamlib.VirtualFileSystem.__init__(self, baseurl)
00058         self.reinit(baseurl, create=create)
00059 
00060         # For testing purposes, allow setting a limit on write operations
00061         # after which an exception gets raised. If set to None, no crash.
00062         self.crash_limit = None
00063         self.crash_counter = 0
00064 
00065     def maybe_crash(self): # pragma: no cover
00066         if self.crash_limit is not None:
00067             self.crash_counter += 1
00068             if self.crash_counter >= self.crash_limit:
00069                 raise Exception('Crashing as requested after %d writes' %
00070                                     self.crash_counter)
00071 
00072     def reinit(self, baseurl, create=False):
00073         # We fake chdir so that it doesn't mess with the caller's 
00074         # perception of current working directory. This also benefits
00075         # unit tests. To do this, we store the baseurl as the cwd.
00076         tracing.trace('baseurl=%s', baseurl)
00077         tracing.trace('create=%s', create)
00078         self.cwd = os.path.abspath(baseurl)
00079         if not self.isdir('.'):
00080             if create:
00081                 tracing.trace('creating %s', baseurl)
00082                 try:
00083                     os.mkdir(baseurl)
00084                 except OSError, e: # pragma: no cover
00085                     # The directory might have been created concurrently
00086                     # by someone else!
00087                     if e.errno != errno.EEXIST:
00088                         raise
00089             else:
00090                 err = errno.ENOENT
00091                 raise OSError(err, os.strerror(err), self.cwd)
00092 
00093     def getcwd(self):
00094         return self.cwd
00095 
00096     def chdir(self, pathname):
00097         tracing.trace('LocalFS(%s).chdir(%s)', self.baseurl, pathname)
00098         newcwd = os.path.abspath(self.join(pathname))
00099         if not os.path.isdir(newcwd):
00100             raise OSError('%s is not a directory' % newcwd)
00101         self.cwd = newcwd
00102 
00103     def lock(self, lockname, data):
00104         tracing.trace('lockname=%s', lockname)
00105         try:
00106             self.write_file(lockname, data)
00107         except OSError, e:
00108             if e.errno == errno.EEXIST:
00109                 raise obnamlib.LockFail("Lock %s already exists" % lockname)
00110             else:
00111                 raise # pragma: no cover
00112 
00113     def unlock(self, lockname):
00114         tracing.trace('lockname=%s', lockname)
00115         if self.exists(lockname):
00116             self.remove(lockname)
00117 
00118     def join(self, pathname):
00119         return os.path.join(self.cwd, pathname)
00120 
00121     def remove(self, pathname):
00122         tracing.trace('remove %s', pathname)
00123         os.remove(self.join(pathname))
00124         self.maybe_crash()
00125 
00126     def rename(self, old, new):
00127         tracing.trace('rename %s %s', old, new)
00128         os.rename(self.join(old), self.join(new))
00129         self.maybe_crash()
00130 
00131     def lstat(self, pathname):
00132         (ret, dev, ino, mode, nlink, uid, gid, rdev, size, blksize, blocks,
00133          atime_sec, atime_nsec, mtime_sec, mtime_nsec, 
00134          ctime_sec, ctime_nsec) = obnamlib._obnam.lstat(self.join(pathname))
00135         if ret != 0:
00136             raise OSError((ret, os.strerror(ret), pathname))
00137         return obnamlib.Metadata(
00138                     st_dev=dev,
00139                     st_ino=ino,
00140                     st_mode=mode,
00141                     st_nlink=nlink,
00142                     st_uid=uid,
00143                     st_gid=gid,
00144                     st_rdev=rdev,
00145                     st_size=size,
00146                     st_blksize=blksize,
00147                     st_blocks=blocks,
00148                     st_atime_sec=atime_sec,
00149                     st_atime_nsec=atime_nsec,
00150                     st_mtime_sec=mtime_sec,
00151                     st_mtime_nsec=mtime_nsec,
00152                     st_ctime_sec=ctime_sec,
00153                     st_ctime_nsec=ctime_nsec
00154                 )
00155 
00156     def get_username(self, uid):
00157         return pwd.getpwuid(uid)[0]
00158 
00159     def get_groupname(self, gid):
00160         return grp.getgrgid(gid)[0]
00161 
00162     def llistxattr(self, filename): # pragma: no cover
00163         ret = obnamlib._obnam.llistxattr(self.join(filename))
00164         if type(ret) is int:
00165             raise OSError((ret, os.strerror(ret), filename))
00166         return [s for s in ret.split('\0') if s]
00167 
00168     def lgetxattr(self, filename, attrname): # pragma: no cover
00169         ret = obnamlib._obnam.lgetxattr(self.join(filename), attrname)
00170         if type(ret) is int:
00171             raise OSError((ret, os.strerror(ret), filename))
00172         return ret
00173 
00174     def lsetxattr(self, filename, attrname, attrvalue): # pragma: no cover
00175         ret = obnamlib._obnam.lsetxattr(self.join(filename), 
00176                                         attrname, attrvalue)
00177         if ret != 0:
00178             raise OSError((ret, os.strerror(ret), filename))
00179 
00180     def lchown(self, pathname, uid, gid): # pragma: no cover
00181         tracing.trace('lchown %s %d %d', pathname, uid, gid)
00182         os.lchown(self.join(pathname), uid, gid)
00183 
00184     def chmod(self, pathname, mode):
00185         tracing.trace('chmod %s %o', pathname, mode)
00186         os.chmod(self.join(pathname), mode)
00187 
00188     def lutimes(self, pathname, atime_sec, atime_nsec, mtime_sec, mtime_nsec):
00189         assert atime_sec is not None
00190         assert atime_nsec is not None
00191         assert mtime_sec is not None
00192         assert mtime_nsec is not None
00193         ret = obnamlib._obnam.utimensat(self.join(pathname), 
00194                                         atime_sec, atime_nsec, 
00195                                         mtime_sec, mtime_nsec)
00196         if ret != 0:
00197             raise OSError(ret, os.strerror(ret), pathname)
00198 
00199     def link(self, existing, new):
00200         tracing.trace('existing=%s', existing)
00201         tracing.trace('new=%s', new)
00202         os.link(self.join(existing), self.join(new))
00203         self.maybe_crash()
00204 
00205     def readlink(self, pathname):
00206         return os.readlink(self.join(pathname))
00207 
00208     def symlink(self, existing, new):
00209         tracing.trace('existing=%s', existing)
00210         tracing.trace('new=%s', new)
00211         os.symlink(existing, self.join(new))
00212         self.maybe_crash()
00213 
00214     def open(self, pathname, mode):
00215         tracing.trace('pathname=%s', pathname)
00216         tracing.trace('mode=%s', mode)
00217         f = LocalFSFile(self.join(pathname), mode)
00218         tracing.trace('opened %s', pathname)
00219         try:
00220             flags = fcntl.fcntl(f.fileno(), fcntl.F_GETFL)
00221             flags |= os.O_NOATIME
00222             fcntl.fcntl(f.fileno(), fcntl.F_SETFL, flags)
00223         except IOError, e: # pragma: no cover
00224             tracing.trace('fcntl F_SETFL failed: %s', repr(e))
00225             return f # ignore any problems setting flags
00226         tracing.trace('returning ok')
00227         return f
00228 
00229     def exists(self, pathname):
00230         return os.path.exists(self.join(pathname))
00231 
00232     def isdir(self, pathname):
00233         return os.path.isdir(self.join(pathname))
00234 
00235     def mknod(self, pathname, mode):
00236         tracing.trace('pathmame=%s', pathname)
00237         tracing.trace('mode=%o', mode)
00238         os.mknod(self.join(pathname), mode)
00239 
00240     def mkdir(self, pathname):
00241         tracing.trace('mkdir %s', pathname)
00242         os.mkdir(self.join(pathname))
00243         self.maybe_crash()
00244 
00245     def makedirs(self, pathname):
00246         tracing.trace('makedirs %s', pathname)
00247         os.makedirs(self.join(pathname))
00248         self.maybe_crash()
00249 
00250     def rmdir(self, pathname):
00251         tracing.trace('rmdir %s', pathname)
00252         os.rmdir(self.join(pathname))
00253         self.maybe_crash()
00254 
00255     def cat(self, pathname):
00256         pathname = self.join(pathname)
00257         f = self.open(pathname, 'rb')
00258         chunks = []
00259         while True:
00260             chunk = f.read(self.chunk_size)
00261             if not chunk:
00262                 break
00263             chunks.append(chunk)
00264             self.bytes_read += len(chunk)
00265         f.close()
00266         data = ''.join(chunks)
00267         return data
00268 
00269     def write_file(self, pathname, contents):
00270         tracing.trace('write_file %s', pathname)
00271         tempname = self._write_to_tempfile(pathname, contents)
00272         path = self.join(pathname)
00273         try:
00274             os.link(tempname, path)
00275         except OSError, e: # pragma: no cover
00276             os.remove(tempname)
00277             raise
00278         os.remove(tempname)
00279         self.maybe_crash()
00280 
00281     def overwrite_file(self, pathname, contents):
00282         tracing.trace('overwrite_file %s', pathname)
00283         tempname = self._write_to_tempfile(pathname, contents)
00284         path = self.join(pathname)
00285         os.rename(tempname, path)
00286         self.maybe_crash()
00287                 
00288     def _write_to_tempfile(self, pathname, contents):
00289         path = self.join(pathname)
00290         dirname = os.path.dirname(path)
00291         if not os.path.exists(dirname):
00292             tracing.trace('os.makedirs(%s)' % dirname)
00293             os.makedirs(dirname)
00294 
00295         fd, tempname = tempfile.mkstemp(dir=dirname)
00296         os.close(fd)
00297         f = self.open(tempname, 'wb')
00298 
00299         pos = 0
00300         while pos < len(contents):
00301             chunk = contents[pos:pos+self.chunk_size]
00302             f.write(chunk)
00303             pos += len(chunk)
00304             self.bytes_written += len(chunk)
00305         f.close()
00306         return tempname
00307 
00308     def listdir(self, dirname):
00309         return os.listdir(self.join(dirname))
00310 
00311     def listdir2(self, dirname):
00312         result = []
00313         for name in self.listdir(dirname):
00314             try:
00315                 st = self.lstat(os.path.join(dirname, name))
00316             except OSError, e: # pragma: no cover
00317                 st = e
00318             result.append((name, st))
00319         return result
00320