Back to index

moin  1.9.0~rc2
caching.py
Go to the documentation of this file.
00001 # -*- coding: iso-8859-1 -*-
00002 """
00003     MoinMoin caching module
00004 
00005     @copyright: 2001-2004 by Juergen Hermann <jh@web.de>,
00006                 2006-2009 MoinMoin:ThomasWaldmann,
00007                 2008 MoinMoin:ThomasPfaff
00008     @license: GNU GPL, see COPYING for details.
00009 """
00010 
00011 import os
00012 import shutil
00013 import tempfile
00014 
00015 from MoinMoin import log
00016 logging = log.getLogger(__name__)
00017 
00018 from MoinMoin import config
00019 from MoinMoin.util import filesys, lock, pickle, PICKLE_PROTOCOL
00020 
00021 
00022 class CacheError(Exception):
00023     """ raised if we have trouble locking, reading or writing """
00024     pass
00025 
00026 
00027 def get_arena_dir(request, arena, scope):
00028     if scope == 'item': # arena is a Page instance
00029         # we could move cache out of the page directory and store it to cache_dir
00030         return arena.getPagePath('cache', check_create=1)
00031     elif scope == 'wiki':
00032         return os.path.join(request.cfg.cache_dir, request.cfg.siteid, arena)
00033     elif scope == 'farm':
00034         return os.path.join(request.cfg.cache_dir, '__common__', arena)
00035     elif scope == 'dir':
00036         # arena is a specific directory, just use it
00037         return arena
00038     return None
00039 
00040 
00041 def get_cache_list(request, arena, scope):
00042     arena_dir = get_arena_dir(request, arena, scope)
00043     try:
00044         return filesys.dclistdir(arena_dir)
00045     except OSError:
00046         return []
00047 
00048 
00049 class CacheEntry:
00050     def __init__(self, request, arena, key, scope='wiki', do_locking=True,
00051                  use_pickle=False, use_encode=False):
00052         """ init a cache entry
00053             @param request: the request object
00054             @param arena: either a string or a page object, when we want to use
00055                           page local cache area
00056             @param key: under which key we access the cache content
00057             @param scope: the scope where we are caching:
00058                           'item' - an item local cache
00059                           'wiki' - a wiki local cache
00060                           'farm' - a cache for the whole farm
00061                           'dir' - just use some specific directory
00062             @param do_locking: if there should be a lock, normally True
00063             @param use_pickle: if data should be pickled/unpickled (nice for arbitrary cache content)
00064             @param use_encode: if data should be encoded/decoded (nice for readable cache files)
00065         """
00066         self.request = request
00067         self.key = key
00068         self.locking = do_locking
00069         self.use_pickle = use_pickle
00070         self.use_encode = use_encode
00071         self.arena_dir = get_arena_dir(request, arena, scope)
00072         if not os.path.exists(self.arena_dir):
00073             os.makedirs(self.arena_dir)
00074         self._fname = os.path.join(self.arena_dir, key)
00075 
00076         # used by file-like api:
00077         self._lock = None  # either a read or a write lock
00078         self._fileobj = None  # open cache file object
00079         self._tmp_fname = None  # name of temporary file (used for write)
00080         self._mode = None  # mode of open file object
00081 
00082 
00083     def _filename(self):
00084         # DEPRECATED - please use file-like api
00085         return self._fname
00086 
00087     def exists(self):
00088         return os.path.exists(self._fname)
00089 
00090     def mtime(self):
00091         # DEPRECATED for checking a changed on-disk cache, please use
00092         # self.uid() for this, see below
00093         try:
00094             return os.path.getmtime(self._fname)
00095         except (IOError, OSError):
00096             return 0
00097 
00098     def size(self):
00099         try:
00100             return os.path.getsize(self._fname)
00101         except (IOError, OSError):
00102             return 0
00103 
00104     def uid(self):
00105         """ Return a value that likely changes when the on-disk cache was updated.
00106 
00107             See docstring of MoinMoin.util.filesys.fuid for details.
00108         """
00109         return filesys.fuid(self._fname)
00110 
00111     def needsUpdate(self, filename, attachdir=None):
00112         # following code is not necessary. will trigger exception and give same result
00113         #if not self.exists():
00114         #    return 1
00115 
00116         try:
00117             ctime = os.path.getmtime(self._fname)
00118             ftime = os.path.getmtime(filename)
00119         except os.error:
00120             return 1
00121 
00122         needsupdate = ftime > ctime
00123 
00124         # if a page depends on the attachment dir, we check this, too:
00125         if not needsupdate and attachdir:
00126             try:
00127                 ftime2 = os.path.getmtime(attachdir)
00128             except os.error:
00129                 ftime2 = 0
00130             needsupdate = ftime2 > ctime
00131 
00132         return needsupdate
00133 
00134     def lock(self, mode, timeout=10.0):
00135         """
00136         acquire a lock for <mode> ("r" or "w").
00137         we just raise a CacheError if this doesn't work.
00138 
00139         Note:
00140          * .open() calls .lock(), .close() calls .unlock() if do_locking is True.
00141          * if you need to do a read-modify-write, you want to use a CacheEntry
00142            with do_locking=False and manually call .lock('w') and .unlock().
00143         """
00144         lock_dir = os.path.join(self.arena_dir, '__lock__')
00145         if 'r' in mode:
00146             _lock = lock.LazyReadLock(lock_dir, 60.0)
00147         elif 'w' in mode:
00148             _lock = lock.LazyWriteLock(lock_dir, 60.0)
00149         acquired = _lock.acquire(timeout)
00150         if acquired:
00151             self._lock = _lock
00152         else:
00153             self._lock = None
00154             err = "Can't acquire %s lock in %s" % (mode, lock_dir)
00155             logging.error(err)
00156             raise CacheError(err)
00157 
00158     def unlock(self):
00159         """
00160         release the lock.
00161         """
00162         if self._lock:
00163             self._lock.release()
00164             self._lock = None
00165 
00166     # file-like interface ----------------------------------------------------
00167 
00168     def open(self, filename=None, mode='r', bufsize=-1):
00169         """ open the cache for reading/writing
00170 
00171         Typical usage:
00172             try:
00173                 cache.open('r')  # open file, create locks
00174                 data = cache.read()
00175             finally:
00176                 cache.close()  # important to close file and remove locks
00177 
00178         @param filename: must be None (default - automatically determine filename)
00179         @param mode: 'r' (read, default), 'w' (write)
00180                      Note: if mode does not include 'b' (binary), it will be
00181                            automatically changed to include 'b'.
00182         @param bufsize: size of read/write buffer (default: -1 meaning automatic)
00183         @return: None (the opened file object is kept in self._fileobj and used
00184                  implicitely by read/write/close functions of CacheEntry object.
00185         """
00186         assert self._fileobj is None, 'caching: trying to open an already opened cache'
00187         assert filename is None, 'caching: giving a filename is not supported (yet?)'
00188         assert 'r' in mode or 'w' in mode, 'caching: mode must contain "r" or "w"'
00189 
00190         if 'b' not in mode:
00191             mode += 'b'  # we want to use binary mode, ever!
00192         self._mode = mode  # for self.close()
00193 
00194         if self.locking:
00195             self.lock(mode)
00196         try:
00197             if 'r' in mode:
00198                 filename = self._fname
00199                 self._fileobj = open(filename, mode, bufsize)
00200             elif 'w' in mode:
00201                 # we do not write content to old inode, but to a new file
00202                 # so we don't need to lock when we just want to read the file
00203                 # (at least on POSIX, this works)
00204                 filename = None
00205                 fd, filename = tempfile.mkstemp('.tmp', self.key, self.arena_dir)
00206                 self._tmp_fname = filename
00207                 self._fileobj = os.fdopen(fd, mode, bufsize)
00208         except IOError, err:
00209             if 'w' in mode:
00210                 # IOerror for 'r' can be just a non-existing file, do not log that,
00211                 # but if open fails for 'w', we likely have some bigger problem:
00212                 logging.error(str(err))
00213             raise CacheError(str(err))
00214 
00215     def read(self, size=-1):
00216         """ read data from cache file
00217 
00218         @param size: how many bytes to read (default: -1 == everything)
00219         @return: read data (str)
00220         """
00221         return self._fileobj.read(size)
00222 
00223     def write(self, data):
00224         """ write data to cache file
00225 
00226         @param data: write data (str)
00227         """
00228         self._fileobj.write(data)
00229 
00230     def close(self):
00231         """ close cache file (and release lock, if any) """
00232         try:
00233             if self._fileobj:
00234                 self._fileobj.close()
00235                 self._fileobj = None
00236                 if 'w' in self._mode:
00237                     filesys.chmod(self._tmp_fname, 0666 & config.umask) # fix mode that mkstemp chose
00238                     # this is either atomic or happening with real locks set:
00239                     filesys.rename(self._tmp_fname, self._fname)
00240         finally:
00241             if self.locking:
00242                 self.unlock()
00243 
00244     # ------------------------------------------------------------------------
00245 
00246     def update(self, content):
00247         try:
00248             if hasattr(content, 'read'):
00249                 # content is file-like
00250                 assert not (self.use_pickle or self.use_encode), 'caching: use_pickle and use_encode not supported with file-like api'
00251                 try:
00252                     self.open(mode='w')
00253                     shutil.copyfileobj(content, self)
00254                 finally:
00255                     self.close()
00256             else:
00257                 # content is a string
00258                 if self.use_pickle:
00259                     content = pickle.dumps(content, PICKLE_PROTOCOL)
00260                 elif self.use_encode:
00261                     content = content.encode(config.charset)
00262 
00263                 try:
00264                     self.open(mode='w')
00265                     self.write(content)
00266                 finally:
00267                     self.close()
00268         except (pickle.PicklingError, OSError, IOError, ValueError), err:
00269             raise CacheError(str(err))
00270 
00271     def content(self):
00272         # no file-like api yet, we implement it when we need it
00273         try:
00274             try:
00275                 self.open(mode='r')
00276                 data = self.read()
00277             finally:
00278                 self.close()
00279             if self.use_pickle:
00280                 data = pickle.loads(data)
00281             elif self.use_encode:
00282                 data = data.decode(config.charset)
00283             return data
00284         except (pickle.UnpicklingError, IOError, EOFError, ValueError), err:
00285             raise CacheError(str(err))
00286 
00287     def remove(self):
00288         if self.locking:
00289             self.lock('w')
00290         try:
00291             try:
00292                 os.remove(self._fname)
00293             except OSError:
00294                 pass
00295         finally:
00296             if self.locking:
00297                 self.unlock()
00298 
00299