Back to index

plone3  3.1.7
ZVCStorageTool.py
Go to the documentation of this file.
00001 #########################################################################
00002 # Copyright (c) 2004, 2005, 2006 Alberto Berti, Gregoire Weber. 
00003 # All Rights Reserved.
00004 # 
00005 # This file is part of CMFEditions.
00006 # 
00007 # CMFEditions is free software; you can redistribute it and/or modify
00008 # it under the terms of the GNU General Public License as published by
00009 # the Free Software Foundation; either version 2 of the License, or
00010 # (at your option) any later version.
00011 # 
00012 # CMFEditions is distributed in the hope that it will be useful,
00013 # but WITHOUT ANY WARRANTY; without even the implied warranty of
00014 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00015 # GNU General Public License for more details.
00016 # 
00017 # You should have received a copy of the GNU General Public License
00018 # along with CMFEditions; if not, write to the Free Software
00019 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
00020 #########################################################################
00021 """Histories Storage using ZVC
00022 
00023 $Id: ZVCStorageTool.py,v 1.18 2005/03/11 11:05:12 varun-rastogi Exp $
00024 """
00025 __version__ = "$Revision: 1.18 $"
00026 
00027 import logging
00028 import time
00029 import types
00030 from StringIO import StringIO
00031 from cPickle import Pickler, Unpickler, dumps, loads, HIGHEST_PROTOCOL
00032 from zope.interface import implements
00033 
00034 from Globals import InitializeClass
00035 from BTrees.OOBTree import OOBTree
00036 from BTrees.IOBTree import IOBTree
00037 from Persistence import Persistent
00038 from AccessControl import ClassSecurityInfo
00039 
00040 from OFS.SimpleItem import SimpleItem
00041 from Products.PageTemplates.PageTemplateFile import PageTemplateFile
00042 
00043 from Products.CMFCore.utils import UniqueObject, getToolByName
00044 from Products.CMFCore.permissions import ManagePortal
00045 
00046 from Products.ZopeVersionControl.ZopeRepository import ZopeRepository
00047 from Products.ZopeVersionControl.Utility import VersionControlError
00048 from Products.ZopeVersionControl.EventLog import LogEntry
00049 
00050 from Products.CMFEditions.interfaces import IStorageTool
00051 from Products.CMFEditions.interfaces.IStorage import IStorage
00052 from Products.CMFEditions.interfaces.IStorage import IPurgeSupport
00053 from Products.CMFEditions.interfaces.IStorage import IHistory
00054 from Products.CMFEditions.interfaces.IStorage import IVersionData
00055 from Products.CMFEditions.interfaces.IStorage import IStreamableReference
00056 
00057 from Products.CMFEditions.interfaces.IStorage import StorageError
00058 from Products.CMFEditions.interfaces.IStorage import StorageRegisterError
00059 from Products.CMFEditions.interfaces.IStorage import StorageSaveError
00060 from Products.CMFEditions.interfaces.IStorage import StorageRetrieveError
00061 from Products.CMFEditions.interfaces.IStorage import StorageUnregisteredError
00062 from Products.CMFEditions.interfaces.IStorage import StoragePurgeError
00063 
00064 logger = logging.getLogger('CMFEditions')
00065 
00066 def deepCopy(obj):
00067     stream = StringIO()
00068     p = Pickler(stream, 1)
00069     p.dump(obj)
00070     stream.seek(0)
00071     u = Unpickler(stream)
00072     return u.load()
00073 
00074 def getSize(obj):
00075     """Calculate the size as cheap as possible
00076     """
00077     # Try the cheap variants first.
00078     # Actually the checks ensure the code never fails but beeing sure
00079     # is better.
00080     try:
00081         # check if to return zero (length is zero)
00082         if len(obj) == 0:
00083             return 0
00084     except:
00085         pass
00086         
00087     try:
00088         # check if ``IStreamableReference``
00089         if IStreamableReference.isImplementedBy(obj):
00090             size = obj.getSize()
00091             if size is not None:
00092                 return size
00093     except:
00094         pass
00095         
00096     try:
00097         # string
00098         if isinstance(obj, types.StringTypes):
00099             return len(obj)
00100     except:
00101         pass
00102         
00103     try:
00104         # file like object
00105         methods = dir(obj)
00106         if "seek" in methods and "tell" in methods:
00107             currentPos = obj.tell()
00108             obj.seek(0, 2)
00109             size = obj.tell()
00110             obj.seek(currentPos)
00111             return size
00112     except:
00113         pass
00114     
00115     try:
00116         # fallback: pickling the object
00117         stream = StringIO()
00118         p = Pickler(stream, 1)
00119         p.dump(obj)
00120         size = stream.tell()
00121     except:
00122         size = None
00123     
00124     return size
00125 
00126 
00127 class ZVCStorageTool(UniqueObject, SimpleItem):
00128     """Zope Version Control Based Version Storage
00129     
00130     There exist two selector schemas:
00131     
00132     - the one that counts removed versions also
00133     - the one that counts non removed version only
00134     
00135     Intended Usage:
00136     
00137     For different use cases different numbering schemas are used:
00138     
00139     - When iterating over the history the removed versions (usually) 
00140       aren't of interest. Thus the next valid version may be accessed
00141       by incrementing the selector and vice versa.
00142     - When retrieving a version beeing able to access removed version
00143       or correctly spoken a substitute (pretending to be the removed 
00144       version) is important when reconstructing relations between 
00145       objects.
00146     """
00147 
00148     __implements__ = (
00149         IPurgeSupport,
00150         IStorage,
00151         SimpleItem.__implements__,
00152     )
00153     implements(IStorageTool)
00154 
00155     id = 'portal_historiesstorage'
00156     alternative_id = 'portal_zvcstorage'
00157     
00158     meta_type = 'CMFEditions Portal ZVC based Histories Storage Tool'
00159     
00160     storageStatistics = PageTemplateFile('www/storageStatistics.pt',
00161                                          globals(),
00162                                          __name__='modifierEditForm')
00163     manage_options = ({'label' : 'Statistics (may take time)', 'action' : 'storageStatistics'}, ) \
00164                      + SimpleItem.manage_options[:]
00165 
00166     # make exceptions available trough the tool
00167     StorageError = StorageError
00168     StorageRetrieveError = StorageRetrieveError
00169     
00170     # shadow storage contains ZVCs __vc_info__ for every non purged 
00171     # version
00172     _shadowStorage = None
00173     
00174     # the ZVC repository ("the" version storage)
00175     zvc_repo = None
00176     
00177     security = ClassSecurityInfo()
00178     
00179     # -------------------------------------------------------------------
00180     # methods implementing IStorage
00181     # -------------------------------------------------------------------
00182 
00183     security.declarePrivate('isRegistered')
00184     def isRegistered(self, history_id):
00185         """See IStorage.
00186         """
00187         # Do not wake up the ZODB (aka write to it) if there wasn't any
00188         # version saved yet.
00189         shadow_storage = self._getShadowStorage(autoAdd=False)
00190         if shadow_storage is None:
00191             return False
00192         return shadow_storage.isRegistered(history_id)
00193         
00194     security.declarePrivate('register')
00195     def register(self, history_id, object, referenced_data={}, metadata=None):
00196         """See IStorage.
00197         """
00198         # check if already registered
00199         if self.isRegistered(history_id):
00200             return
00201         
00202         # No ZVC info available at register time
00203         shadowInfo = {"vc_info": None}
00204         zvc_method = self._getZVCRepo().applyVersionControl
00205         try:
00206             return self._applyOrCheckin(zvc_method, history_id, shadowInfo,
00207                                         object, referenced_data, metadata)
00208         except VersionControlError:
00209             raise StorageRegisterError(
00210                 "Registering the object with history id '%s' failed. "
00211                 "The underlying storage implementation reported an error."
00212                 % history_id)
00213         
00214     security.declarePrivate('save')
00215     def save(self, history_id, object, referenced_data={}, metadata=None):
00216         """See IStorage.
00217         """
00218         # check if already registered
00219         if not self.isRegistered(history_id):
00220             raise StorageUnregisteredError(
00221                 "Saving an unregistered object is not possible. "
00222                 "Register the object with history id '%s' first. "
00223                 % history_id)
00224         
00225         # retrieve the ZVC info from the youngest version
00226         history = self._getShadowHistory(history_id, autoAdd=True)
00227         shadowInfo = history.retrieve(selector=None, countPurged=True)
00228         
00229         zvc_method = self._getZVCRepo().checkinResource
00230         try:
00231             return self._applyOrCheckin(zvc_method, history_id, shadowInfo,
00232                                         object, referenced_data, metadata)
00233         except VersionControlError:
00234             # this shouldn't really happen
00235             raise StorageSaveError(
00236                 "Saving the object with history id '%s' failed. "
00237                 "The underlying storage implementation reported an error."
00238                 % history_id)
00239 
00240     security.declarePrivate('retrieve')
00241     def retrieve(self, history_id, selector=None, 
00242                  countPurged=True, substitute=True):
00243         """See ``IStorage`` and Comments in ``IPurgePolicy``
00244         """
00245         zvc_repo = self._getZVCRepo()
00246         zvc_histid, zvc_selector = \
00247             self._getZVCAccessInfo(history_id, selector, countPurged)
00248         
00249         if zvc_histid is None:
00250             raise StorageRetrieveError(
00251                 "Retrieving version '%s' of object with history id '%s' "
00252                 "failed. A history with the given history id does not exist."
00253                 % (selector, history_id))
00254         
00255         if zvc_selector is None:
00256             raise StorageRetrieveError(
00257                 "Retrieving version '%s' of object with history id '%s' "
00258                 "failed. The version does not exist."
00259                 % (selector, history_id))
00260         
00261         # retrieve the object
00262         try:
00263             zvc_obj = zvc_repo.getVersionOfResource(zvc_histid, zvc_selector)
00264         except VersionControlError:
00265             # this should never happen
00266             raise StorageRetrieveError(
00267                 "Retrieving version '%s' of object with history id '%s' "
00268                 "failed. The underlying storage implementation reported "
00269                 "an error." % (selector, history_id))
00270         
00271         # retrieve metadata
00272         # TODO: read this from the shadow storage directly
00273         metadata = self._retrieveMetadataFromZVC(zvc_histid, zvc_selector)
00274         
00275         # wrap object and referenced data
00276         object = zvc_obj.getWrappedObject()
00277         referenced_data = zvc_obj.getReferencedData()
00278         data = VersionData(object, referenced_data, metadata)
00279         
00280         # check if retrieved a replacement for a removed object and 
00281         # if so check if a substitute is available
00282         if substitute and isinstance(data.object, Removed):
00283             # delegate retrieving to purge policy if one is available
00284             # if none is available just return the replacement for the
00285             # removed object
00286             policy = getToolByName(self, 'portal_purgepolicy', None)
00287             if policy is not None:
00288                 data = policy.retrieveSubstitute(history_id, selector, 
00289                                                  default=data)
00290         return data
00291 
00292     security.declarePrivate('getHistory')
00293     def getHistory(self, history_id, countPurged=True, substitute=True):
00294         """See IStorage.
00295         """
00296         return LazyHistory(self, history_id, countPurged, substitute)
00297 
00298     security.declarePrivate('getModificationDate')
00299     def getModificationDate(self, history_id, selector=None, 
00300                             countPurged=True, substitute=True):
00301         """See IStorage.
00302         """
00303         vdata = self.retrieve(history_id, selector, countPurged, substitute)
00304         return vdata.object.object.modified()
00305 
00306 
00307     # -------------------------------------------------------------------
00308     # methods implementing IPurgeSupport
00309     # -------------------------------------------------------------------
00310 
00311     security.declarePrivate('purge')
00312     def purge(self, history_id, selector, metadata={}, countPurged=True):
00313         """See ``IPurgeSupport``
00314         """
00315         zvc_repo = self._getZVCRepo()
00316         zvc_histid, zvc_selector = \
00317             self._getZVCAccessInfo(history_id, selector, countPurged)
00318         if zvc_histid is None:
00319             raise StoragePurgeError(
00320                 "Purging version '%s' of object with history id '%s' "
00321                 "failed. A history with the given history id does not exist."
00322                 % (selector, history_id))
00323         
00324         if zvc_selector is None:
00325             raise StoragePurgeError(
00326                 "Purging version '%s' of object with history id '%s' "
00327                 "failed. The version does not exist." 
00328                 % (selector, history_id))
00329         
00330         # digging into ZVC internals:
00331         # Get a reference to the version stored in the ZVC history storage
00332         #
00333         # Implementation Note:
00334         #
00335         # ZVCs ``getVersionOfResource`` is quite more complex. But as we 
00336         # do not use labeling and branches it is not a problem to get the
00337         # version in the following simple way.
00338         zvc_history = zvc_repo.getVersionHistory(zvc_histid)
00339         version = zvc_history.getVersionById(zvc_selector)
00340         data = version._data
00341         
00342         if not isinstance(data.getWrappedObject(), Removed):
00343             # purge version in shadow storages history
00344             history = self._getShadowHistory(history_id)
00345             
00346             # update administrative data
00347             history.purge(selector, metadata, countPurged)
00348             
00349             # prepare replacement for the deleted object and metadata
00350             removedInfo = Removed("purged", metadata)
00351             
00352             # digging into ZVC internals: remove the stored object
00353             version._data = ZVCAwareWrapper(removedInfo, None, metadata)
00354             
00355             # digging into ZVC internals: replace the message
00356             logEntry = self._retrieveZVCLogEntry(zvc_histid, zvc_selector)
00357             logEntry.message = self._encodeMetadata(metadata)
00358 
00359 
00360     # -------------------------------------------------------------------
00361     # private helper methods
00362     # -------------------------------------------------------------------
00363 
00364     def _applyOrCheckin(self, zvc_method, history_id, shadowInfo, 
00365                         object, referenced_data, metadata):
00366         """Just centralizing similar code.
00367         """
00368         # delegate the decision if and what to purge to the purge policy 
00369         # tool if one exists. If the call returns ``False`` do not save 
00370         # or register the current version.
00371         policy = getToolByName(self, 'portal_purgepolicy', None)
00372         if policy is not None:
00373             if not policy.beforeSaveHook(history_id, object, metadata):
00374                 # returning None signalizes that the version wasn't saved
00375                 return None
00376         
00377         # calculate the approximate size taking into account the object 
00378         # and the referenced_data (overwriting the archivists size as the
00379         # storage knows it better)
00380         approxSize = getSize(object) + getSize(referenced_data)
00381         metadata["sys_metadata"]["approxSize"] = approxSize
00382         
00383         # prepare the object for beeing saved with ZVC
00384         #
00385         # - Recall the ``__vc_info__`` from the most current version
00386         #   (selector=None).
00387         # - Wrap the object, the referenced data and metadata
00388         vc_info = self._getVcInfo(object, shadowInfo)
00389         zvc_obj = ZVCAwareWrapper(object, referenced_data, metadata, 
00390                                   vc_info)
00391         message = self._encodeMetadata(metadata)
00392         
00393         # call appropriate ZVC method
00394         zvc_method(zvc_obj, message)
00395         
00396         # save the ``__vc_info__`` attached by the zvc call from above
00397         # and cache the metadata in the shadow storage
00398         shadowInfo = {
00399             "vc_info": zvc_obj.__vc_info__,
00400             "metadata": metadata,
00401         }
00402         history = self._getShadowHistory(history_id, autoAdd=True)
00403         return history.save(shadowInfo)
00404 
00405     def _getShadowStorage(self, autoAdd=True):
00406         """Returns the Shadow Storage
00407         
00408         Returns None if there wasn't ever saved any version yet.
00409         """
00410         if self._shadowStorage is None:
00411             if not autoAdd:
00412                 return None
00413             self._shadowStorage = ShadowStorage()
00414         return self._shadowStorage
00415 
00416     def _getShadowHistory(self, history_id, autoAdd=False):
00417         """Returns a History from the Shadow Storage
00418         """
00419         return self._getShadowStorage().getHistory(history_id, autoAdd)
00420 
00421     def _getZVCRepo(self):
00422         """Returns the Zope Version Control Repository
00423         
00424         Instantiates one with the first call.
00425         """
00426         if self.zvc_repo is None:
00427             self.zvc_repo = ZopeRepository('repo', 'ZVC Storage')
00428         return self.zvc_repo
00429 
00430     def _getZVCAccessInfo(self, history_id, selector, countPurged):
00431         """Returns the ZVC history id and selector
00432         
00433         Returns a tuple with the ZVC history id and selector.
00434         Returns None as history id if such history doesn't exist.
00435         Returns None as selector if the version does not exist.
00436         """
00437         history = self._getShadowHistory(history_id)
00438         if history is None:
00439             # no history
00440             return None, None
00441         
00442         shadowInfo = history.retrieve(selector, countPurged)
00443         if shadowInfo is None:
00444             # no version
00445             return False, None
00446         
00447         # history and version exists
00448         zvc_hid = shadowInfo["vc_info"].history_id
00449         zvc_vid = str(history.getVersionId(selector, countPurged) + 1)
00450         return zvc_hid, zvc_vid
00451 
00452     def _getVcInfo(self, obj, shadowInfo, set_checked_in=False):
00453         """Recalls ZVC Related Informations and Attaches them to the Object
00454         """
00455         vc_info = deepCopy(shadowInfo["vc_info"])
00456         if vc_info is None:
00457             return None
00458         
00459         # fake sticky information (no branches)
00460         vc_info.sticky = None
00461         
00462         # On revert operations the repository expects the object 
00463         # to be in CHECKED_IN state.
00464         if set_checked_in:
00465             vc_info.status = vc_info.CHECKED_IN
00466         else:
00467             vc_info.status = vc_info.CHECKED_OUT
00468         
00469         # fake the version to be able to save a retrieved version later
00470         zvc_repo = self._getZVCRepo()
00471         obj.__vc_info__ = vc_info
00472         vc_info.version_id = str(len(zvc_repo.getVersionIds(obj)))
00473         return vc_info
00474 
00475     def _retrieveZVCLogEntry(self, zvc_histid, zvc_selector):
00476         """Retrieves the metadata from ZVCs log
00477         
00478         Unfortunately this may get costy with long histories.
00479         We should really store metadata in the shadow storage in the
00480         future or loop over the log in reverse.
00481         
00482         XXX also store (only store) the metadata in the shadow before 1.0beta1
00483         """
00484         zvc_repo = self._getZVCRepo()
00485         log = zvc_repo.getVersionHistory(zvc_histid).getLogEntries()
00486         checkin = LogEntry.ACTION_CHECKIN
00487         entries = [e for e in log if e.version_id==zvc_selector and e.action==checkin]
00488         
00489         # just make a log entry if something wrong happened
00490         if len(entries) != 1:
00491             logger.log(logging.INFO, "CMFEditions ASSERT:"
00492                      "Uups, an object has been stored %s times with the same "
00493                      "history '%s'!!!" % (len(entries), zvc_selector))
00494         
00495         return entries[0]
00496 
00497     def _encodeMetadata(self, metadata):
00498         # metadata format is:
00499         #    - first line with trailing \x00: comment or empty comment
00500         #    - then: pickled metadata (incl. comment)
00501         try:
00502             comment = metadata['sys_metadata']['comment']
00503             comment = dumps(comment)
00504         except KeyError:
00505             comment = ''
00506         return '\x00\n'.join((comment, dumps(metadata, HIGHEST_PROTOCOL)))
00507 
00508     def _retrieveMetadataFromZVC(self, zvc_histid, zvc_selector):
00509         logEntry = self._retrieveZVCLogEntry(zvc_histid, zvc_selector)
00510         metadata = loads(logEntry.message.split('\x00\n', 1)[1])
00511         return metadata
00512 
00513 
00514     # -------------------------------------------------------------------
00515     # Migration Support
00516     #
00517     # - Migration from 1.0alpha3 --> 1.0beta1
00518     # -------------------------------------------------------------------
00519 
00520     def _is10alpha3Layout(self):
00521         """Returns True if Storage is of 1.0alpha3 layout
00522         """
00523         return getattr(self, "_history_id_mapping", None) is not None
00524     
00525     def migrateStorage(self):
00526         """Migrate the Storage to Newest Layout
00527         """
00528         # check if already done
00529         if not self._is10alpha3Layout():
00530             logger.log(logging.INFO, "CMFEditions storage migration:"
00531                 "Storage already migrated.")
00532             return None
00533         
00534         startTime = time.time()
00535         logger.log(logging.INFO, "CMFEditions storage migration:"
00536             "started migrating the whole storage")
00537         from Products.ZopeVersionControl.Utility import VersionInfo
00538         
00539         # build reverse mapping: zvc history id --> CMFEditions history id
00540         logger.log(logging.INFO, "CMFEditions storage migration:"
00541             "preparing history mapping CMFEditions <--> ZVC")
00542         hidMapping = self._history_id_mapping
00543         hidReverseMapping = {}
00544         for hid, zvcHid in hidMapping.items():
00545             hidReverseMapping[zvcHid.history_id] = hid
00546             logger.log(logging.INFO, "CMFEditions storage migration:"
00547                 " %6i <--> %s" % (hid, zvcHid.history_id))
00548 
00549         # iterate over all histories
00550         logger.log(logging.INFO, "CMFEditions storage migration:"
00551             "iterating over all histories:")
00552         nbrOfMigratedHistories = 0
00553         nbrOfMigratedVersions = 0
00554         repo = self._getZVCRepo()
00555         for zvcHid in repo._histories.keys():
00556             zvcHistory = repo.getVersionHistory(zvcHid)
00557             zvcVersionIds = zvcHistory.getVersionIds()
00558             history_id = hidReverseMapping[zvcHid]
00559             history = self._getShadowHistory(history_id, autoAdd=True)
00560             logger.log(logging.INFO, "CMFEditions storage migration:"
00561                 " migrating %s versions of history %s (ZVC: %s)" 
00562                 % (len(zvcVersionIds), history_id, zvcHid))
00563             nbrOfMigratedHistories += 1
00564             
00565             # iterate over all versions
00566             for zvcVid in zvcVersionIds:
00567                 obj = zvcHistory.getVersionById(zvcVid)
00568                 vc_info = VersionInfo(zvcHid, zvcVid, VersionInfo.CHECKED_IN)
00569                 vc_info.timestamp = obj.date_created
00570                 metadata = self._retrieveMetadataFromZVC(zvcHid, zvcVid)
00571                 
00572                 # calculating approximate size
00573                 zvc_obj = repo.getVersionOfResource(zvcHid, zvcVid)
00574                 obj = zvc_obj.getWrappedObject()
00575                 referenced_data = zvc_obj.getReferencedData()
00576                 approxSize = getSize(obj) + getSize(referenced_data)
00577                 metadata["sys_metadata"]["approxSize"] = approxSize
00578                 
00579                 # we do not calculate version aware parent references
00580                 # (it's possible but rather complicated)
00581                 
00582                 # preparing administrative data
00583                 shadowInfo = {
00584                     "vc_info": vc_info,
00585                     "metadata": metadata,
00586                 }
00587                 
00588                 # save metadata in shadow history
00589                 logger.log(logging.INFO, "CMFEditions storage migration:"
00590                     " migrating version %s:" % (int(zvcVid)-1))
00591                 history.save(shadowInfo)
00592                 
00593                 app_metadata = metadata.get("app_metadata", {})
00594                 if app_metadata:
00595                     logger.log(logging.INFO, "CMFEditions storage migration:"
00596                         " application metadata:")
00597                     for item in app_metadata.items():
00598                         logger.log(logging.INFO,
00599                             "CMFEditions storage migration: %s = %s" % item)
00600                 sys_metadata = metadata.get("sys_metadata", {})
00601                 if sys_metadata:
00602                     logger.log(logging.INFO, "CMFEditions storage migration:"
00603                         " system metadata:")
00604                     for item in sys_metadata.items():
00605                         logger.log(logging.INFO,
00606                             "CMFEditions storage migration: %s = %s" % item)
00607                 nbrOfMigratedVersions += 1
00608         
00609         # delete the old metadata
00610         del self._history_id_mapping
00611         
00612         # log a summary
00613         totalTime = round(time.time() - startTime, 2)
00614         logger.log(logging.INFO, "CMFEditions storage migration:"
00615             "migrated %s histories and a total of %s versions in %.2f seconds" 
00616             % (nbrOfMigratedHistories, nbrOfMigratedVersions, totalTime))
00617         
00618         # XXX have to add purge policy
00619         
00620         return (nbrOfMigratedHistories, nbrOfMigratedVersions, totalTime)
00621 
00622     # -------------------------------------------------------------------
00623     # ZMI methods
00624     # -------------------------------------------------------------------
00625 
00626     security.declareProtected(ManagePortal, 'zmi_getStorageStatistics')
00627     def zmi_getStorageStatistics(self):
00628         """
00629         """
00630         startTime = time.time()
00631         # get all history ids (incl. such that were deleted in the portal)
00632         storage = self._getShadowStorage(autoAdd=False)
00633         if storage is not None:
00634             historyIds = storage._storage
00635         else:
00636             historyIds = {}
00637         hidhandler = getToolByName(self, "portal_historyidhandler")
00638         portal_paths_len = len(getToolByName(self, "portal_url")())
00639         
00640         # collect interesting informations
00641         histories = []
00642         for hid in historyIds.keys():
00643             history = self.getHistory(hid)
00644             length = len(history)
00645             shadowStorage = self._getShadowHistory(hid)
00646             size = 0
00647             sizeState = "n/a"
00648             if shadowStorage is not None:
00649                 size, sizeState = shadowStorage.getSize()
00650             
00651             workingCopy = hidhandler.queryObject(hid)
00652             if workingCopy is not None:
00653                 url = workingCopy.absolute_url()
00654                 path = url[portal_paths_len:]
00655                 portal_type = workingCopy.getPortalTypeName()
00656             else:
00657                 path = None
00658                 url = None
00659                 retrieved = self.retrieve(hid).object.object
00660                 portal_type = retrieved.getPortalTypeName()
00661             histData = {
00662                 "history_id": hid, 
00663                 "length": length, 
00664                 "url": url, 
00665                 "path": path, 
00666                 "portal_type": portal_type, 
00667                 "size": size,
00668                 "sizeState": sizeState,
00669             }
00670             histories.append(histData)
00671         
00672         # collect history ids with still existing working copies
00673         existing = []
00674         existingHistories = 0
00675         existingVersions = 0
00676         existingSize = 0
00677         deleted = []
00678         deletedHistories = 0
00679         deletedVersions = 0
00680         deletedSize = 0
00681         for histData in histories:
00682             if histData["path"] is None:
00683                 deleted.append(histData)
00684                 deletedHistories += 1
00685                 deletedVersions += histData["length"]
00686                 deletedSize += 0 # TODO
00687             else:
00688                 existing.append(histData)
00689                 existingHistories += 1
00690                 existingVersions += histData["length"]
00691                 existingSize += 0 # TODO
00692         
00693         processingTime = "%.2f" % round(time.time() - startTime, 2)
00694         histories = existingHistories+deletedHistories
00695         versions = existingVersions+deletedVersions
00696         
00697         if histories:
00698             totalAverage = "%.1f" % round(float(versions)/histories, 1)
00699         else:
00700             totalAverage = "n/a"
00701         
00702         if existingHistories:
00703             existingAverage = "%.1f" % \
00704                 round(float(existingVersions)/existingHistories, 1)
00705         else:
00706             existingAverage = "n/a"
00707         
00708         if deletedHistories:
00709             deletedAverage = "%.1f" % \
00710                 round(float(deletedVersions)/deletedHistories, 1)
00711         else:
00712             deletedAverage = "n/a"
00713         
00714         return {
00715             "existing": existing, 
00716             "deleted": deleted, 
00717             "summaries": {
00718                 "time": processingTime,
00719                 "totalHistories": histories,
00720                 "totalVersions": versions,
00721                 "totalAverage": totalAverage,
00722                 "existingHistories": existingHistories,
00723                 "existingVersions": existingVersions,
00724                 "existingAverage": existingAverage,
00725                 "deletedHistories": deletedHistories,
00726                 "deletedVersions": deletedVersions,
00727                 "deletedAverage": deletedAverage,
00728             }
00729         }
00730 
00731 InitializeClass(ZVCStorageTool)
00732 
00733 
00734 class ShadowStorage(Persistent):
00735     """Container for Shadow Histories
00736     
00737     Only cares about containerish operations.
00738     """
00739     def __init__(self):
00740         # Using a OOBtree to allow history ids of any type. The type 
00741         # of the history ids higly depends on the unique id tool which
00742         # we isn't under our control.
00743         self._storage = OOBTree()
00744 
00745     def isRegistered(self, history_id):
00746         """Returns True if a History With the Given History id Exists
00747         """
00748         return history_id in self._storage
00749 
00750     def getHistory(self, history_id, autoAdd=False):
00751         """Returns the History Object of the Given ``history_id``.
00752         
00753         Returns None if ``autoAdd`` is False and the history 
00754         does not exist. Else prepares and returns an empty history.
00755         """
00756         # Create a new history if there isn't one yet
00757         if autoAdd and not self.isRegistered(history_id):
00758             self._storage[history_id] = ShadowHistory()
00759         return self._storage.get(history_id, None)
00760 
00761 InitializeClass(ShadowStorage)
00762 
00763 
00764 class ShadowHistory(Persistent):
00765     """Purge Aware History for Storage Related Metadata
00766     """
00767     def __init__(self):
00768         # Using a IOBtree as we know the selectors are integers.
00769         # The full history contains shadow data for every saved version. 
00770         # A counter is needed as IOBTree doesn't have a list like append.
00771         self._full = IOBTree()
00772         self.nextVersionId = 0
00773         
00774         # Indexes to the full histories versions
00775         self._available = []
00776         
00777         # aproximative size of the history
00778         self._approxSize = 0
00779         self._sizeInaccurate = False
00780 
00781     def save(self, data):
00782         """Saves data in the history
00783         
00784         Returns the version id of the saved version.
00785         """
00786         version_id = self.nextVersionId
00787         self._full[version_id] = deepCopy(data)
00788         self._available.append(version_id)
00789         # Provokes a write conflict if two saves happen the same
00790         # time. That's exactly what's desired.
00791         self.nextVersionId += 1
00792         
00793         # update the histories size:
00794         size = data["metadata"]["sys_metadata"].get("approxSize", None)
00795         if size is None:
00796             self._sizeInaccurate = True
00797         else:
00798             self._approxSize += size
00799         
00800         return version_id
00801 
00802     def retrieve(self, selector, countPurged):
00803         """Retrieves the Selected Data From the History
00804         
00805         The caller has to make a copy if he passed the data to another 
00806         caller.
00807         
00808         Returns None if the selected version does not exist.
00809         """
00810         version_id = self.getVersionId(selector, countPurged)
00811         if version_id is None:
00812             return None
00813         return self._full[version_id]
00814 
00815     def purge(self, selector, data, countPurged):
00816         """Purge selected version from the history
00817         """
00818         # find the position to purge
00819         version_pos = self._getVersionPos(selector, countPurged)
00820         version_id = self._available[version_pos]
00821         
00822         # update the histories size
00823         sys_metadata = self._full[version_id]["metadata"]["sys_metadata"]
00824         size = sys_metadata.get("approxSize", None)
00825         if size is None:
00826             self._sizeInaccurate = True
00827         else:
00828             self._approxSize -= size
00829             if self._approxSize < 0:
00830                 self._approxSize = 0
00831                 self._sizeInaccurate = True
00832         
00833         # update the metadata
00834         self._full[version_id]["metadata"] = deepCopy(data)
00835         # purge the reference
00836         del self._available[version_pos]
00837 
00838     def getLength(self, countPurged):
00839         """Length of the History Either Counting Purged Versions or Not
00840         """
00841         if countPurged:
00842             return self.nextVersionId
00843         else:
00844             return len(self._available)
00845 
00846     def getSize(self):
00847         """Returns the size including the quality of the size
00848         """
00849         # don't like exceptions taking down CMFEditions
00850         if getattr(self, "_sizeInaccurate", None) is None:
00851             return 0, "not available"
00852         if self._sizeInaccurate:
00853             return self._approxSize, "inaccurate"
00854         else:
00855             return self._approxSize, "approximate"
00856 
00857     def getVersionId(self, selector, countPurged):
00858         """Returns the Effective Version id depending the selector type
00859         
00860         Returns ``None`` if the selected version does not exist.
00861         """
00862         if selector is not None:
00863             selector = int(selector)
00864         
00865         ##### looking at special selectors first (None, negative)
00866         length = self.getLength(countPurged)
00867         # checking for ``None`` selector (youngest version)
00868         if selector is None:
00869             return length - 1
00870         # checking if positive selector tries to look into future
00871         if selector >= length:
00872             return None
00873         # check if negative selector and if it looks to far into past
00874         if selector < 0:
00875             selector = length - selector
00876             if selector < 0:
00877                 return None
00878         
00879         #### normal cases (0 <= selectors < length)
00880         if countPurged:
00881             # selector is a normal selector
00882             return selector
00883         else:
00884             # selector is a positional selector
00885             return self._available[selector]
00886 
00887     def _getVersionPos(self, selector, countPurged):
00888         """Returns the Position in the Version History 
00889         
00890         The position returned does not count purged versions.
00891         """
00892         if not countPurged:
00893             if selector is None:
00894                 # version counting starts with 0
00895                 selector = self.getLength(countPurged=False) - 1
00896             return int(selector)
00897         
00898         # Lets search from the end of the available list as it is more 
00899         # likely that a younger versions position has to be returned.
00900         # Let's work on a copy to not trigger an unecessary ZODB store
00901         # operations.
00902         history = self._available[:]
00903         history.reverse()
00904         try:
00905             selector = len(history) - 1 - history.index(selector)
00906         except ValueError:
00907             selector = None
00908         return selector
00909 
00910 InitializeClass(ShadowHistory)
00911 
00912 
00913 class ZVCAwareWrapper(Persistent):
00914     """ZVC assumes the stored object has a getPhysicalPath method.
00915     
00916     ZVC, arghh ...
00917     """
00918     def __init__(self, object, referenced_data, metadata, vc_info=None):
00919         self._object = object
00920         self._referenced_data = referenced_data
00921         self._physicalPath = \
00922             metadata['sys_metadata'].get('physicalPath', ())[:] # copy
00923         if vc_info is not None:
00924             self.__vc_info__ = vc_info
00925         
00926     def getWrappedObject(self):
00927         return self._object
00928         
00929     def getReferencedData(self):
00930         return self._referenced_data
00931         
00932     def getPhysicalPath(self):
00933         return self._physicalPath
00934 
00935 InitializeClass(ZVCAwareWrapper)
00936 
00937 
00938 class Removed(Persistent):
00939     """Indicates that removement of data
00940     """
00941     
00942     def __init__(self, reason, metadata):
00943         """Store Removed Info
00944         """
00945         self.reason = reason
00946         self.metadata = metadata
00947 
00948 
00949 class VersionData:
00950     __implements__ = (IVersionData, )
00951     
00952     def __init__(self, object, referenced_data, metadata):
00953         self.object = object
00954         self.referenced_data = referenced_data
00955         self.metadata = metadata
00956 
00957     def isValid(self):
00958         """Returns True if Valid (not Purged)
00959         """
00960         return not isinstance(self.object, Removed)
00961 
00962 class LazyHistory:
00963     """Lazy history adapter.
00964     """
00965     
00966     __implements__ = (
00967         IHistory,
00968     )
00969 
00970     def __init__(self, storage, history_id, countPurged=True, substitute=True):
00971         """See IHistory.
00972         """
00973         history = storage._getShadowHistory(history_id)
00974         if history is None:
00975             self._length = 0
00976         else:
00977             self._length = history.getLength(countPurged)
00978         self._history_id = history_id
00979         self._countPurged = countPurged
00980         self._substitute = substitute
00981         self._retrieve = storage.retrieve
00982 
00983     def __len__(self):
00984         """See IHistory.
00985         """
00986         return self._length
00987 
00988     def __getitem__(self, selector):
00989         """See IHistory.
00990         """
00991         return self._retrieve(self._history_id, selector, self._countPurged, 
00992                               self._substitute)
00993 
00994     def __iter__(self):
00995         """See IHistory.
00996         """
00997         return GetItemIterator(self.__getitem__,
00998                                stopExceptions=(StorageRetrieveError,))
00999 
01000 
01001 class GetItemIterator:
01002     """Iterator object using a getitem implementation to iterate over.
01003     """
01004     def __init__(self, getItem, stopExceptions):
01005         self._getItem = getItem
01006         self._stopExceptions = stopExceptions
01007         self._pos = -1
01008 
01009     def __iter__(self):
01010         return self
01011         
01012     def next(self):
01013         self._pos += 1
01014         try:
01015             return self._getItem(self._pos)
01016         except self._stopExceptions:
01017             raise StopIteration()