Back to index

plone3  3.1.7
migrations.py
Go to the documentation of this file.
00001 import sys
00002 import transaction
00003 
00004 from Acquisition import aq_base
00005 from Globals import PersistentMapping
00006 from StringIO import StringIO
00007 from Products.CMFCore.utils import getToolByName
00008 from Products.Archetypes.Extensions.utils import install_uidcatalog
00009 from Products.Archetypes.config import *
00010 from Products.Archetypes.interfaces.base import IBaseObject
00011 
00012 # WARNING!
00013 # Using full transactions after every migration step may be dangerous but it's
00014 # required if you don't have enough space and memory
00015 USE_FULL_TRANSACTIONS = False
00016 
00017 # value used for empty relationships
00018 EMPTY_RELATIONSHIP = 'related'
00019 
00020 class StdoutStringIO(StringIO):
00021     """StringIO that also writes to stdout
00022     """
00023     
00024     def write(self, s):
00025         print >> sys.stdout, str(s),
00026         StringIO.write(self, s)
00027 
00028 def reinstallArchetypes(portal, out):
00029     """let's quickinstaller (re)install Archetypes and it's dependencies
00030     """
00031     qi = getToolByName(portal, 'portal_quickinstaller')
00032     products = ('MimetypesRegistry', 'PortalTransforms', 'Archetypes', )
00033     print >>out, 'Reinstalling Archetypes and it\'s dependencies'
00034     for product in products:
00035         if qi.isProductInstalled(product):
00036             qi.reinstallProducts([product])
00037             print >>out, '... reinstalling %s' % product
00038         else:
00039             qi.installProducts([product])
00040             print >>out, '... installing %s' % product
00041     print >>out, 'Done\n'
00042        
00043 def fixArchetypesTool(portal, out):
00044     at = portal.archetype_tool
00045 
00046     if not hasattr(at, '_templates'):
00047         #They come in pairs
00048         at._templates = PersistentMapping()
00049         at._registeredTemplates = PersistentMapping()
00050 
00051     if not hasattr(at, 'catalog_map'):
00052         at.catalog_map = PersistentMapping()
00053 
00054     install_uidcatalog(portal, out)
00055 
00056 
00057 def migrateReferences(portal, out):
00058     # FIRST
00059     # a 1.2 -> 1.3 (new annotation style) migration path
00060     
00061     at = getToolByName(portal, TOOL_NAME)
00062     rc = getToolByName(portal, REFERENCE_CATALOG)
00063     uc = getToolByName(portal, UID_CATALOG)
00064 
00065     count=0
00066     
00067     # Old 1.2 style references are stored inside archetype_tool on the 'ref'
00068     # attribute
00069     refs = getattr(at, 'refs', None)
00070     if refs is not None:
00071         print >>out, 'migrating reference from Archetypes 1.2'
00072         count=0
00073         print >>out, "Old references are stored in %s, so migrating them to new style reference annotations.\n" % (TOOL_NAME)
00074         allbrains = uc()
00075         for brain in allbrains:
00076             sourceObj = brain.getObject()
00077             sourceUID = getattr(aq_base(sourceObj), olduididx, None)
00078             if sourceUID is None: continue
00079             # references migration starts
00080             for targetUID, relationship in refs.get(sourceUID, []):
00081                 # get target object
00082                 targetBrains = uc(**{olduididx:targetUID})
00083                 assert(len(targetBrains) == 1,'catalog query for OLD uid (%s) returned %d results instead of 1' % (targetUID,len(targetBrains)))
00084                 targetObj=targetBrains[0].getObject()
00085                 # fix empty relationship
00086                 if not relationship:
00087                     relationship = EMPTY_RELATIONSHIP
00088                 # create new style reference
00089                 rc.addReference(sourceObj, targetObj, relationship)
00090                 count+=1        
00091                 if not count % 10:
00092                     print >>out, '.',
00093                 # avoid eating up all RAM
00094                 if not count % 250:
00095                     print >>out, '*',
00096                     transaction.savepoint(optimistic=True)
00097             print >>out, "\n%s old references migrated." % count
00098         # after all remove the old-style reference attribute
00099         delattr(at, 'refs')
00100         print >>out, 'Done\n'
00101         if USE_FULL_TRANSACTIONS:
00102             transaction.commit()
00103         else:
00104             transaction.savepoint(optimistic=True)
00105     
00106     else:
00107         # SECOND
00108         # a 1.3.b2 -> 1.3 (new annotation style) migration path
00109         # We had a reference catalog, make sure its doing annotation
00110         # based references
00111     
00112         # reference metadata cannot be restored since reference-catalog is no more
00113         # a btree and in AT 1.3.b2 reference_catalog was a btreefolder
00114 
00115         print >>out, 'migrating reference from Archetypes 1.3. beta2\n'
00116 
00117         refs = rc()
00118         rc.manage_catalogClear()
00119         for brain in refs:
00120             sourceObject = rc.lookupObject(brain.sourceUID)
00121             if sourceObject is None: continue
00122             targetObject=rc.lookupObject(brain.targetUID)
00123             if not targetObject:
00124                 print >>out,  'mirateReferences: Warning: no targetObject found for UID ',brain.targetUID
00125                 continue
00126             relationship = brain.relationship
00127             # fix empty relationship
00128             if not relationship:
00129                 relationship = EMPTY_RELATIONSHIP
00130             sourceObject.addReference(targetObject, relationship = relationship)
00131             count+=1
00132             if not count % 10:
00133                 print >>out, '.',
00134             # avoid eating up all RAM
00135             if not count % 250:
00136                 print >>out, '*',
00137                 transaction.savepoint(optimistic=True)
00138 
00139         print >>out, "%s old references migrated (reference metadata not restored)." % count
00140         print >>out, '\nDone\n'
00141         if USE_FULL_TRANSACTIONS:
00142             transaction.commit()
00143         else:
00144             transaction.savepoint(optimistic=True)
00145 
00146     print >>out, "Migrated References"
00147 
00148 
00149     #Reindex for new UUIDs
00150     uc.manage_reindexIndex()
00151     rc.manage_reindexIndex()
00152 
00153 olduididx = 'old_tmp_at_uid'
00154 
00155 def migrateUIDs(portal, out):
00156     count=0
00157     uc = getToolByName(portal, UID_CATALOG)    
00158     print >>out, 'Migrating uids\n'
00159     
00160     # temporary add a new index    
00161     if olduididx not in uc.indexes():
00162         uc.addIndex(olduididx, 'FieldIndex', extra=None)
00163         if not olduididx in uc.schema():
00164             uc.addColumn(olduididx)
00165     
00166     # clear UID Catalog 
00167     uc.manage_catalogClear()
00168     
00169     # rebuild UIDS on objects and in catalog
00170     allbrains = portal.portal_catalog()
00171     for brain in allbrains:
00172         # get a uid for each thingie
00173         obj = brain.getObject()
00174         if not IBaseObject.isImplementedBy(obj): 
00175             continue #its no Archetype instance, so leave it
00176         
00177         objUID = getattr(aq_base(obj), '_uid', None)        
00178         if objUID is not None: #continue    # not an old style AT?
00179             setattr(obj, olduididx, objUID) # this one can be part of the catalog
00180             delattr(obj, '_uid')
00181             setattr(obj, UUID_ATTR, None)
00182         obj._register()            # creates a new UID
00183         obj._updateCatalog(portal) # to be sure
00184         count+=1
00185         if not count % 10:
00186             print >>out, '.',
00187         # avoid eating up all RAM
00188         if not count % 250:
00189             print >>out, '*',
00190             transaction.savepoint(optimistic=True)
00191     print >>out, '\nDone\n'
00192     if USE_FULL_TRANSACTIONS:
00193         transaction.commit()
00194     else:
00195         transaction.savepoint(optimistic=True)
00196 
00197     print >>out, count, "UID's migrated."
00198 
00199 def removeOldUIDs(portal, out):
00200     # remove temporary needed index 
00201     uc = getToolByName(portal, UID_CATALOG)    
00202     print >>out, 'Removing old uids\n'
00203     if olduididx in uc.indexes():
00204         uc.delIndex(olduididx)
00205         if olduididx in uc.schema():
00206             uc.delColumn(olduididx)
00207     count=0
00208     allbrains = uc()
00209     for brain in allbrains:
00210         #Get a uid for each thingie
00211         obj = brain.getObject()
00212         objUID = getattr(aq_base(obj), olduididx, None)        
00213         if objUID is None: continue # not an old style AT
00214         delattr(obj, olduididx)
00215         obj._updateCatalog(portal) 
00216         count+=1
00217         if not count % 10:
00218             print >>out, '.',
00219         # avoid eating up all RAM
00220         if not count % 250:
00221             print >>out, '*',
00222             transaction.savepoint(optimistic=True)
00223 
00224     if USE_FULL_TRANSACTIONS:
00225         transaction.commit()
00226     else:
00227         transaction.savepoint(optimistic=True)
00228 
00229     print >>out, "\n%s old UID attributes removed." % count
00230     print >>out, 'Done\n'
00231 
00232 def migrateSchemas(portal, out):
00233     at = getToolByName(portal, TOOL_NAME)
00234     msg = at.manage_updateSchema(update_all=1)
00235     if USE_FULL_TRANSACTIONS:
00236         transaction.commit()
00237     else:
00238         transaction.savepoint(optimistic=True)
00239     print >>out, msg
00240 
00241 def migrateCatalogIndexes(portal, out):
00242     def addIndex(catalog, indexName, indexType):
00243         # copy of code in utils.py:install_referenceCatalog
00244         try:
00245             catalog.addIndex(indexName, indexType, extra=None)
00246         except:
00247             pass
00248         try:
00249             if not indexName in catalog.schema():
00250                 catalog.addColumn(indexName)
00251         except:
00252             pass
00253     
00254     rc = getToolByName(portal, REFERENCE_CATALOG)
00255     add_indexes = ('targetId', 'FieldIndex'),
00256     [addIndex(rc, n, t) for n, t in add_indexes]
00257     
00258 def refreshCatalogs(portal, out):
00259     uc = getToolByName(portal, UID_CATALOG)
00260     rc = getToolByName(portal, REFERENCE_CATALOG)
00261     print >>out, 'Refreshing uid catalog'
00262     uc.refreshCatalog(clear=1)
00263     print >>out, 'Refreshing reference catalog'
00264     rc.refreshCatalog(clear=1)
00265 
00266     if USE_FULL_TRANSACTIONS:
00267         transaction.commit()
00268     else:
00269         transaction.savepoint(optimistic=True)
00270 
00271 
00272 def migrate(self):
00273     """migrate an AT site"""
00274     out = StdoutStringIO()
00275     portal = getToolByName(self,'portal_url').getPortalObject()
00276 
00277     print >>out, "Begin Migration"
00278 
00279     fixArchetypesTool(portal, out)
00280     reinstallArchetypes(portal,out)
00281     migrateSchemas(portal, out)
00282     migrateUIDs(portal, out)
00283     migrateReferences(portal,out)
00284     removeOldUIDs(portal, out)
00285     migrateCatalogIndexes(portal, out)
00286     refreshCatalogs(portal, out)
00287     print >>out, "Archetypes Migration Successful"
00288     return out.getvalue()