Back to index

moin  1.9.0~rc2
Functions | Variables
MoinMoin.script.old.migration.12_to_13_mig05 Namespace Reference

Functions

def gather_editlog
def gather_pagedirs
def remove_trash
def generate_pages
def generate_editlog

Variables

dictionary info = {}
dictionary info2 = {}
dictionary exists = {}
list pagelist = []
string origdir = 'data.pre-mig5'

Function Documentation

this gathers everything that is in edit-log into internal
    data structures, converting to the future format

Definition at line 88 of file 12_to_13_mig05.py.

00088 
00089 def gather_editlog(dir_from, el_from):
00090     """ this gathers everything that is in edit-log into internal
00091         data structures, converting to the future format
00092     """
00093     if not os.path.exists(el_from):
00094         return
00095     for l in open(el_from):
00096         data = l.rstrip('\n').split('\t')
00097         origlen = len(data)
00098         while len(data) < 7: data.append('')
00099         (pagename, ip, timestamp, host, id, comment, action) = data
00100         if origlen == 6:
00101             action = comment
00102             comment = ''
00103 
00104         extra = ''
00105         if action == 'SAVE/REVERT': # we missed to convert that in mig4
00106             ts = long(comment) # must be long for py 2.2.x
00107             if ts < 4000000000: # UNIX timestamp (secs)
00108                 extra = str(wikiutil.timestamp2version(ts))
00109             else: # usecs timestamp
00110                 extra = str(ts)
00111             # later we convert this timestamp to a revision number
00112             comment = ''
00113         if action in ['ATTNEW', 'ATTDRW', 'ATTDEL', ]:
00114             extra = comment # filename
00115             comment = '' # so we can use comments on ATT* in future
00116 
00117         timestamp = long(timestamp) # must be long for py 2.2.x
00118         data = [timestamp, '', action, pagename, ip, host, id, extra, comment]
00119 
00120         entry = info.get(pagename, {})
00121         entry[timestamp] = [None, data]
00122         info[pagename] = entry

Here is the caller graph for this function:

def MoinMoin.script.old.migration.12_to_13_mig05.gather_pagedirs (   dir_from,
  is_backupdir = 0 
)
this gathers information from the pagedirs, i.e. text and backup
    files (and also the local editlog) and tries to merge/synchronize
    with the informations gathered from editlog

Definition at line 123 of file 12_to_13_mig05.py.

00123 
00124 def gather_pagedirs(dir_from, is_backupdir=0):
00125     """ this gathers information from the pagedirs, i.e. text and backup
00126         files (and also the local editlog) and tries to merge/synchronize
00127         with the informations gathered from editlog
00128     """
00129     global pagelist
00130     pagelist = listdir(dir_from)
00131     for pagename in pagelist:
00132         editlog_from = opj(dir_from, pagename, 'edit-log')
00133         gather_editlog(dir_from, editlog_from)
00134 
00135         entry = info.get(pagename, {})
00136 
00137         loglist = [] # editlog timestamps of page revisions
00138         for ts, data in entry.items():
00139             if data[1][2] in ['SAVE', 'SAVENEW', 'SAVE/REVERT', ]:
00140                 loglist.append(ts)
00141         loglist.sort()
00142         lleftover = loglist[:]
00143 
00144         # remember the latest log entry
00145         if lleftover:
00146             llatest = lleftover[-1]
00147         else:
00148             llatest = None
00149 
00150         backupdir_from = opj(dir_from, pagename, 'backup')
00151         if os.path.exists(backupdir_from):
00152             backuplist = listdir(backupdir_from)
00153             bleftover = backuplist[:]
00154             for bfile in backuplist:
00155                 backup_from = opj(backupdir_from, bfile)
00156                 ts = long(bfile)
00157                 if ts in loglist: # we have an editlog entry, exact match
00158                     entry[ts][0] = backup_from
00159                     lleftover.remove(ts)
00160                     bleftover.remove(bfile)
00161 
00162         text_from = opj(dir_from, pagename, 'text')
00163         found_text = False
00164         if os.path.exists(text_from): # we have a text file, it should match latest log entry
00165             exists[pagename] = True
00166             mtime = os.path.getmtime(text_from)
00167             if llatest and llatest in lleftover:
00168                 ts = llatest
00169                 if abs(wikiutil.timestamp2version(mtime) - ts) < 2000000: # less than a second diff
00170                     entry[ts][0] = text_from
00171                     lleftover.remove(ts)
00172                     found_text = True
00173             else: # we have no log entries left 8(
00174                 ts = wikiutil.timestamp2version(mtime)
00175                 data = [ts, '', 'SAVE', pagename, '', '', '', '', 'missing editlog entry for this page version']
00176                 entry[ts] = [text_from, data]
00177         else:
00178             # this page was maybe deleted, so we remember for later:
00179             exists[pagename] = False
00180             if llatest in lleftover: # if a page is deleted, the last log entry has no file
00181                 entry[llatest][0] = None
00182                 lleftover.remove(llatest)
00183 
00184         if os.path.exists(backupdir_from):
00185             backuplist = listdir(backupdir_from)
00186             for bfile in backuplist:
00187                 if not bfile in bleftover: continue
00188                 backup_from = opj(backupdir_from, bfile)
00189                 bts = long(bfile) # must be long for py 2.2.x
00190                 for ts in lleftover:
00191                     tdiff = abs(bts-ts)
00192                     if tdiff < 2000000: # editlog, inexact match
00193                         entry[ts][0] = backup_from
00194                         lleftover.remove(ts)
00195                         bleftover.remove(bfile)
00196                     elif 3599000000 <= tdiff <= 3601000000: # editlog, win32 daylight saving bug
00197                         entry[ts][0] = backup_from
00198                         lleftover.remove(ts)
00199                         bleftover.remove(bfile)
00200                         print "Warning: Win32 daylight saving bug encountered & fixed!"
00201 
00202             if len(bleftover) == 1 and len(lleftover) == 1: # only 1 left, must be this
00203                 backup_from = opj(backupdir_from, bleftover[0])
00204                 entry[lleftover[0]][0] = backup_from
00205                 lleftover = []
00206                 bleftover = []
00207 
00208             # fake some log entries
00209             for bfile in bleftover:
00210                 backup_from = opj(backupdir_from, bfile)
00211                 bts = long(bfile) # must be long py 2.2.x
00212                 data = [ts, '', 'SAVE', pagename, '', '', '', '', 'missing editlog entry for this page version']
00213                 entry[bts] = [backup_from, data]
00214 
00215         # check if we still haven't matched the "text" file
00216         if not found_text and os.path.exists(text_from):
00217             if llatest in lleftover: # latest log entry still free
00218                 entry[llatest][0] = text_from # take it. do not care about mtime of file.
00219                 lleftover.remove(llatest)
00220             else: # log for "text" file is missing or latest was taken by other rev 8(
00221                 mtime = os.path.getmtime(text_from)
00222                 ts = wikiutil.timestamp2version(mtime) # take mtime, we have nothing better
00223                 data = [ts, '', 'SAVE', pagename, '', '', '', '', 'missing editlog entry for this page version']
00224                 entry[ts] = [text_from, data]
00225 
00226         # delete unmatching log entries
00227         for ts in lleftover:
00228             #print "XXX Deleting leftover log entry: %r" % entry[ts]
00229             del entry[ts]
00230 
00231         info[pagename] = entry

Here is the call graph for this function:

Definition at line 285 of file 12_to_13_mig05.py.

00285 
00286 def generate_editlog(dir_from, dir_to):
00287     editlog = {}
00288     for pagename in info2:
00289         entry = info2.get(pagename, {})
00290         for ts in entry:
00291             file_from, data = entry[ts]
00292             editlog[ts] = data
00293 
00294     tslist = editlog.keys()
00295     tslist.sort()
00296 
00297     editlog_file = opj(dir_to, 'edit-log')
00298     f = open(editlog_file, 'w')
00299     for ts in tslist:
00300         data = editlog[ts]
00301         f.write('\t'.join(data)+'\n')
00302     f.close()
00303 

Definition at line 241 of file 12_to_13_mig05.py.

00241 
00242 def generate_pages(dir_from, dir_to):
00243     for pagename in info2:
00244         entry = info2.get(pagename, {})
00245         tslist = entry.keys()
00246         if tslist:
00247             pagedir = opj(dir_to, 'pages', pagename)
00248             os.makedirs(opj(pagedir, 'revisions'))
00249             editlog_file = opj(pagedir, 'edit-log')
00250             f = open(editlog_file, 'w')
00251             rev = 0
00252             tslist.sort()
00253             for ts in tslist:
00254                 rev += 1
00255                 revstr = '%08d' % rev
00256                 file_from, data = entry[ts]
00257                 data[0] = str(ts)
00258                 data[1] = revstr
00259                 if data[2].endswith('/REVERT'):
00260                     # replace the timestamp with the revision number
00261                     revertts = long(data[7]) # must be long for py 2.2.x
00262                     try:
00263                         revertrev = int(entry[revertts][1][1])
00264                     except KeyError:
00265                         # never should trigger...
00266                         print "********* KeyError %s entry[%d][1][1] **********" % (pagename, revertts)
00267                         revertrev = 0
00268                     data[7] = '%08d' % revertrev
00269                 f.write('\t'.join(data)+'\n')
00270                 if file_from is not None:
00271                     file_to = opj(pagedir, 'revisions', revstr)
00272                     copy_file(file_from, file_to)
00273             f.close()
00274 
00275             curr_file = opj(pagedir, 'current')
00276             f = open(curr_file, 'w')
00277             f.write(revstr)
00278             f.close()
00279 
00280         att_from = opj(dir_from, 'pages', pagename, 'attachments')
00281         if os.path.exists(att_from):
00282             att_to = opj(pagedir, 'attachments')
00283             copy_dir(att_from, att_to)
00284 

Here is the call graph for this function:

Definition at line 232 of file 12_to_13_mig05.py.

00232 
00233 def remove_trash(dir_from):
00234     for pagename in info:
00235         # omit dead pages and MoinEditorBackup
00236         if pagename in pagelist and (
00237            os.path.exists(opj(dir_from, pagename, 'text')) or
00238            os.path.exists(opj(dir_from, pagename, 'backup'))
00239            ) and not pagename.endswith('MoinEditorBackup'):
00240             info2[pagename] = info[pagename]


Variable Documentation

Definition at line 85 of file 12_to_13_mig05.py.

Definition at line 83 of file 12_to_13_mig05.py.

Definition at line 84 of file 12_to_13_mig05.py.

Definition at line 304 of file 12_to_13_mig05.py.

Definition at line 86 of file 12_to_13_mig05.py.