Back to index

moin  1.9.0~rc2
wikisync.py
Go to the documentation of this file.
00001 # -*- coding: iso-8859-1 -*-
00002 """
00003     MoinMoin - Wiki Synchronisation
00004 
00005     @copyright: 2006 MoinMoin:AlexanderSchremmer
00006     @license: GNU GPL, see COPYING for details.
00007 """
00008 
00009 import os
00010 import socket
00011 import xmlrpclib
00012 
00013 from MoinMoin import wikiutil
00014 from MoinMoin.util import lock, pickle
00015 from MoinMoin.Page import Page
00016 from MoinMoin.PageEditor import PageEditor
00017 from MoinMoin.packages import unpackLine, packLine
00018 
00019 
00020 MIMETYPE_MOIN = "text/wiki"
00021 # sync directions
00022 UP, DOWN, BOTH = range(3)
00023 
00024 
00025 def normalise_pagename(page_name, prefix):
00026     """ Checks if the page_name starts with the prefix.
00027         Returns None if it does not, otherwise strips the prefix.
00028     """
00029     if prefix:
00030         if not page_name.startswith(prefix):
00031             return None
00032         else:
00033             return page_name[len(prefix):]
00034     else:
00035         return page_name
00036 
00037 
00038 class UnsupportedWikiException(Exception):
00039     pass
00040 
00041 
00042 class NotAllowedException(Exception):
00043     pass
00044 
00045 
00046 class SyncPage(object):
00047     """ This class represents a page in one or two wiki(s). """
00048     def __init__(self, name, local_rev=None, remote_rev=None, local_name=None, remote_name=None,
00049                  local_deleted=False, remote_deleted=False):
00050         """ Creates a SyncPage instance.
00051             @param name: The canonical name of the page, without prefixes.
00052             @param local_rev: The revision of the page in the local wiki.
00053             @param remote_rev: The revision of the page in the remote wiki.
00054             @param local_name: The page name of the page in the local wiki.
00055             @param remote_name: The page name of the page in the remote wiki.
00056         """
00057         self.name = name
00058         self.local_rev = local_rev
00059         self.remote_rev = remote_rev
00060         self.local_name = local_name
00061         self.remote_name = remote_name
00062         assert local_rev or remote_rev
00063         assert local_name or remote_name
00064         self.local_deleted = local_deleted
00065         self.remote_deleted = remote_deleted
00066         self.local_mime_type = MIMETYPE_MOIN   # XXX no usable storage API yet
00067         self.remote_mime_type = MIMETYPE_MOIN
00068         assert remote_rev != 99999999
00069 
00070     def __repr__(self):
00071         return repr("<Sync Page %r>" % unicode(self))
00072 
00073     def __unicode__(self):
00074         return u"%s[%s|%s]<%r:%r>" % (self.name, self.local_name, self.remote_name, self.local_rev, self.remote_rev)
00075 
00076     def __lt__(self, other):
00077         return self.name < other.name
00078 
00079     def __hash__(self):
00080         """ Ensures that the hash value of this page only depends on the canonical name. """
00081         return hash(self.name)
00082 
00083     def __eq__(self, other):
00084         if not isinstance(other, SyncPage):
00085             return False
00086         return self.name == other.name
00087 
00088     def add_missing_pagename(self, local, remote):
00089         """ Checks if the particular concrete page names are unknown and fills
00090             them in.
00091         """
00092         if self.local_name is None:
00093             n_name = normalise_pagename(self.remote_name, remote.prefix)
00094             assert n_name is not None
00095             self.local_name = (local.prefix or "") + n_name
00096         elif self.remote_name is None:
00097             n_name = normalise_pagename(self.local_name, local.prefix)
00098             assert n_name is not None
00099             self.remote_name = (remote.prefix or "") + n_name
00100 
00101         return self # makes using list comps easier
00102 
00103     def filter(cls, sp_list, func):
00104         """ Returns all pages in sp_list that let func return True
00105             for the canonical page name.
00106         """
00107         return [x for x in sp_list if func(x.name)]
00108     filter = classmethod(filter)
00109 
00110     def merge(cls, local_list, remote_list):
00111         """ Merges two lists of SyncPages into one, migrating attributes like the names. """
00112         # map page names to SyncPage objects :-)
00113         d = dict(zip(local_list, local_list))
00114         for sp in remote_list:
00115             if sp in d:
00116                 d[sp].remote_rev = sp.remote_rev
00117                 d[sp].remote_name = sp.remote_name
00118                 d[sp].remote_deleted = sp.remote_deleted
00119                 # XXX merge mime type here
00120             else:
00121                 d[sp] = sp
00122         return d.keys()
00123     merge = classmethod(merge)
00124 
00125     def is_only_local(self):
00126         """ Is true if the page is only in the local wiki. """
00127         return not self.remote_rev
00128 
00129     def is_only_remote(self):
00130         """ Is true if the page is only in the remote wiki. """
00131         return not self.local_rev
00132 
00133     def is_local_and_remote(self):
00134         """ Is true if the page is in both wikis. """
00135         return self.local_rev and self.remote_rev
00136 
00137 
00138 class RemoteWiki(object):
00139     """ This class should be the base for all implementations of remote wiki
00140         classes. """
00141 
00142     def __repr__(self):
00143         """ Returns a representation of the instance for debugging purposes. """
00144         return NotImplemented
00145 
00146     def get_interwiki_name(self):
00147         """ Returns the interwiki name of the other wiki. """
00148         return NotImplemented
00149 
00150     def get_iwid(self):
00151         """ Returns the InterWiki ID. """
00152         return NotImplemented
00153 
00154     def get_pages(self, **kw):
00155         """ Returns a list of SyncPage instances. """
00156         return NotImplemented
00157 
00158 
00159 class MoinRemoteWiki(RemoteWiki):
00160     """ Used for MoinMoin wikis reachable via XMLRPC. """
00161     def __init__(self, request, interwikiname, prefix, pagelist, user, password, verbose=False):
00162         self.request = request
00163         self.prefix = prefix
00164         self.pagelist = pagelist
00165         self.verbose = verbose
00166         _ = self.request.getText
00167 
00168         wikitag, wikiurl, wikitail, wikitag_bad = wikiutil.resolve_interwiki(self.request, interwikiname, '')
00169         self.wiki_url = wikiutil.mapURL(self.request, wikiurl)
00170         self.valid = not wikitag_bad
00171         self.xmlrpc_url = self.wiki_url + "?action=xmlrpc2"
00172         if not self.valid:
00173             self.connection = None
00174             return
00175 
00176         self.connection = self.createConnection()
00177 
00178         try:
00179             iw_list = self.connection.interwikiName()
00180         except socket.error:
00181             raise UnsupportedWikiException(_("The wiki is currently not reachable."))
00182         except xmlrpclib.Fault, err:
00183             raise UnsupportedWikiException("xmlrpclib.Fault: %s" % str(err))
00184 
00185         if user and password:
00186             token = self.connection.getAuthToken(user, password)
00187             if token:
00188                 self.token = token
00189             else:
00190                 raise NotAllowedException(_("Invalid username or password."))
00191         else:
00192             self.token = None
00193 
00194         self.remote_interwikiname = remote_interwikiname = iw_list[0]
00195         self.remote_iwid = remote_iwid = iw_list[1]
00196         self.is_anonymous = remote_interwikiname is None
00197         if not self.is_anonymous and interwikiname != remote_interwikiname:
00198             raise UnsupportedWikiException(_("The remote wiki uses a different InterWiki name (%(remotename)s)"
00199                                              " internally than you specified (%(localname)s).") % {
00200                 "remotename": wikiutil.escape(remote_interwikiname), "localname": wikiutil.escape(interwikiname)})
00201 
00202         if self.is_anonymous:
00203             self.iwid_full = packLine([remote_iwid])
00204         else:
00205             self.iwid_full = packLine([remote_iwid, interwikiname])
00206 
00207     def createConnection(self):
00208         return xmlrpclib.ServerProxy(self.xmlrpc_url, allow_none=True, verbose=self.verbose)
00209 
00210     # Public methods
00211     def get_diff_pre(self, pagename, from_rev, to_rev, n_name=None):
00212         """ Returns the binary diff of the remote page named pagename, given
00213             from_rev and to_rev. Generates the call. """
00214         return "getDiff", (pagename, from_rev, to_rev, n_name)
00215 
00216     def get_diff_post(self, value):
00217         """ Processes the return value of the call generated by get_diff_pre. """
00218         if isinstance(value, xmlrpclib.Fault):
00219             if value.faultCode == "INVALID_TAG":
00220                 return None
00221             raise value
00222         value["diff"] = str(value["diff"]) # unmarshal Binary object
00223         return value
00224 
00225     def merge_diff_pre(self, pagename, diff, local_rev, delta_remote_rev, last_remote_rev, interwiki_name, n_name):
00226         """ Merges the diff into the page on the remote side. Generates the call. """
00227         return "mergeDiff", (pagename, xmlrpclib.Binary(diff), local_rev, delta_remote_rev, last_remote_rev, interwiki_name, n_name)
00228 
00229     def merge_diff_post(self, result):
00230         """ Processes the return value of the call generated by merge_diff_pre.  """
00231         if isinstance(result, xmlrpclib.Fault):
00232             if result.faultCode == "NOT_ALLOWED":
00233                 raise NotAllowedException
00234             raise result
00235         return result
00236 
00237     def delete_page_pre(self, pagename, last_remote_rev, interwiki_name):
00238         """ Deletes a remote page. Generates the call. """
00239         return "mergeDiff", (pagename, None, None, None, last_remote_rev, interwiki_name, None)
00240 
00241     def delete_page_post(self, result):
00242         """ Processes the return value of the call generated by delete_page_pre. """
00243         if isinstance(result, xmlrpclib.Fault):
00244             if result.faultCode == "NOT_ALLOWED":
00245                 return result.faultString
00246             raise result
00247         return ""
00248 
00249     def create_multicall_object(self):
00250         """ Generates an object that can be used like a MultiCall instance. """
00251         return xmlrpclib.MultiCall(self.connection)
00252 
00253     def prepare_multicall(self):
00254         """ Can be used to return initial calls that e.g. authenticate the user.
00255             @return: [(funcname, (arg,+)*]
00256         """
00257         if self.token:
00258             return [("applyAuthToken", (self.token, ))]
00259         return []
00260 
00261     def delete_auth_token(self):
00262         if self.token:
00263             self.connection.deleteAuthToken(self.token)
00264             self.token = None
00265 
00266     # Methods implementing the RemoteWiki interface
00267 
00268     def get_interwiki_name(self):
00269         return self.remote_interwikiname
00270 
00271     def get_iwid(self):
00272         return self.remote_iwid
00273 
00274     def get_pages(self, **kwargs):
00275         options = {"include_revno": True,
00276                    "include_deleted": True,
00277                    "exclude_non_writable": kwargs["exclude_non_writable"],
00278                    "include_underlay": False,
00279                    "prefix": self.prefix,
00280                    "pagelist": self.pagelist,
00281                    "mark_deleted": True}
00282         if self.token:
00283             m = xmlrpclib.MultiCall(self.connection)
00284             m.applyAuthToken(self.token)
00285             m.getAllPagesEx(options)
00286             tokres, pages = m()
00287         else:
00288             pages = self.connection.getAllPagesEx(options)
00289         rpages = []
00290         for name, revno in pages:
00291             normalised_name = normalise_pagename(name, self.prefix)
00292             if normalised_name is None:
00293                 continue
00294             if abs(revno) != 99999999: # I love sane in-band signalling
00295                 remote_rev = abs(revno)
00296                 remote_deleted = revno < 0
00297                 rpages.append(SyncPage(normalised_name, remote_rev=remote_rev, remote_name=name, remote_deleted=remote_deleted))
00298         return rpages
00299 
00300     def __repr__(self):
00301         return "<MoinRemoteWiki wiki_url=%r valid=%r>" % (getattr(self, "wiki_url", Ellipsis), getattr(self, "valid", Ellipsis))
00302 
00303 
00304 class MoinLocalWiki(RemoteWiki):
00305     """ Used for the current MoinMoin wiki. """
00306     def __init__(self, request, prefix, pagelist):
00307         self.request = request
00308         self.prefix = prefix
00309         self.pagelist = pagelist
00310 
00311     def getGroupItems(self, group_list):
00312         """ Returns all page names that are listed on the page group_list. """
00313         pages = []
00314         for group_pagename in group_list:
00315             pages.extend(request.groups.get(group_pagename, []))
00316         return [self.createSyncPage(x) for x in pages]
00317 
00318     def createSyncPage(self, page_name):
00319         normalised_name = normalise_pagename(page_name, self.prefix)
00320         if normalised_name is None:
00321             return None
00322         page = Page(self.request, page_name)
00323         revno = page.get_real_rev()
00324         if revno == 99999999: # I love sane in-band signalling
00325             return None
00326         return SyncPage(normalised_name, local_rev=revno, local_name=page_name, local_deleted=not page.exists())
00327 
00328     # Public methods:
00329 
00330     # Methods implementing the RemoteWiki interface
00331     def delete_page(self, pagename, comment):
00332         page = PageEditor(self.request, pagename)
00333         try:
00334             page.deletePage(comment)
00335         except PageEditor.AccessDenied, (msg, ):
00336             return msg
00337         return ""
00338 
00339     def get_interwiki_name(self):
00340         return self.request.cfg.interwikiname
00341 
00342     def get_iwid(self):
00343         return self.request.cfg.iwid
00344 
00345     def get_pages(self, **kwargs):
00346         assert not kwargs
00347         if self.prefix or self.pagelist:
00348             def page_filter(name, prefix=(self.prefix or ""), pagelist=self.pagelist):
00349                 n_name = normalise_pagename(name, prefix)
00350                 if not n_name:
00351                     return False
00352                 if not pagelist:
00353                     return True
00354                 return n_name in pagelist
00355         else:
00356             page_filter = lambda x: True
00357         pages = []
00358         for x in self.request.rootpage.getPageList(exists=False, include_underlay=False, filter=page_filter):
00359             sp = self.createSyncPage(x)
00360             if sp:
00361                 pages.append(sp)
00362         return pages
00363 
00364     def __repr__(self):
00365         return "<MoinLocalWiki>"
00366 
00367 
00368 # ------------------ Tags ------------------
00369 
00370 
00371 class Tag(object):
00372     """ This class is used to store information about merging state. """
00373 
00374     def __init__(self, remote_wiki, remote_rev, current_rev, direction, normalised_name):
00375         """ Creates a new Tag.
00376 
00377         @param remote_wiki: The identifier of the remote wiki.
00378         @param remote_rev: The revision number on the remote end.
00379         @param current_rev: The related local revision.
00380         @param direction: The direction of the sync, encoded as an integer.
00381         """
00382         assert (isinstance(remote_wiki, basestring) and isinstance(remote_rev, int)
00383                 and isinstance(current_rev, int) and isinstance(direction, int)
00384                 and isinstance(normalised_name, basestring))
00385         self.remote_wiki = remote_wiki
00386         self.remote_rev = remote_rev
00387         self.current_rev = current_rev
00388         self.direction = direction
00389         self.normalised_name = normalised_name
00390 
00391     def __repr__(self):
00392         return u"<Tag normalised_pagename=%r remote_wiki=%r remote_rev=%r current_rev=%r>" % (getattr(self, "normalised_name", "UNDEF"), self.remote_wiki, self.remote_rev, self.current_rev)
00393 
00394     def __cmp__(self, other):
00395         if not isinstance(other, Tag):
00396             return NotImplemented
00397         return cmp(self.current_rev, other.current_rev)
00398 
00399 
00400 class AbstractTagStore(object):
00401     """ This class is an abstract base class that shows how to implement classes
00402         that manage the storage of tags. """
00403 
00404     def __init__(self, page):
00405         """ Subclasses don't need to call this method. It is just here to enforce
00406         them having accept a page argument at least. """
00407         pass
00408 
00409     def dump(self):
00410         """ Returns all tags for a given item as a string. """
00411         return repr(self.get_all_tags())
00412 
00413     def add(self, **kwargs):
00414         """ Adds a Tag object to the current TagStore. """
00415         print "Got tag for page %r: %r" % (self.page, kwargs)
00416         return NotImplemented
00417 
00418     def get_all_tags(self):
00419         """ Returns a list of all Tag objects associated to this page. """
00420         return NotImplemented
00421 
00422     def get_last_tag(self):
00423         """ Returns the newest tag. """
00424         return NotImplemented
00425 
00426     def clear(self):
00427         """ Removes all tags. """
00428         return NotImplemented
00429 
00430     def fetch(self, iwid_full=None, direction=None):
00431         """ Fetches tags by a special IWID or interwiki name. """
00432         return NotImplemented
00433 
00434 
00435 class PickleTagStore(AbstractTagStore):
00436     """ This class manages the storage of tags in pickle files. """
00437 
00438     def __init__(self, page):
00439         """ Creates a new TagStore that uses pickle files.
00440 
00441         @param page: a Page object where the tags should be related to
00442         """
00443 
00444         self.page = page
00445         self.filename = page.getPagePath('synctags', use_underlay=0, check_create=1, isfile=1)
00446         lock_dir = os.path.join(page.getPagePath('cache', use_underlay=0, check_create=1), '__taglock__')
00447         self.rlock = lock.ReadLock(lock_dir, 60.0)
00448         self.wlock = lock.WriteLock(lock_dir, 60.0)
00449 
00450         if not self.rlock.acquire(3.0):
00451             raise EnvironmentError("Could not lock in PickleTagStore")
00452         try:
00453             self.load()
00454         finally:
00455             self.rlock.release()
00456 
00457     def load(self):
00458         """ Loads the tags from the data file. """
00459         try:
00460             datafile = file(self.filename, "rb")
00461             self.tags = pickle.load(datafile)
00462         except (IOError, EOFError):
00463             self.tags = []
00464         else:
00465             datafile.close()
00466 
00467     def commit(self):
00468         """ Writes the memory contents to the data file. """
00469         datafile = file(self.filename, "wb")
00470         pickle.dump(self.tags, datafile, pickle.HIGHEST_PROTOCOL)
00471         datafile.close()
00472 
00473     # public methods ---------------------------------------------------
00474     def add(self, **kwargs):
00475         if not self.wlock.acquire(3.0):
00476             raise EnvironmentError("Could not lock in PickleTagStore")
00477         try:
00478             self.load()
00479             self.tags.append(Tag(**kwargs))
00480             self.commit()
00481         finally:
00482             self.wlock.release()
00483 
00484     def get_all_tags(self):
00485         return self.tags[:]
00486 
00487     def get_last_tag(self):
00488         temp = self.tags[:]
00489         temp.sort()
00490         if not temp:
00491             return None
00492         return temp[-1]
00493 
00494     def clear(self):
00495         self.tags = []
00496         if not self.wlock.acquire(3.0):
00497             raise EnvironmentError("Could not lock in PickleTagStore")
00498         try:
00499             self.commit()
00500         finally:
00501             self.wlock.release()
00502 
00503     def fetch(self, iwid_full, direction=None):
00504         iwid_full = unpackLine(iwid_full)
00505         matching_tags = []
00506         for t in self.tags:
00507             t_iwid_full = unpackLine(t.remote_wiki)
00508             if ((t_iwid_full[0] == iwid_full[0]) # either match IWID or IW name
00509                 or (len(t_iwid_full) == 2 and len(iwid_full) == 2 and t_iwid_full[1] == iwid_full[1])
00510                 ) and (direction is None or t.direction == direction):
00511                 matching_tags.append(t)
00512         return matching_tags
00513 
00514 
00515 # currently we just have one implementation, so we do not need
00516 # a factory method
00517 TagStore = PickleTagStore
00518