Back to index

moin  1.9.0~rc2
build.py
Go to the documentation of this file.
00001 # -*- coding: iso-8859-1 -*-
00002 """
00003 MoinMoin - build xapian search engine's index
00004 
00005 @copyright: 2006-2009 MoinMoin:ThomasWaldmann
00006 @license: GNU GPL, see COPYING for details.
00007 """
00008 
00009 import os
00010 import errno
00011 import shutil
00012 
00013 from MoinMoin.script import MoinScript
00014 
00015 class IndexScript(MoinScript):
00016     """\
00017 Purpose:
00018 ========
00019 This tool allows you to control xapian's index of Moin.
00020 
00021 Detailed Instructions:
00022 ======================
00023 General syntax: moin [options] index build [build-options]
00024 
00025 [options] usually should be:
00026     --config-dir=/path/to/my/cfg/ --wiki-url=wiki.example.org/
00027 
00028 [build-options] see below:
00029     Please note:
00030     * You must run this script as the owner of the wiki files,
00031       usually this is the web server user.
00032     * You may add the build-option --files=files.lst to let the indexer
00033       also consider the filesystem filenames contained in that file (one
00034       filename per line). Search results from these files will be "found"
00035       under a special pseudo page called FS (like File System).
00036       Without this option, the indexer will just consider pages and attachments.
00037 
00038     1. Conditionally (considering modification time) update the index:
00039        moin ... index build --mode=update
00040 
00041     2. Unconditionally add to the index:
00042        moin ... index build --mode=add
00043 
00044     3. Completely rebuild the index (1-stage):
00045        moin ... index build --mode=rebuild
00046 
00047        Note: until it has completely built the new index, the wiki will still
00048        use the old index. After rebuild has completed, it kills the old index
00049        and moves the new index into its place.
00050        If the wiki uses the index at that moment, that might have unwanted side
00051        effects. If you want to avoid that and you can accept a short downtime,
00052        consider using this safer method:
00053 
00054        Completely rebuild the index (2-stage):
00055        # takes long, does not interfere with wiki searches:
00056        moin ... index build --mode=buildnewindex
00057        stop this moin wiki process(es)
00058        # quick, replaces the old index with the new one:
00059        moin ... index build --mode=usenewindex
00060        start this moin wiki process(es)
00061 """
00062 
00063     def __init__(self, argv, def_values):
00064         MoinScript.__init__(self, argv, def_values)
00065         self.parser.add_option(
00066             "--files", metavar="FILES", dest="file_list",
00067             help="filename of file list, e.g. files.lst (one file per line)"
00068         )
00069         self.parser.add_option(
00070             "--mode", metavar="MODE", dest="mode",
00071             help="either add (unconditionally add), update (conditional update), rebuild (complete 1-stage index rebuild)"
00072                  " or buildnewindex and usenewindex (complete 2-stage index rebuild)"
00073         )
00074 
00075     def mainloop(self):
00076         self.init_request()
00077         # Do we have additional files to index?
00078         if self.options.file_list:
00079             self.files = file(self.options.file_list)
00080         else:
00081             self.files = None
00082         self.command()
00083 
00084 class PluginScript(IndexScript):
00085     """ Xapian index build script class """
00086 
00087     def command(self):
00088         from MoinMoin.search.Xapian import XapianIndex
00089         mode = self.options.mode
00090         if mode in ('rebuild', 'buildnewindex'):
00091             # rebuilding the DB into a new index directory, so the rebuild
00092             # process does not interfere with the currently in-use DB
00093             idx_mode, idx_name = 'add', 'index.new'
00094         elif mode in ('add', 'update'):
00095             # update/add in-place
00096             idx_mode, idx_name = mode, 'index'
00097         elif mode == 'usenewindex':
00098             pass # nothing todo
00099         else:
00100             pass # XXX give error msg about invalid mode
00101 
00102         if mode != 'usenewindex':
00103             idx = XapianIndex(self.request, name=idx_name)
00104             idx.indexPages(self.files, idx_mode)
00105 
00106         if mode in ('rebuild', 'usenewindex'):
00107             # 'rebuild' is still a bit dirty, because just killing old index will
00108             # fail currently running searches. Thus, maybe do this in a time
00109             # with litte wiki activity or better use 'buildnewindex' and
00110             # 'usenewindex' (see above).
00111             # XXX code here assumes that idx.db is a directory
00112             # TODO improve this with xapian stub DBs
00113             idx_old = XapianIndex(self.request, name='index').db
00114             idx_new = XapianIndex(self.request, name='index.new').db
00115             try:
00116                 shutil.rmtree(idx_old)
00117             except OSError, err:
00118                 if err.errno != errno.ENOENT: # ignore it if we have no current index
00119                     raise
00120             os.rename(idx_new, idx_old)
00121