Back to index

nordugrid-arc-nox  1.1.0~rc6
Public Member Functions | Public Attributes | Private Member Functions
arcom.store.transdbstore.TransDBStore Class Reference
Inheritance diagram for arcom.store.transdbstore.TransDBStore:
Inheritance graph
[legend]
Collaboration diagram for arcom.store.transdbstore.TransDBStore:
Collaboration graph
[legend]

List of all members.

Public Member Functions

def __init__
def __del__
def getDBFlags
def getDBReady
def list
def get
def set
def restart
def terminate
def lock
def unlock

Public Attributes

 dbp
 database
 dbenv
 dbenv_flags
 deadlock_retries
 cachesize
 sleeptime
 datadir
 non_existent_object
 llock

Private Member Functions

def __configureDB
def __err
def __opendbenv
def __opendb

Detailed Description

Definition at line 28 of file transdbstore.py.


Constructor & Destructor Documentation

def arcom.store.transdbstore.TransDBStore.__init__ (   self,
  storecfg,
  non_existent_object = {} 
)
Constructor of TransDBStore.

TransDBStore(storecfg)

'storecfg' is an XMLNode with a 'DataDir'
'non_existent_object' will be returned if an object not found

Reimplemented from arcom.store.basestore.BaseStore.

Definition at line 30 of file transdbstore.py.

00030 
00031     def __init__(self, storecfg, non_existent_object = {}):
00032         """ Constructor of TransDBStore.
00033 
00034         TransDBStore(storecfg)
00035 
00036         'storecfg' is an XMLNode with a 'DataDir'
00037         'non_existent_object' will be returned if an object not found
00038         """
00039 
00040         BaseStore.__init__(self, storecfg, non_existent_object)
00041         log.msg(arc.VERBOSE, "TransDBStore constructor called")
00042         log.msg(arc.VERBOSE, "datadir:", self.datadir)
00043 
00044         # db and transaction pointers
00045         self.dbp  = None
00046         
00047         self.database  = "store.db"
00048         self.dbenv = db.DBEnv(0)
00049         
00050         self.__configureDB(storecfg)
00051         self.dbenv.set_cachesize(0, self.cachesize, 0)
00052         # if key not found, raise DBNotFoundError:
00053         self.dbenv.set_get_returns_none(0) 
00054 
00055         self.__opendbenv()
00056         log.msg(arc.INFO, "db environment opened")
    
exit gracefully, i.e., close dbp and dbenv

Definition at line 57 of file transdbstore.py.

00057 
00058     def __del__(self):
00059         """
00060         exit gracefully, i.e., close dbp and dbenv
00061         """
00062         self.__err()
00063         self.terminate()

Here is the call graph for this function:

Here is the caller graph for this function:


Member Function Documentation

def arcom.store.transdbstore.TransDBStore.__configureDB (   self,
  storecfg 
) [private]

Definition at line 64 of file transdbstore.py.

00064 
00065     def __configureDB(self, storecfg):
00066         try:
00067             self.dbenv_flags = int(str(storecfg.Get("DBEnvFlags")))
00068         except:
00069             self.dbenv_flags = db.DB_CREATE | \
00070                                db.DB_RECOVER | \
00071                                db.DB_INIT_LOCK | \
00072                                db.DB_INIT_LOG | \
00073                                db.DB_INIT_MPOOL | \
00074                                db.DB_INIT_TXN
00075         try:
00076             self.deadlock_retries = int(str(storecfg.Get('DeadlockRetries')))
00077         except:
00078             log.msg(arc.WARNING, "couldn't find DeadlockRetries, using 5 as default")
00079             self.deadlock_retries = 5
00080         try:
00081             self.cachesize = int(str(storecfg.Get('CacheSize')))
00082         except:
00083             self.cachesize = 10 * 1024 * 1024
00084             log.msg(arc.WARNING, "couldn't find CacheSize, using %d as default"%self.cachesize)
00085         try:
00086             self.cachesize = int(str(storecfg.Get('SleepTime')))
00087         except:
00088             self.sleeptime = 2
00089             log.msg(arc.WARNING, "couldn't find SleepTime, using %d as default"%self.cachesize)

close the db pointer dpb if not closed already

Definition at line 90 of file transdbstore.py.

00090 
00091     def __err(self):
00092         """
00093         close the db pointer dpb if not closed already
00094         """
00095         if self.dbp != None:
00096             self.dbp.close(db.DB_NOSYNC)
00097             log.msg(arc.INFO, "database closed")
00098             #log.msg()

Here is the caller graph for this function:

def arcom.store.transdbstore.TransDBStore.__opendb (   self,
  dbp 
) [private]
Open the db using dbp as db handle

Definition at line 122 of file transdbstore.py.

00122 
00123     def __opendb(self, dbp):
00124         """
00125         Open the db using dbp as db handle
00126         """
00127         while dbp == None:
00128             dbp = db.DB(self.dbenv,0)
00129                         
00130             # todo: db performance can be fine-tuned with
00131             # proper pagesize, berkeley-db/db/ref/am_conf/pagesize.html
00132 #            try:
00133 #                dbp.set_pagesize(512)
00134 #            except db.DBError, msg:
00135 #                self.__err()
00136 #                log.msg()
00137 #                log.msg(arc.ERROR, "Error in dbp.set_pagesize")
00138 #                self.terminate()
00139             try:
00140                 # using linear hashing for db
00141                 dbp.open(self.database, dbtype = db.DB_HASH, 
00142                          flags = self.getDBFlags())
00143             except db.DBError, (errnum, strerror): 
00144                 # It is expected that this condition will be triggered when
00145                 # client sites starts up.
00146                 # It can take a while for the master site to be found and
00147                 # synced, and no DB will be available until then
00148                 if errnum == errno.ENOENT:
00149                     log.msg(arc.VERBOSE, "No stock db available yet - retrying")
00150                     log.msg()
00151                     try:
00152                         dbp.close(0)
00153                     except db.DBError, (errnum2, strerror2):
00154                         self.__del__()
00155                         
00156                         log.msg()
00157                         log.msg(arc.ERROR, "unexpected error closing after failed open")
00158                         sys.exit(1)
00159                         raise db.DBError, strerror2
00160                     dbp = None
00161                     time.sleep(self.sleeptime)
00162                     continue
00163                 elif errnum == db.DB_LOCK_DEADLOCK:
00164                     log.msg(arc.VERBOSE, "got deadlock - retrying")
00165                     try:
00166                         dbp.close(0)
00167                     except db.DBError, (errnum2, strerror2):
00168                         log.msg()
00169                         log.msg(arc.ERROR, "unexpected error closing after failed open")
00170                         #self.__del__()
00171                         
00172                         raise db.DBError, strerror2
00173                     dbp = None
00174                     time.sleep(self.sleeptime)
00175                     continue
00176                 else:
00177                     log.msg()
00178                     return None
00179                     #self.__del__()                    
00180             except:
00181                 log.msg()
00182                 return None
00183                 #self.__del__()
00184                 
00185         return dbp
00186 
    

Here is the call graph for this function:

Here is the caller graph for this function:

open the db env

Definition at line 99 of file transdbstore.py.

00099 
00100     def __opendbenv(self):
00101         """
00102         open the db env
00103         """
00104         try:
00105             self.dbenv.open(self.datadir, self.dbenv_flags, 0)
00106         except db.DBError, msg:
00107             log.msg()
00108             log.msg(arc.ERROR, "Caught exception during DB environment open.")
00109             self.terminate()
00110             #raise db.DBError, msg
        

Here is the call graph for this function:

Here is the caller graph for this function:

Returns the object with the given ID.

get(ID)

'ID' is the ID of the requested object.
If there is no object with this ID, returns the given non_existent_object value.

Definition at line 212 of file transdbstore.py.

00212 
00213     def get(self, ID):
00214         """ Returns the object with the given ID.
00215 
00216         get(ID)
00217 
00218         'ID' is the ID of the requested object.
00219         If there is no object with this ID, returns the given non_existent_object value.
00220         """
00221 
00222         if not self.getDBReady():
00223             raise db.DBError, "db not ready"
00224 
00225         self.dbp = self.__opendb(self.dbp)
00226 
00227         try:
00228             object = self.dbp.get(ID)
00229             try:
00230                 # using cPickle.loads for loading
00231                 object = cPickle.loads(object)
00232             except:
00233                 log.msg()
00234             if not object:
00235                 return copy.deepcopy(self.non_existent_object)
00236             return object
00237         #todo: handle all errors, e.g., deadlock and dead handle
00238         except db.DBNotFoundError:
00239             return copy.deepcopy(self.non_existent_object)
00240         except db.DBKeyEmptyError:
00241             return copy.deepcopy(self.non_existent_object)
00242         except db.DBLockDeadlockError:
00243             log.msg(arc.INFO, "Got deadlock error")            
00244             log.msg()
00245             raise db.DBError, "db deadlock"
00246         except db.DBRepHandleDeadError:
00247             log.msg(arc.INFO, "Got rep_dead_handle error")
00248             self.__err()
00249             self.dbp = None
00250             raise
00251         except db.DBError, msg:
00252             self.__del__()
00253             log.msg()
00254             log.msg(arc.ERROR, "Error getting %s"%ID)
00255             sys.exit(1)
00256             raise db.DBError, "Error listing db"
        

Here is the call graph for this function:

Here is the caller graph for this function:

Reimplemented in storage.ahash.replicatedahash.ReplicationStore.

Definition at line 111 of file transdbstore.py.

00111 
00112     def getDBFlags(self):
00113         # DB_AUTO_COMMIT ensures that all db modifications will
00114         # automatically be enclosed in transactions
00115         return db.DB_CREATE | db.DB_AUTO_COMMIT

Here is the caller graph for this function:

Dummy method always assuming db is ready

Reimplemented in storage.ahash.replicatedahash.ReplicationStore.

Definition at line 116 of file transdbstore.py.

00116 
00117     def getDBReady(self):
00118         """
00119         Dummy method always assuming db is ready
00120         """
00121         return True

Here is the caller graph for this function:

List the IDs of the existing entries.

list()

Returns a list of IDs.

Definition at line 187 of file transdbstore.py.

00187 
00188     def list(self):
00189         """ List the IDs of the existing entries.
00190         
00191         list()
00192         
00193         Returns a list of IDs.
00194         """
00195 
00196         if not self.getDBReady():
00197             raise db.DBError, "db not ready"
00198         self.dbp = self.__opendb(self.dbp)
00199         try:
00200             object = self.dbp.keys()
00201             return object
00202         except db.DBLockDeadlockError:
00203             log.msg(arc.INFO, "Got deadlock error")            
00204             log.msg()
00205             raise db.DBError, "db deadlock"
00206         except db.DBError:
00207             self.__del__()
00208             log.msg()
00209             log.msg(arc.ERROR, "Error listing db")
00210             sys.exit(1)
00211             raise db.DBError, "Error listing db"

Here is the call graph for this function:

def arcom.store.basestore.BaseStore.lock (   self,
  blocking = True 
) [inherited]
Acquire the lock.

lock(blocking = True)

'blocking': if blocking is True, then this only returns when the lock is acquired.
If it is False, then it returns immediately with False if the lock is not available,
or with True if it could be acquired.

Reimplemented in storage.ahash.replicatedahash.ReplicationStore.

Definition at line 27 of file basestore.py.

00027 
00028     def lock(self, blocking = True):
00029         """ Acquire the lock.
00030 
00031         lock(blocking = True)
00032 
00033         'blocking': if blocking is True, then this only returns when the lock is acquired.
00034         If it is False, then it returns immediately with False if the lock is not available,
00035         or with True if it could be acquired.
00036         """
00037         return self.llock.acquire(blocking)

Here is the caller graph for this function:

Definition at line 317 of file transdbstore.py.

00317 
00318     def restart(self):
00319         try:
00320             self.dbenv.close(0)
00321             log.msg(arc.INFO, "db environment closed")
00322             self.dbenv = db.DBEnv(0)
00323             self.__opendbenv()
00324         except db.DBError, msg:
00325             log.msg()
00326             log.msg(arc.ERROR, "error closing environment")
00327             #raise db.DBError, msg

Here is the call graph for this function:

def arcom.store.transdbstore.TransDBStore.set (   self,
  ID,
  object 
)
Stores an object with the given ID..

set(ID, object)

'ID' is the ID of the object
'object' is the object itself
If there is already an object with this ID it will be overwritten completely.
If deadlock is caught, retry DeadlockRetries times

Definition at line 257 of file transdbstore.py.

00257 
00258     def set(self, ID, object):
00259         """ Stores an object with the given ID..
00260 
00261         set(ID, object)
00262 
00263         'ID' is the ID of the object
00264         'object' is the object itself
00265         If there is already an object with this ID it will be overwritten completely.
00266         If deadlock is caught, retry DeadlockRetries times
00267         """
00268         if not ID:
00269             raise Exception, 'ID is empty'
00270 
00271         if not self.getDBReady():
00272             return
00273         self.dbp = self.__opendb(self.dbp)
00274         retry = True
00275         retry_count = 0
00276 
00277         # try deadlock_retries times if receiving catching DBLockDeadlockError
00278         while retry:
00279             try:
00280                 if object == None:
00281                     self.dbp.delete(ID)
00282                 else:
00283                     # note that object needs to be converted to string
00284                     # using cPickle.dumps for converting
00285                     self.dbp.put(ID, cPickle.dumps(object, cPickle.HIGHEST_PROTOCOL))
00286                 retry = False
00287             except db.DBLockDeadlockError:
00288                 log.msg(arc.INFO, "Got deadlock error")
00289                 # need to close transaction handle as well
00290                 time.sleep(0.2)
00291                 if retry_count < self.deadlock_retries:
00292                     log.msg(arc.VERBOSE, "got DBLockDeadlockError")
00293                     retry_count += 1
00294                     log.msg(arc.VERBOSE, "retrying transaction", retry_count)
00295                     retry = True
00296                 else:
00297                     log.msg(arc.VERBOSE, "Deadlock exception, giving up...")
00298                     retry = False
00299             except db.DBRepHandleDeadError:
00300                 log.msg(arc.INFO, "Got rep_dead_handle error")            
00301                 #log.msg()
00302                 self.__err()
00303                 self.dbp = None
00304                 raise
00305             except db.DBAccessError:
00306                 log.msg(arc.WARNING,"Read-only db. I'm not a master.")
00307                 raise
00308             except db.DBNotFoundError:
00309                 log.msg(arc.WARNING, "cannot delete non-existing entries")
00310                 retry = False
00311             except:
00312                 self.__del__()
00313                 log.msg()
00314                 log.msg(arc.ERROR, "Error setting %s"%ID)
00315                 retry = False
00316                 sys.exit(1)

Here is the call graph for this function:

Here is the caller graph for this function:

Definition at line 328 of file transdbstore.py.

00328 
00329     def terminate(self):
00330         try:
00331             self.dbenv.close(0)
00332             log.msg(arc.INFO, "db environment closed")
00333         except db.DBError, msg:
00334             log.msg()
00335             log.msg(arc.ERROR, "error closing environment")
00336             #raise db.DBError, msg

Here is the caller graph for this function:

def arcom.store.basestore.BaseStore.unlock (   self) [inherited]
Release the lock.

unlock()

Reimplemented in storage.ahash.replicatedahash.ReplicationStore.

Definition at line 38 of file basestore.py.

00038 
00039     def unlock(self):
00040         """ Release the lock.
00041 
00042         unlock()
00043         """
00044         self.llock.release()
        

Here is the caller graph for this function:


Member Data Documentation

Reimplemented in storage.ahash.replicatedahash.ReplicationStore.

Definition at line 80 of file transdbstore.py.

Definition at line 46 of file transdbstore.py.

Definition at line 18 of file basestore.py.

Definition at line 47 of file transdbstore.py.

Definition at line 66 of file transdbstore.py.

Definition at line 44 of file transdbstore.py.

Definition at line 75 of file transdbstore.py.

Definition at line 25 of file basestore.py.

Definition at line 20 of file basestore.py.

Definition at line 87 of file transdbstore.py.


The documentation for this class was generated from the following file: