Back to index

nordugrid-arc-nox  1.1.0~rc6
Public Member Functions | Public Attributes
storage.ahash.replicatedahash.ReplicationManager Class Reference

List of all members.

Public Member Functions

def __init__
def isMaster
def heartbeatThread
def getRole
def setRole
def getSiteList
def start
def electionThread
def startElection
def beginRole
def send
def repSend
def sendNewSiteMsg
def sendHeartbeatMsg
def sendElectionMsg
def sendNewMasterMsg
def processMessage
def event_callback

Public Attributes

 role
 elected
 stop_electing
 ahash_send
 locker
 hostMap
 my_replica
 eid
 url
 masterID
 dbenv
 dbReady
 pool
 semapool
 election_thread
 comm_ready
 heartbeat_period
 check_heartbeat
 master_timestamp

Detailed Description

class managing replicas, elections and message handling

Definition at line 348 of file replicatedahash.py.


Constructor & Destructor Documentation

def storage.ahash.replicatedahash.ReplicationManager.__init__ (   self,
  ahash_send,
  dbenv,
  my_replica,
  other_replicas,
  dbReady 
)

Definition at line 353 of file replicatedahash.py.

00353 
00354     def __init__(self, ahash_send, dbenv, my_replica, other_replicas, dbReady):
00355         
00356         # no master is found yet
00357         self.role = db.DB_EID_INVALID
00358         self.elected = False
00359         self.stop_electing = False
00360         self.ahash_send = ahash_send
00361         self.locker = ReadWriteLock()
00362         global hostMap
00363         self.hostMap = hostMap
00364         self.locker.acquire_write()
00365         self.hostMap[my_replica['id']] = my_replica
00366         for replica in other_replicas:
00367             self.hostMap[replica['id']] = replica
00368         self.locker.release_write()
00369         self.my_replica = my_replica
00370         self.eid = my_replica['id']
00371         self.url = my_replica['url']
00372         # assume no master yet
00373         self.masterID = db.DB_EID_INVALID
00374         self.dbenv = dbenv
00375         self.dbReady = dbReady
00376         #self.dbenv.set_verbose(db.DB_VERB_REPLICATION, True)
00377         # tell dbenv to uset our event callback function
00378         # to handle various events
00379         self.dbenv.set_event_notify(self.event_callback)
00380         self.pool = None
00381         self.semapool = threading.BoundedSemaphore(128)
00382         self.election_thread = threading.Thread(target=self.electionThread, args=[])
00383         self.comm_ready = False
00384         self.heartbeat_period = 10
00385         threading.Thread(target=self.heartbeatThread, 
00386                          args=[self.heartbeat_period]).start()
00387         self.check_heartbeat = False
00388         self.master_timestamp = 0
        

Member Function Documentation

Definition at line 481 of file replicatedahash.py.

00481 
00482     def beginRole(self, role):
00483         try:
00484             # check if role has changed, to avoid too much write blocking
00485             if self.getRole() != role:
00486                 log.msg(arc.INFO, "new role")
00487                 self.setRole(role)
00488             self.dbenv.rep_start(role==db.DB_REP_MASTER and 
00489                                  db.DB_REP_MASTER or db.DB_REP_CLIENT)
00490         except:
00491             log.msg(arc.ERROR, "Couldn't begin role")
00492             log.msg()
00493         return

Here is the call graph for this function:

Here is the caller graph for this function:

Definition at line 442 of file replicatedahash.py.

00442 
00443     def electionThread(self):
00444         role = db.DB_EID_INVALID
00445         try:
00446             log.msg(arc.VERBOSE, "entered election thread")
00447             # send a message to discover if clients are offline
00448             self.sendElectionMsg()
00449             self.locker.acquire_read()
00450             num_reps = len([id for id,rep in self.hostMap.items() if rep["status"] != "offline"])
00451             self.locker.release_read()
00452             votes = num_reps/2 + 1
00453             if votes < 2:
00454                 votes = 2
00455             log.msg(arc.VERBOSE, "%s: my role is" % self.url, role)
00456             self.dbenv.rep_elect(num_reps, votes)
00457             # wait one second for election results
00458             time.sleep(1)
00459             role = self.getRole()
00460             log.msg(arc.VERBOSE, "%s: my role is now" % self.url, role)
00461             if self.elected:
00462                 self.elected = False
00463                 self.dbenv.rep_start(db.DB_REP_MASTER)                    
00464         except:
00465             log.msg(arc.ERROR, "Couldn't run election")
00466             log.msg(arc.VERBOSE, "num_reps is %(nr)d, votes is %(v)d, hostMap is %(hm)s" % {'nr':num_reps, 'v':votes, 'hm':str(self.hostMap)})
00467             time.sleep(2)
00468         log.msg(arc.VERBOSE, self.url, "tried election with %d replicas" % num_reps)
            

Here is the call graph for this function:

Here is the caller graph for this function:

def storage.ahash.replicatedahash.ReplicationManager.event_callback (   self,
  dbenv,
  which,
  info 
)
Callback function used to determine whether the local environment is a 
replica or a master. This is called by the replication framework
when the local replication environment changes state
app = dbenv.get_private()

Definition at line 759 of file replicatedahash.py.

00759 
00760     def event_callback(self, dbenv, which, info):
00761         """
00762         Callback function used to determine whether the local environment is a 
00763         replica or a master. This is called by the replication framework
00764         when the local replication environment changes state
00765         app = dbenv.get_private()
00766         """
00767 
00768         log.msg("entering event_callback")
00769         info = None
00770         try:
00771             if which == db.DB_EVENT_REP_MASTER:
00772                 log.msg(arc.VERBOSE, "I am now a master")
00773                 log.msg(arc.VERBOSE, "received DB_EVENT_REP_MASTER")
00774                 self.setRole(db.DB_REP_MASTER)
00775                 self.dbReady(True)
00776                 # use threaded send to avoid blocking
00777                 #threading.Thread(target=self.sendNewMasterMsg, args=[]).start()
00778                 self.pool.queueTask(self.sendNewMasterMsg)
00779             elif which == db.DB_EVENT_REP_CLIENT:
00780                 log.msg(arc.VERBOSE, "I am now a client")
00781                 self.setRole(db.DB_REP_CLIENT)
00782                 self.dbReady(True)
00783             elif which == db.DB_EVENT_REP_STARTUPDONE:
00784                 log.msg(arc.VERBOSE, ("Replication startup done",which,info))
00785             elif which == db.DB_EVENT_REP_PERM_FAILED:
00786                 log.msg(arc.VERBOSE, "Getting permission failed")
00787             elif which == db.DB_EVENT_WRITE_FAILED:
00788                 log.msg(arc.VERBOSE, "Write failed")
00789             elif which == db.DB_EVENT_REP_NEWMASTER:
00790                 log.msg(arc.VERBOSE, "New master elected")
00791                 # give master 5 seconds to celebrate victory
00792                 time.sleep(5)
00793                 self.check_heartbeat = True
00794             elif which == db.DB_EVENT_REP_ELECTED:
00795                 log.msg(arc.VERBOSE, "I won the election: I am the MASTER")
00796                 self.elected = True
00797                 # use threaded send to avoid blocking
00798                 #threading.Thread(target=self.sendNewMasterMsg, args=[]).start()
00799                 self.pool.queueTask(self.sendNewMasterMsg)
00800             elif which == db.DB_EVENT_PANIC:
00801                 log.msg(arc.ERROR, "Oops! Internal DB panic!")
00802                 raise db.DBRunRecoveryError, "Please run recovery."
00803         except db.DBRunRecoveryError:
00804             sys.exit(1)      
00805         except:
00806             log.msg()

Here is the call graph for this function:

Definition at line 415 of file replicatedahash.py.

00415 
00416     def getRole(self):
00417         role = self.role
00418         return role

Here is the caller graph for this function:

Definition at line 422 of file replicatedahash.py.

00422 
00423     def getSiteList(self):
00424         self.locker.acquire_read()
00425         site_list = copy.deepcopy(self.hostMap)
00426         self.locker.release_read()
00427         return site_list
    
Thread for sending heartbeat messages
Heartbeats are only sendt when master is elected and 
running (i.e., when check_heartbeat is true)
Only clients sends heartbeats
If heartbeat is not answered, re-election will be initiated in send method

Definition at line 393 of file replicatedahash.py.

00393 
00394     def heartbeatThread(self, period):
00395         """
00396         Thread for sending heartbeat messages
00397         Heartbeats are only sendt when master is elected and 
00398         running (i.e., when check_heartbeat is true)
00399         Only clients sends heartbeats
00400         If heartbeat is not answered, re-election will be initiated in send method
00401         """
00402         time.sleep(10)
00403         
00404         # todo: implement list of time_since_heartbeat so the master can mark clients 
00405         # as offline when low write activity
00406         while True:
00407             if self.role == db.DB_REP_CLIENT:
00408                 if self.masterID != db.DB_EID_INVALID:
00409                     # use threaded send to avoid blocking
00410                     #threading.Thread(target=self.sendHeartbeatMsg, args=[]).start()
00411                     self.pool.queueTask(self.sendHeartbeatMsg)
00412             # add some randomness to avoid bugging the master too much
00413             log.msg(arc.VERBOSE, ("heartbeat", self.masterID, self.role))
00414             time.sleep(period)

Here is the call graph for this function:

Definition at line 389 of file replicatedahash.py.

00389 
00390     def isMaster(self):
00391         is_master = self.role == db.DB_REP_MASTER
00392         return is_master

Here is the caller graph for this function:

def storage.ahash.replicatedahash.ReplicationManager.processMessage (   self,
  control,
  record,
  eid,
  retlsn,
  sender,
  msgID 
)
Function to process incoming messages, forwarding 
them to self.dbenv.rep_process_message()

Definition at line 638 of file replicatedahash.py.

00638 
00639     def processMessage(self, control, record, eid, retlsn, sender, msgID):
00640         """
00641         Function to process incoming messages, forwarding 
00642         them to self.dbenv.rep_process_message()
00643         """
00644         log.msg(arc.VERBOSE, "entering processMessage from ", sender)
00645 
00646         self.locker.acquire_read()
00647         urls = [rep['url'] for id,rep in self.hostMap.items() if rep['status']=='online']
00648         self.locker.release_read()
00649         if sender['url'] == self.url:
00650             log.msg(arc.ERROR, "received message from myself!")
00651             return "failed" 
00652         if not sender['url'] in urls:
00653             log.msg(arc.VERBOSE, "received from new sender or sender back online")
00654             really_new = False
00655             try:
00656                 # check if we know this one
00657                 newid = [id for id,rep in self.hostMap.items() if rep['url']==sender['url']][0]
00658             except:
00659                 # nope, never heard about it
00660                 newid = len(self.hostMap)+1
00661                 really_new = True
00662             sender['id'] = newid
00663             self.locker.acquire_write()
00664             self.hostMap[newid] = sender
00665             self.locker.release_write()
00666             # return hostMap to sender
00667             if really_new:
00668                 # use threaded send to avoid blocking
00669                 #threading.Thread(target=self.sendNewSiteMsg, args=[sender]).start()
00670                 self.pool.queueTask(self.sendNewSiteMsg, args=sender)
00671         if msgID == MASTER_MESSAGE:
00672             # sender is master, find local id for sender and set as masterID
00673             log.msg(arc.VERBOSE, "received master id")
00674             self.masterID = [id for id,rep in self.hostMap.items() if rep['url']==sender['url']][0]
00675             self.master_timestamp = time.time()
00676             return "processed"
00677         if msgID == HEARTBEAT_MESSAGE:
00678             log.msg(arc.VERBOSE, "received HEARTBEAT_MESSAGE")
00679             return "processed"
00680         if msgID == ELECTION_MESSAGE:
00681             log.msg(arc.VERBOSE, "received ELECTION_MESSAGE")
00682             return "processed"
00683         if msgID == NEWSITE_MESSAGE:
00684             # if unknown changes in record, update hostMap
00685             log.msg(arc.VERBOSE, "received NEWSITE_MESSAGE")
00686             for replica in record.values():
00687                 if  not replica['url'] in urls:
00688                     really_new = False
00689                     try:
00690                         # check if we know this one
00691                         newid = [id for id,rep in self.hostMap.items() if rep['url']==replica['url']][0]
00692                     except:
00693                         # nope, never heard about it
00694                         newid = len(self.hostMap)+1
00695                         really_new = True
00696                     self.locker.acquire_write()
00697                     # really new sender, appending to host map
00698                     replica['id'] = newid
00699                     self.hostMap[newid] = replica
00700                     self.locker.release_write()
00701                     # say hello to my new friend
00702                     if really_new:
00703                         # use threaded send to avoid blocking
00704                         #threading.Thread(target=self.sendNewSiteMsg, args=[replica]).start()
00705                         self.pool.queueTask(self.sendNewSiteMsg, args=replica)
00706             return "processed"
00707         try:
00708             eid = [id for id,rep in self.hostMap.items() if rep['url']==sender['url']][0]
00709         except:
00710             return "notfound"
00711         if eid == self.masterID:
00712             self.master_timestamp = time.time()
00713         try:
00714             log.msg(arc.VERBOSE, "processing message from %d"%eid)
00715             res, retlsn = self.dbenv.rep_process_message(control, record, eid)
00716         except db.DBNotFoundError:
00717             log.msg(arc.ERROR, "Got dbnotfound")
00718             log.msg(arc.ERROR, (control, record, eid, retlsn, sender, msgID))
00719             #log.msg()
00720             return "failed"
00721         except:
00722             log.msg(arc.ERROR, "couldn't process message")
00723             log.msg(arc.ERROR, (control, record, eid, retlsn, sender, msgID))
00724             log.msg()
00725             return "failed"
00726         
00727         if res == db.DB_REP_NEWSITE:
00728             log.msg(arc.VERBOSE, "received DB_REP_NEWSITE from %s"%str(sender))
00729             if self.isMaster():
00730                 # use threaded send to avoid blocking
00731                 #threading.Thread(target=self.sendNewMasterMsg, args=[eid]).start()
00732                 self.pool.queueTask(self.sendNewMasterMsg, args=eid)
00733         elif res == db.DB_REP_HOLDELECTION:
00734             log.msg(arc.VERBOSE, "received DB_REP_HOLDELECTION")
00735             self.beginRole(db.DB_EID_INVALID)
00736             self.masterID = db.DB_EID_INVALID
00737             self.startElection()
00738         elif res == db.DB_REP_ISPERM:
00739             log.msg(arc.VERBOSE, "REP_ISPERM returned for LSN %s"%str(retlsn))
00740             self.dbReady(True)
00741             self.setRole(db.DB_REP_CLIENT)
00742         elif res == db.DB_REP_NOTPERM:
00743             log.msg(arc.VERBOSE, "REP_NOTPERM returned for LSN %s"%str(retlsn))
00744         elif res == db.DB_REP_DUPMASTER:
00745             log.msg(arc.VERBOSE, "REP_DUPMASTER received, starting new election")
00746             # yield to revolution, switch to client
00747             self.beginRole(db.DB_EID_INVALID)
00748             self.masterID = db.DB_EID_INVALID
00749             self.startElection()
00750         elif res == db.DB_REP_IGNORE:
00751             log.msg(arc.VERBOSE, "REP_IGNORE received")
00752             log.msg(arc.VERBOSE, (control, record, eid, retlsn, sender, msgID))
00753         elif res == db.DB_REP_JOIN_FAILURE:
00754             log.msg(arc.ERROR, "JOIN_FAILURE received")
00755         else:
00756             log.msg(arc.VERBOSE, "unknown return code %s"%str(res))        
00757 
00758         return "processed"

Here is the call graph for this function:

def storage.ahash.replicatedahash.ReplicationManager.repSend (   self,
  env,
  control,
  record,
  lsn,
  eid,
  flags 
)
callback function for dbenv transport

Definition at line 573 of file replicatedahash.py.

00573 
00574     def repSend(self, env, control, record, lsn, eid, flags):
00575         """
00576         callback function for dbenv transport
00577         """
00578         log.msg(arc.VERBOSE, "entering repSend")
00579         if flags & db.DB_REP_PERMANENT and lsn != None:
00580             self.semapool.acquire()
00581             res = self.send(env, control, record, lsn, eid, flags, REP_MESSAGE)
00582             self.semapool.release()
00583             return res
00584         else:
00585             #threading.Thread(target=self.send, args=[env, control, record, lsn, eid, flags, REP_MESSAGE]).start()
00586             self.pool.queueTask(self.send, args=[env, control, record, lsn, eid, flags, REP_MESSAGE])
00587             return 0

Here is the call graph for this function:

Here is the caller graph for this function:

def storage.ahash.replicatedahash.ReplicationManager.send (   self,
  env,
  control,
  record,
  lsn,
  eid,
  flags,
  msgID 
)
callback function for dbenv transport
If no reply within 10 seconds, return 1

Definition at line 494 of file replicatedahash.py.

00494 
00495     def send(self, env, control, record, lsn, eid, flags, msgID):
00496         """
00497         callback function for dbenv transport
00498         If no reply within 10 seconds, return 1
00499         """
00500         # wrap control, record, lsn, eid, flags and sender into dict
00501         # note: could be inefficient to send sender info for every message
00502         # if bandwidth and latency is low
00503         log.msg(arc.VERBOSE, "entering send")
00504         sender = self.my_replica
00505         msg = {'control':control,
00506                'record':record,
00507                'lsn':lsn,
00508                'eid':eid,
00509                'msgID':msgID,
00510                'sender':sender}
00511         
00512         
00513         retval = 1
00514         if msgID == NEWSITE_MESSAGE:
00515             eids = [control['id']]
00516             msg['control'] = None
00517         elif msgID == HEARTBEAT_MESSAGE:
00518             eids = [self.masterID]
00519         elif eid == db.DB_EID_BROADCAST:
00520             # send to all
00521             eids = [id for id,rep in self.hostMap.items()]    
00522         else:
00523             eids = [eid]
00524             if not self.hostMap.has_key(eid):
00525                 return retval
00526         for id in eids:
00527             if id == self.eid:
00528                 continue
00529             try:
00530                 msg['eid'] = id
00531                 resp = ["waiting"]
00532                 url = self.hostMap[id]['url']
00533                 time_since_send = time.time()
00534                 resp[0] = self.ahash_send(url, msg)
00535                 if str(resp[0]) == "processed":
00536                     # if at least one msg is sent, we're happy
00537                     retval = 0
00538                     # received message so sender cannot be offline
00539                     if self.hostMap[id]['status'] == 'offline':
00540                         self.locker.acquire_write()
00541                         self.hostMap[id]['status'] = 'online'
00542                         self.locker.release_write()
00543             except:
00544                 # assume url is disconnected
00545                 log.msg(arc.WARNING, "failed to send to", id, "of", str(eids))
00546                 self.locker.acquire_write()
00547                 
00548                 if id == self.masterID:
00549                     # timeout if I've heard nothing from the master
00550                     # or master doesn't reply
00551                     timeout = time.time() - self.master_timestamp > self.heartbeat_period*2\
00552                               or time.time()-time_since_send < 55
00553                     if timeout:
00554                         log.msg(arc.INFO, "Master is offline, starting re-election")
00555                         # in case more threads misses the master
00556                         if self.masterID != db.DB_EID_INVALID:
00557                             self.locker.release_write()
00558                             self.beginRole(db.DB_EID_INVALID)
00559                             self.masterID = db.DB_EID_INVALID
00560                             self.locker.acquire_write()
00561                             self.startElection()
00562                         # only set master to offline if it has really timed out
00563                         self.hostMap[id]['status'] = "offline"
00564                         if msgID == NEWSITE_MESSAGE:
00565                             record['status'] = "offline"
00566                 # cannot have less than 2 online sites
00567                 elif len([id2 for id2,rep in self.hostMap.items() if rep['status']=="online"]) > 2:
00568                     self.hostMap[id]['status'] = "offline"
00569                     if msgID == NEWSITE_MESSAGE:
00570                         record['status'] = "offline"
00571                 self.locker.release_write()
00572         return retval
    

Here is the call graph for this function:

Here is the caller graph for this function:

def storage.ahash.replicatedahash.ReplicationManager.sendElectionMsg (   self,
  eid = db.DB_EID_BROADCAST 
)
If elected, broadcast master id to all clients

Definition at line 618 of file replicatedahash.py.

00618 
00619     def sendElectionMsg(self, eid=db.DB_EID_BROADCAST):
00620         """
00621         If elected, broadcast master id to all clients
00622         """
00623         log.msg(arc.VERBOSE, "entering sendNewMasterMsg")
00624         self.semapool.acquire()
00625         ret = self.send(None, None, None, None, eid, None, ELECTION_MESSAGE)
00626         self.semapool.release()
00627         return ret
    

Here is the call graph for this function:

Here is the caller graph for this function:

if new site is discovered sendHeartbeatMsg will send 
the hostMap to the new site

Definition at line 607 of file replicatedahash.py.

00607 
00608     def sendHeartbeatMsg(self):
00609         """
00610         if new site is discovered sendHeartbeatMsg will send 
00611         the hostMap to the new site
00612         """
00613         log.msg(arc.VERBOSE, "entering sendHeartbeatMsg")
00614         self.semapool.acquire()
00615         ret = self.send(None, None, None, None, None, None, HEARTBEAT_MESSAGE)
00616         self.semapool.release()
00617         return ret

Here is the call graph for this function:

Here is the caller graph for this function:

def storage.ahash.replicatedahash.ReplicationManager.sendNewMasterMsg (   self,
  eid = db.DB_EID_BROADCAST 
)
If elected, broadcast master id to all clients

Definition at line 628 of file replicatedahash.py.

00628 
00629     def sendNewMasterMsg(self, eid=db.DB_EID_BROADCAST):
00630         """
00631         If elected, broadcast master id to all clients
00632         """
00633         log.msg(arc.VERBOSE, "entering sendNewMasterMsg")
00634         self.semapool.acquire()
00635         ret = self.send(None, None, None, None, eid, None, MASTER_MESSAGE)
00636         self.semapool.release()
00637         return ret
    

Here is the call graph for this function:

Here is the caller graph for this function:

if new site is discovered sendNewSiteMsg will send 
the hostMap to the new site

Definition at line 588 of file replicatedahash.py.

00588 
00589     def sendNewSiteMsg(self, new_replica):
00590         """
00591         if new site is discovered sendNewSiteMsg will send 
00592         the hostMap to the new site
00593         """
00594         log.msg(arc.VERBOSE, "entering sendNewSiteMsg")
00595         self.locker.acquire_read()
00596         site_list = copy.deepcopy(self.hostMap)
00597         self.locker.release_read()
00598         ret = 1
00599         while ret:
00600             self.semapool.acquire()
00601             ret = self.send(None, new_replica, site_list, None, None, None, NEWSITE_MESSAGE)
00602             self.semapool.release()
00603             if new_replica['status'] == 'offline':
00604                 break
00605             time.sleep(30)
00606         return ret
    

Here is the call graph for this function:

Here is the caller graph for this function:

Definition at line 419 of file replicatedahash.py.

00419 
00420     def setRole(self, role):
00421         self.role = role
    

Here is the caller graph for this function:

def storage.ahash.replicatedahash.ReplicationManager.start (   self,
  nthreads,
  flags 
)

Definition at line 428 of file replicatedahash.py.

00428 
00429     def start(self, nthreads, flags):
00430         log.msg(arc.VERBOSE, "entering start")
00431         self.pool = ThreadPool(nthreads)
00432         while not self.comm_ready:
00433             time.sleep(2)
00434         try:
00435             self.dbenv.rep_set_transport(self.eid, self.repSend)
00436             self.dbenv.rep_start(db.DB_REP_CLIENT)
00437             log.msg(arc.VERBOSE, ("rep_start called with REP_CLIENT", self.hostMap))
00438             self.startElection()
00439         except:
00440             log.msg(arc.ERROR, "Couldn't start replication framework")
00441             log.msg()

Here is the call graph for this function:

Definition at line 469 of file replicatedahash.py.

00469 
00470     def startElection(self):
00471         log.msg(arc.VERBOSE, "entering startElection")
00472         self.stop_electing = False
00473         self.dbReady(False)
00474         try:
00475             # try to join election_thread first to make sure only one election thread is running
00476             self.election_thread.join()
00477             self.election_thread = threading.Thread(target=self.electionThread, args=[])
00478         except:
00479             pass    
00480         self.election_thread.start()

Here is the call graph for this function:

Here is the caller graph for this function:


Member Data Documentation

Definition at line 359 of file replicatedahash.py.

Definition at line 386 of file replicatedahash.py.

Definition at line 382 of file replicatedahash.py.

Definition at line 373 of file replicatedahash.py.

Definition at line 374 of file replicatedahash.py.

Definition at line 369 of file replicatedahash.py.

Definition at line 357 of file replicatedahash.py.

Definition at line 381 of file replicatedahash.py.

Definition at line 383 of file replicatedahash.py.

Definition at line 362 of file replicatedahash.py.

Definition at line 360 of file replicatedahash.py.

Definition at line 387 of file replicatedahash.py.

Definition at line 372 of file replicatedahash.py.

Definition at line 368 of file replicatedahash.py.

Definition at line 379 of file replicatedahash.py.

Definition at line 356 of file replicatedahash.py.

Definition at line 380 of file replicatedahash.py.

Definition at line 358 of file replicatedahash.py.

Definition at line 370 of file replicatedahash.py.


The documentation for this class was generated from the following file: