Back to index

nordugrid-arc-nox  1.1.0~rc6
Public Member Functions | Public Attributes | Static Public Attributes
storage.shepherd.hardlinkingbackend.HardlinkingBackend Class Reference
Inheritance diagram for storage.shepherd.hardlinkingbackend.HardlinkingBackend:
Inheritance graph
[legend]

List of all members.

Public Member Functions

def __init__
def checkingThread
def copyTo
def copyFrom
def prepareToGet
def prepareToPut
def remove
def list
def getAvailableSpace
def generateLocalID
def matchProtocols
def checksum

Public Attributes

 ssl_config
 file_arrived
 datadir
 transferdir
 turlprefix
 idstore

Static Public Attributes

list public_request_names = []
list supported_protocols = ['http']

Detailed Description

Superclass for http protocols using hardlinking scheme.
Don't use this class directly. Use subclasses instead.

Definition at line 13 of file hardlinkingbackend.py.


Constructor & Destructor Documentation

def storage.shepherd.hardlinkingbackend.HardlinkingBackend.__init__ (   self,
  backendcfg,
  ns_uri,
  file_arrived,
  ssl_config 
)
docstring for __init__

Reimplemented in storage.shepherd.hardlinkingbackend.ApacheBackend.

Definition at line 21 of file hardlinkingbackend.py.

00021 
00022     def __init__(self, backendcfg, ns_uri, file_arrived, ssl_config):
00023         """docstring for __init__"""
00024         self.ssl_config = ssl_config
00025         self.file_arrived = file_arrived
00026         #self.ns = arc.NS('she', ns_uri)
00027         self.datadir = str(backendcfg.Get('StoreDir'))
00028         if not self.datadir:
00029             raise Exception, 'No StoreDir given in the Shepherd backend config!'
00030         self.transferdir = str(backendcfg.Get('TransferDir'))
00031         self.turlprefix = str(backendcfg.Get('TURLPrefix'))
00032         if not self.turlprefix:
00033             raise Exception, 'No TURLPrefix is given in the Shepherd backend config!'
00034         if not self.turlprefix[-1] == '/':
00035             self.turlprefix = self.turlprefix + '/'
00036         if not os.path.exists(self.datadir):
00037             os.mkdir(self.datadir)
00038         if not os.path.exists(self.transferdir):
00039             os.mkdir(self.transferdir)
00040         else:
00041             for filename in os.listdir(self.transferdir):
00042                 if not ".py" in filename: # need this for apache backend
00043                     os.remove(os.path.join(self.transferdir, filename))
00044         self.idstore = {}
00045         threading.Thread(target = self.checkingThread, args = [5]).start()        
    

Member Function Documentation

docstring for checkingThread

Definition at line 46 of file hardlinkingbackend.py.

00046 
00047     def checkingThread(self, period):
00048         """docstring for checkingThread"""
00049         while True:
00050             try:
00051                 time.sleep(period)
00052                 for localID, referenceID in self.idstore.items():
00053                     filename = os.path.join(self.datadir, localID)
00054                     try:
00055                         nlink = os.stat(filename)[stat.ST_NLINK]
00056                     except:
00057                         # if the file does not exist, maybe it's already removed
00058                         del self.idstore[localID]
00059                         nlink = 0
00060                     log.msg(arc.VERBOSE, 'checking', localID, referenceID, nlink)
00061                     if nlink == 1:
00062                         # if there is just one link for this file, it is already removed from the transfer dir
00063                         self.file_arrived(referenceID)
00064                         del self.idstore[localID]
00065             except:
00066                 log.msg()
00067     
00068         

def storage.shepherd.hardlinkingbackend.HardlinkingBackend.checksum (   self,
  localID,
  checksumType 
)

Definition at line 135 of file hardlinkingbackend.py.

00135 
00136     def checksum(self, localID, checksumType):
00137         return create_checksum(file(os.path.join(self.datadir, localID), 'rb'), checksumType)
00138 

Here is the call graph for this function:

def storage.shepherd.hardlinkingbackend.HardlinkingBackend.copyFrom (   self,
  localID,
  turl,
  protocol 
)

Definition at line 75 of file hardlinkingbackend.py.

00075 
00076     def copyFrom(self, localID, turl, protocol):
00077         # TODO: download to a separate file, and if checksum OK, then copy the file 
00078         f = file(os.path.join(self.datadir, localID), 'wb')
00079         log.msg(arc.VERBOSE, self.turlprefix, 'Downloading file from', turl)
00080         download_from_turl(turl, protocol, f, ssl_config = self.ssl_config)
00081         f.close()

Here is the call graph for this function:

def storage.shepherd.hardlinkingbackend.HardlinkingBackend.copyTo (   self,
  localID,
  turl,
  protocol 
)

Definition at line 69 of file hardlinkingbackend.py.

00069 
00070     def copyTo(self, localID, turl, protocol):
00071         f = file(os.path.join(self.datadir, localID),'rb')
00072         log.msg(arc.VERBOSE, self.turlprefix, 'Uploading file to', turl)
00073         upload_to_turl(turl, protocol, f, ssl_config = self.ssl_config)
00074         f.close()
    

Here is the call graph for this function:

Definition at line 129 of file hardlinkingbackend.py.

00129 
00130     def generateLocalID(self):
00131         return arc.UUID()

Definition at line 122 of file hardlinkingbackend.py.

00122 
00123     def getAvailableSpace(self):
00124         try:
00125             f = os.statvfs(self.datadir)
00126             return f.f_frsize * f.f_bavail
00127         except:
00128             return None

Definition at line 119 of file hardlinkingbackend.py.

00119 
00120     def list(self):
00121         return os.listdir(os.datadir)

Definition at line 132 of file hardlinkingbackend.py.

00132 
00133     def matchProtocols(self, protocols):
00134         return [protocol for protocol in protocols if protocol in self.supported_protocols]

def storage.shepherd.hardlinkingbackend.HardlinkingBackend.prepareToGet (   self,
  referenceID,
  localID,
  protocol 
)

Reimplemented in storage.shepherd.hardlinkingbackend.ApacheBackend.

Definition at line 82 of file hardlinkingbackend.py.

00082 
00083     def prepareToGet(self, referenceID, localID, protocol):
00084         if protocol not in self.supported_protocols:
00085             raise Exception, 'Unsupported protocol: ' + protocol
00086         turl_id = arc.UUID()
00087         try:
00088             filepath = os.path.join(self.datadir, localID)
00089             # set it to readonly
00090             os.chmod(filepath, 0400)
00091             os.link(filepath, os.path.join(self.transferdir, turl_id))
00092             log.msg(arc.VERBOSE, self.turlprefix, '++', self.idstore)
00093             turl = self.turlprefix + turl_id
00094             return turl
00095         except:
00096             return None

def storage.shepherd.hardlinkingbackend.HardlinkingBackend.prepareToPut (   self,
  referenceID,
  localID,
  protocol 
)

Reimplemented in storage.shepherd.hardlinkingbackend.ApacheBackend.

Definition at line 97 of file hardlinkingbackend.py.

00097 
00098     def prepareToPut(self, referenceID, localID, protocol):
00099         if protocol not in self.supported_protocols:
00100             raise Exception, 'Unsupported protocol: ' + protocol
00101         turl_id = arc.UUID()
00102         datapath = os.path.join(self.datadir, localID)
00103         f = file(datapath, 'wb')
00104         f.close()
00105         os.chmod(datapath, 0600)
00106         os.link(datapath, os.path.join(self.transferdir, turl_id))
00107         self.idstore[localID] = referenceID
00108         log.msg(arc.VERBOSE, self.turlprefix, '++', self.idstore)
00109         turl = self.turlprefix + turl_id
00110         return turl

Definition at line 111 of file hardlinkingbackend.py.

00111 
00112     def remove(self, localID):
00113         try:
00114             fn = os.path.join(self.datadir, localID)
00115             os.remove(fn)
00116         except:
00117             return 'failed: ' + traceback.format_exc()
00118         return 'removed'


Member Data Documentation

Definition at line 26 of file hardlinkingbackend.py.

Definition at line 24 of file hardlinkingbackend.py.

Definition at line 43 of file hardlinkingbackend.py.

Definition at line 18 of file hardlinkingbackend.py.

Definition at line 23 of file hardlinkingbackend.py.

Definition at line 19 of file hardlinkingbackend.py.

Definition at line 29 of file hardlinkingbackend.py.

Definition at line 30 of file hardlinkingbackend.py.


The documentation for this class was generated from the following file: