Back to index

nordugrid-arc-nox  1.1.0~rc6
common.py
Go to the documentation of this file.
00001 import os, traceback
00002 
00003 # the GUID of the root collection of the global storage namespace
00004 global_root_guid = '0'
00005 # a special entity where the data about Shepherds are stored (SEStore)
00006 sestore_guid = '1'
00007 # a special entity where the list of A-Hashes is stored
00008 ahash_list_guid = '2'
00009 
00010 ALIVE = 'alive'
00011 CREATING = 'creating'
00012 STALLED = 'stalled'
00013 INVALID = 'invalid'
00014 DELETED = 'deleted'
00015 THIRDWHEEL = 'thirdwheel'
00016 OFFLINE = 'offline'
00017 
00018 common_supported_protocols = ['http', 'byteio','external']
00019 CHUNKSIZE = 2**20
00020 
00021 def upload_to_turl(turl, protocol, fobj, size = None, ssl_config = {}):
00022     """docstring for upload_to_turl"""
00023     if protocol not in common_supported_protocols:
00024         raise Exception, 'Unsupported protocol'
00025     if protocol == 'byteio':
00026         from storage.client import ByteIOClient
00027         return ByteIOClient(turl, ssl_config = ssl_config).write(fobj)
00028     elif protocol == 'external':
00029         return 
00030     elif protocol == 'http':
00031         import arc
00032         from arcom import datapoint_from_url
00033         src = datapoint_from_url(fobj.name)
00034         dst = datapoint_from_url(turl, ssl_config)
00035         mover = arc.DataMover()
00036         mover.verbose(False)
00037         mover.retry(False)
00038         status = mover.Transfer(src, dst, arc.FileCache(), arc.URLMap())
00039         return str(status)
00040 
00041 def download_from_turl(turl, protocol, fobj, ssl_config = {}):
00042     """docstring for download_from_turl"""
00043     if protocol not in common_supported_protocols:
00044         raise Exception, 'Unsupported protocol'
00045     if protocol == 'byteio':
00046         from storage.client import ByteIOClient
00047         ByteIOClient(turl, ssl_config = ssl_config).read(file = f)
00048     elif protocol == 'http':
00049         import arc
00050         from arcom import datapoint_from_url
00051         src = datapoint_from_url(turl, ssl_config)
00052         dst = datapoint_from_url(fobj.name)
00053         mover = arc.DataMover()
00054         mover.verbose(False)
00055         mover.retry(False)
00056         status = mover.Transfer(src, dst, arc.FileCache(), arc.URLMap())
00057         return str(status)
00058 
00059 def create_checksum(file, type):
00060     """ Create checksum of a file.
00061     
00062     create_checksum(file, type)
00063     
00064     file is an object with a 'read' method
00065     type is a string indicating the type of the checksum, currently only md5 is supported
00066     
00067     Returns the checksum as a string.
00068     """
00069     if type == 'md5':
00070         return _md5sum(file)
00071     else:
00072         raise Exception, 'Unsupported checksum type'
00073 
00074 try:
00075     import hashlib as md5
00076     md5.new_new = md5.new
00077     md5.new = lambda: md5.new_new('md5')
00078 except:
00079     import md5
00080 
00081 def _md5sum(file):
00082     """ Create md5 checksum of a file.
00083     
00084     _md5sum(file)
00085     
00086     file is an object with a 'read' method
00087     
00088     Returns the checksum as a string.
00089     """
00090     m = md5.new()
00091     while True:
00092         # read the file in chunks
00093         t = file.read(1024)
00094         if len(t) == 0: break # end of file
00095         # put the file content into the md5 object
00096         m.update(t)
00097     # get the checksum from the md5 object
00098     return m.hexdigest()
00099 
00100 def serialize_ids(ids):
00101     """ Serialize a list of IDs into one string.
00102     
00103     serialize_ids(ids)
00104     
00105     ids is a list of IDs, which are strings
00106     
00107     Returns one string which containes all the IDs.
00108     """
00109     return ' '.join([str(id) for id in ids])
00110 
00111 def deserialize_ids(s):
00112     """ Deserialize a list of IDs from a string.
00113     
00114     deserialize_ids(s)
00115     
00116     s is a string created by the 'serialize_ids' method
00117     
00118     Returns the list of IDs (which are all strings) which were originally serialized into the string.
00119     """
00120     return s.split()
00121 
00122 def parse_metadata(metadatalist_node):
00123     """ Return the metadata which was put into an XML representation.
00124     
00125     parse_metadata(metadatalist_node)
00126     
00127     metadatalist_node is an XMLNode which has zero or more children,
00128         and each child should have three children called 'section', 'property', 'value')
00129     
00130     The method get the string value from each tuple of section-property-value,
00131     then creates a dictionary where the (section, property) pairs are the keys,
00132     and the (value) is the value.
00133     
00134     Example:
00135     
00136     input:
00137     
00138         <metadataList>
00139             <metadata>
00140                 <section>entry</section>
00141                 <property>type</property>
00142                 <value>file</value>
00143             </metadata>
00144             <metadata>
00145                 <section>states</section>
00146                 <property>size</property>
00147                 <value>812314</value>
00148             </metadata>
00149         </metadataList>
00150     
00151     output:
00152     
00153         {('entry','type') : 'file', ('states', 'size') : '812314'}
00154     """
00155     # get the number of children
00156     metadatalist_number = metadatalist_node.Size()
00157     # initialize the list
00158     metadata = []
00159     # for each child
00160     for j in range(metadatalist_number):
00161         # get the child node
00162         metadata_node = metadatalist_node.Child(j)
00163         # get the 'section', 'property' and 'value' children's string value
00164         # and append them to the list as (('section','property'),'value')
00165         metadata.append((
00166             (str(metadata_node.Get('section')),str(metadata_node.Get('property'))),
00167                 str(metadata_node.Get('value'))))
00168     # create a dictionary from the list
00169     return dict(metadata)
00170 
00171 def create_metadata(metadata, prefix = ''):
00172     """ Create an XMLTree structure from a dictionary with metadata.
00173     
00174     create_metadata(metadata, prefix = '')
00175     
00176     metadata is a dictionary with ('section','property') as keys and 'value' as values
00177     prefix is a string for the namespace prefix of the XML tag, empty by default.
00178     
00179     Example:
00180     
00181     input:
00182     
00183         {('entry','type') : 'file', ('states', 'size') : '812314'}
00184     
00185     output:
00186     
00187         [('metadata', [('section', 'entry'), ('property', 'type'), ('value', 'file')]),
00188          ('metadata', [('section', 'states'), ('property', 'size'), ('value', '812314')])]
00189          
00190     this output can be used as an XMLTree object, and can be put into an XMLNode, which will look like this:
00191     
00192             <metadata>
00193                 <section>entry</section>
00194                 <property>type</property>
00195                 <value>file</value>
00196             </metadata>
00197             <metadata>
00198                 <section>states</section>
00199                 <property>size</property>
00200                 <value>812314</value>
00201             </metadata>
00202     """
00203     # if it is an empty dictionary, just return an empty list
00204     if not metadata:
00205         return []
00206     # if a prefix is specified
00207     if prefix:
00208         # for each item in the metadata list get the section, property and value and put it into XMLTree form
00209         # the 'metadata', 'section', 'property' and 'value' strings are prefixed with the given prefix and a ':'
00210         return [
00211             (prefix + ':metadata', [ 
00212                 (prefix + ':section', section),
00213                 (prefix + ':property', property),
00214                 (prefix + ':value', value)
00215             ]) for ((section, property), value) in metadata.items()
00216         ]
00217     else:
00218         # if no prefix is specified do not put a colon before the names
00219         return [
00220             ('metadata', [ 
00221                 ('section', section),
00222                 ('property', property),
00223                 ('value', value)
00224             ]) for ((section, property), value) in metadata.items()
00225         ]
00226 
00227 def remove_trailing_slash(LN):
00228     """ Remove the trailing slash of a Logical Name.
00229     
00230     remove_trailing_slash(LN)
00231     
00232     This method checks if the LN ends with a '/', and if it does, cut the trailing slash, and returns the LN.
00233     """
00234     while LN.endswith('/'):
00235         LN = LN[:-1]
00236     return LN
00237 
00238 def splitLN(LN):
00239     """  Split a Logical Name to a 3-tuple: GUID, dirname, basename.
00240     
00241     splitLN(LN)
00242     
00243     The part before the first slash is the GUID, even if it is empty.
00244     The part after the last slash is the basename, could be empty.
00245     Between them is the dirname, could be empty or could contain several more slashes.
00246     
00247     Examples
00248 
00249         splitLN('/dir') returns ('', '', 'dir')
00250         splitLN('guid/dir') returns ('guid', '', 'dir')
00251         splitLN('guid/dir/file') returns ('guid', 'dir', 'file')
00252         splitLN('guid/dir/dir2/file') returns ('guid', 'dir/dir2', 'file')
00253         splitLN('/dir/dir2/file') returns ('', 'dir/dir2', 'file')
00254 
00255     Trailing slash:
00256         splitLN('guid/dir/dir2/') returns ('guid', 'dir/dir2', '')
00257     
00258         splitLN('') returns ('', '', '')
00259         splitLN('/') returns ('', '', '')
00260         splitLN('//') returns ('', '', '')
00261         splitLN('///') returns ('', '/', '')
00262         splitLN('///') returns ('', '//', '')
00263 
00264         splitLN('0') returns ('0', '', '')
00265         splitLN('0/') returns ('0', '', '')
00266         splitLN('something') returns ('something', '', '')
00267         splitLN('something/') returns ('something', '', '')    
00268     """
00269     # split it along the slashes
00270     parts = LN.split('/')
00271     # get the first part (before the first slash)
00272     rootguid = parts.pop(0)
00273     # try to get the last part, if there are more parts
00274     try:
00275         basename = parts.pop()
00276     except:
00277         basename = ''
00278     # put the slashes back between the rest items
00279     dirname = '/'.join(parts)
00280     return rootguid, dirname, basename
00281 
00282 from arcom.security import storage_actions, AuthPolicy, make_decision
00283 
00284 def parse_storage_policy(metadata):
00285     import arc
00286     p = AuthPolicy()
00287     p.set_policy([(property, value) for (section, property), value in metadata.items() if section == 'policy'])
00288     if metadata.has_key(('entry','owner')):
00289         p[metadata[('entry','owner')]] = ['+' + action for action in storage_actions]        
00290     return p
00291 
00292 def make_decision_metadata(metadata, request):
00293     import arc
00294     #return arc.DECISION_PERMIT
00295     policy = parse_storage_policy(metadata).get_policy()
00296     #print 'DECISION NEEDED\nPOLICY:\n%s\nREQUEST:\n%s\n' % (policy, request)
00297     try:
00298         decision = make_decision(policy, request)
00299     except:
00300         print 'DECISION ERROR. PERMITTING.'
00301         decision = arc.DECISION_PERMIT            
00302     #if decision == arc.DECISION_PERMIT:
00303     #    print 'PERMITTED!'
00304     #else:
00305     #    print 'DENIED! (%s)' % decision
00306     return decision