Back to index

nordugrid-arc-nox  1.1.0~rc6
Functions | Variables
storage.common Namespace Reference

Functions

def upload_to_turl
def download_from_turl
def create_checksum
def _md5sum
def serialize_ids
def deserialize_ids
def parse_metadata
def create_metadata
def remove_trailing_slash
def splitLN
def parse_storage_policy
def make_decision_metadata

Variables

string global_root_guid = '0'
string sestore_guid = '1'
string ahash_list_guid = '2'
string ALIVE = 'alive'
string CREATING = 'creating'
string STALLED = 'stalled'
string INVALID = 'invalid'
string DELETED = 'deleted'
string THIRDWHEEL = 'thirdwheel'
string OFFLINE = 'offline'
list common_supported_protocols = ['http', 'byteio','external']
int CHUNKSIZE = 2

Function Documentation

def storage.common._md5sum (   file) [private]
Create md5 checksum of a file.

_md5sum(file)

file is an object with a 'read' method

Returns the checksum as a string.

Definition at line 81 of file common.py.

00081 
00082 def _md5sum(file):
00083     """ Create md5 checksum of a file.
00084     
00085     _md5sum(file)
00086     
00087     file is an object with a 'read' method
00088     
00089     Returns the checksum as a string.
00090     """
00091     m = md5.new()
00092     while True:
00093         # read the file in chunks
00094         t = file.read(1024)
00095         if len(t) == 0: break # end of file
00096         # put the file content into the md5 object
00097         m.update(t)
00098     # get the checksum from the md5 object
00099     return m.hexdigest()

Here is the caller graph for this function:

def storage.common.create_checksum (   file,
  type 
)
Create checksum of a file.

create_checksum(file, type)

file is an object with a 'read' method
type is a string indicating the type of the checksum, currently only md5 is supported

Returns the checksum as a string.

Definition at line 59 of file common.py.

00059 
00060 def create_checksum(file, type):
00061     """ Create checksum of a file.
00062     
00063     create_checksum(file, type)
00064     
00065     file is an object with a 'read' method
00066     type is a string indicating the type of the checksum, currently only md5 is supported
00067     
00068     Returns the checksum as a string.
00069     """
00070     if type == 'md5':
00071         return _md5sum(file)
00072     else:
00073         raise Exception, 'Unsupported checksum type'
00074 
00075 try:
00076     import hashlib as md5
00077     md5.new_new = md5.new
00078     md5.new = lambda: md5.new_new('md5')
00079 except:
00080     import md5

Here is the call graph for this function:

Here is the caller graph for this function:

def storage.common.create_metadata (   metadata,
  prefix = '' 
)
Create an XMLTree structure from a dictionary with metadata.

create_metadata(metadata, prefix = '')

metadata is a dictionary with ('section','property') as keys and 'value' as values
prefix is a string for the namespace prefix of the XML tag, empty by default.

Example:

input:

    {('entry','type') : 'file', ('states', 'size') : '812314'}

output:

    [('metadata', [('section', 'entry'), ('property', 'type'), ('value', 'file')]),
     ('metadata', [('section', 'states'), ('property', 'size'), ('value', '812314')])]
     
this output can be used as an XMLTree object, and can be put into an XMLNode, which will look like this:

        <metadata>
            <section>entry</section>
            <property>type</property>
            <value>file</value>
        </metadata>
        <metadata>
            <section>states</section>
            <property>size</property>
            <value>812314</value>
        </metadata>

Definition at line 171 of file common.py.

00171 
00172 def create_metadata(metadata, prefix = ''):
00173     """ Create an XMLTree structure from a dictionary with metadata.
00174     
00175     create_metadata(metadata, prefix = '')
00176     
00177     metadata is a dictionary with ('section','property') as keys and 'value' as values
00178     prefix is a string for the namespace prefix of the XML tag, empty by default.
00179     
00180     Example:
00181     
00182     input:
00183     
00184         {('entry','type') : 'file', ('states', 'size') : '812314'}
00185     
00186     output:
00187     
00188         [('metadata', [('section', 'entry'), ('property', 'type'), ('value', 'file')]),
00189          ('metadata', [('section', 'states'), ('property', 'size'), ('value', '812314')])]
00190          
00191     this output can be used as an XMLTree object, and can be put into an XMLNode, which will look like this:
00192     
00193             <metadata>
00194                 <section>entry</section>
00195                 <property>type</property>
00196                 <value>file</value>
00197             </metadata>
00198             <metadata>
00199                 <section>states</section>
00200                 <property>size</property>
00201                 <value>812314</value>
00202             </metadata>
00203     """
00204     # if it is an empty dictionary, just return an empty list
00205     if not metadata:
00206         return []
00207     # if a prefix is specified
00208     if prefix:
00209         # for each item in the metadata list get the section, property and value and put it into XMLTree form
00210         # the 'metadata', 'section', 'property' and 'value' strings are prefixed with the given prefix and a ':'
00211         return [
00212             (prefix + ':metadata', [ 
00213                 (prefix + ':section', section),
00214                 (prefix + ':property', property),
00215                 (prefix + ':value', value)
00216             ]) for ((section, property), value) in metadata.items()
00217         ]
00218     else:
00219         # if no prefix is specified do not put a colon before the names
00220         return [
00221             ('metadata', [ 
00222                 ('section', section),
00223                 ('property', property),
00224                 ('value', value)
00225             ]) for ((section, property), value) in metadata.items()
00226         ]

Here is the caller graph for this function:

Deserialize a list of IDs from a string.

deserialize_ids(s)

s is a string created by the 'serialize_ids' method

Returns the list of IDs (which are all strings) which were originally serialized into the string.

Definition at line 111 of file common.py.

00111 
00112 def deserialize_ids(s):
00113     """ Deserialize a list of IDs from a string.
00114     
00115     deserialize_ids(s)
00116     
00117     s is a string created by the 'serialize_ids' method
00118     
00119     Returns the list of IDs (which are all strings) which were originally serialized into the string.
00120     """
00121     return s.split()

Here is the caller graph for this function:

def storage.common.download_from_turl (   turl,
  protocol,
  fobj,
  ssl_config = {} 
)
docstring for download_from_turl

Definition at line 41 of file common.py.

00041 
00042 def download_from_turl(turl, protocol, fobj, ssl_config = {}):
00043     """docstring for download_from_turl"""
00044     if protocol not in common_supported_protocols:
00045         raise Exception, 'Unsupported protocol'
00046     if protocol == 'byteio':
00047         from storage.client import ByteIOClient
00048         ByteIOClient(turl, ssl_config = ssl_config).read(file = f)
00049     elif protocol == 'http':
00050         import arc
00051         from arcom import datapoint_from_url
00052         src = datapoint_from_url(turl, ssl_config)
00053         dst = datapoint_from_url(fobj.name)
00054         mover = arc.DataMover()
00055         mover.verbose(False)
00056         mover.retry(False)
00057         status = mover.Transfer(src, dst, arc.FileCache(), arc.URLMap())
00058         return str(status)

Here is the call graph for this function:

Here is the caller graph for this function:

def storage.common.make_decision_metadata (   metadata,
  request 
)

Definition at line 292 of file common.py.

00292 
00293 def make_decision_metadata(metadata, request):
00294     import arc
00295     #return arc.DECISION_PERMIT
00296     policy = parse_storage_policy(metadata).get_policy()
00297     #print 'DECISION NEEDED\nPOLICY:\n%s\nREQUEST:\n%s\n' % (policy, request)
00298     try:
00299         decision = make_decision(policy, request)
00300     except:
00301         print 'DECISION ERROR. PERMITTING.'
00302         decision = arc.DECISION_PERMIT            
00303     #if decision == arc.DECISION_PERMIT:
00304     #    print 'PERMITTED!'
00305     #else:
00306     #    print 'DENIED! (%s)' % decision
    return decision

Here is the call graph for this function:

Here is the caller graph for this function:

def storage.common.parse_metadata (   metadatalist_node)
Return the metadata which was put into an XML representation.

parse_metadata(metadatalist_node)

metadatalist_node is an XMLNode which has zero or more children,
    and each child should have three children called 'section', 'property', 'value')

The method get the string value from each tuple of section-property-value,
then creates a dictionary where the (section, property) pairs are the keys,
and the (value) is the value.

Example:

input:

    <metadataList>
        <metadata>
            <section>entry</section>
            <property>type</property>
            <value>file</value>
        </metadata>
        <metadata>
            <section>states</section>
            <property>size</property>
            <value>812314</value>
        </metadata>
    </metadataList>

output:

    {('entry','type') : 'file', ('states', 'size') : '812314'}

Definition at line 122 of file common.py.

00122 
00123 def parse_metadata(metadatalist_node):
00124     """ Return the metadata which was put into an XML representation.
00125     
00126     parse_metadata(metadatalist_node)
00127     
00128     metadatalist_node is an XMLNode which has zero or more children,
00129         and each child should have three children called 'section', 'property', 'value')
00130     
00131     The method get the string value from each tuple of section-property-value,
00132     then creates a dictionary where the (section, property) pairs are the keys,
00133     and the (value) is the value.
00134     
00135     Example:
00136     
00137     input:
00138     
00139         <metadataList>
00140             <metadata>
00141                 <section>entry</section>
00142                 <property>type</property>
00143                 <value>file</value>
00144             </metadata>
00145             <metadata>
00146                 <section>states</section>
00147                 <property>size</property>
00148                 <value>812314</value>
00149             </metadata>
00150         </metadataList>
00151     
00152     output:
00153     
00154         {('entry','type') : 'file', ('states', 'size') : '812314'}
00155     """
00156     # get the number of children
00157     metadatalist_number = metadatalist_node.Size()
00158     # initialize the list
00159     metadata = []
00160     # for each child
00161     for j in range(metadatalist_number):
00162         # get the child node
00163         metadata_node = metadatalist_node.Child(j)
00164         # get the 'section', 'property' and 'value' children's string value
00165         # and append them to the list as (('section','property'),'value')
00166         metadata.append((
00167             (str(metadata_node.Get('section')),str(metadata_node.Get('property'))),
00168                 str(metadata_node.Get('value'))))
00169     # create a dictionary from the list
00170     return dict(metadata)

Here is the caller graph for this function:

Definition at line 284 of file common.py.

00284 
00285 def parse_storage_policy(metadata):
00286     import arc
00287     p = AuthPolicy()
00288     p.set_policy([(property, value) for (section, property), value in metadata.items() if section == 'policy'])
00289     if metadata.has_key(('entry','owner')):
00290         p[metadata[('entry','owner')]] = ['+' + action for action in storage_actions]        
00291     return p

Here is the caller graph for this function:

Remove the trailing slash of a Logical Name.

remove_trailing_slash(LN)

This method checks if the LN ends with a '/', and if it does, cut the trailing slash, and returns the LN.

Definition at line 227 of file common.py.

00227 
00228 def remove_trailing_slash(LN):
00229     """ Remove the trailing slash of a Logical Name.
00230     
00231     remove_trailing_slash(LN)
00232     
00233     This method checks if the LN ends with a '/', and if it does, cut the trailing slash, and returns the LN.
00234     """
00235     while LN.endswith('/'):
00236         LN = LN[:-1]
00237     return LN

Here is the caller graph for this function:

Serialize a list of IDs into one string.

serialize_ids(ids)

ids is a list of IDs, which are strings

Returns one string which containes all the IDs.

Definition at line 100 of file common.py.

00100 
00101 def serialize_ids(ids):
00102     """ Serialize a list of IDs into one string.
00103     
00104     serialize_ids(ids)
00105     
00106     ids is a list of IDs, which are strings
00107     
00108     Returns one string which containes all the IDs.
00109     """
00110     return ' '.join([str(id) for id in ids])

Here is the caller graph for this function:

def storage.common.splitLN (   LN)
Split a Logical Name to a 3-tuple: GUID, dirname, basename.

splitLN(LN)

The part before the first slash is the GUID, even if it is empty.
The part after the last slash is the basename, could be empty.
Between them is the dirname, could be empty or could contain several more slashes.

Examples

    splitLN('/dir') returns ('', '', 'dir')
    splitLN('guid/dir') returns ('guid', '', 'dir')
    splitLN('guid/dir/file') returns ('guid', 'dir', 'file')
    splitLN('guid/dir/dir2/file') returns ('guid', 'dir/dir2', 'file')
    splitLN('/dir/dir2/file') returns ('', 'dir/dir2', 'file')

Trailing slash:
    splitLN('guid/dir/dir2/') returns ('guid', 'dir/dir2', '')

    splitLN('') returns ('', '', '')
    splitLN('/') returns ('', '', '')
    splitLN('//') returns ('', '', '')
    splitLN('///') returns ('', '/', '')
    splitLN('///') returns ('', '//', '')

    splitLN('0') returns ('0', '', '')
    splitLN('0/') returns ('0', '', '')
    splitLN('something') returns ('something', '', '')
    splitLN('something/') returns ('something', '', '')    

Definition at line 238 of file common.py.

00238 
00239 def splitLN(LN):
00240     """  Split a Logical Name to a 3-tuple: GUID, dirname, basename.
00241     
00242     splitLN(LN)
00243     
00244     The part before the first slash is the GUID, even if it is empty.
00245     The part after the last slash is the basename, could be empty.
00246     Between them is the dirname, could be empty or could contain several more slashes.
00247     
00248     Examples
00249 
00250         splitLN('/dir') returns ('', '', 'dir')
00251         splitLN('guid/dir') returns ('guid', '', 'dir')
00252         splitLN('guid/dir/file') returns ('guid', 'dir', 'file')
00253         splitLN('guid/dir/dir2/file') returns ('guid', 'dir/dir2', 'file')
00254         splitLN('/dir/dir2/file') returns ('', 'dir/dir2', 'file')
00255 
00256     Trailing slash:
00257         splitLN('guid/dir/dir2/') returns ('guid', 'dir/dir2', '')
00258     
00259         splitLN('') returns ('', '', '')
00260         splitLN('/') returns ('', '', '')
00261         splitLN('//') returns ('', '', '')
00262         splitLN('///') returns ('', '/', '')
00263         splitLN('///') returns ('', '//', '')
00264 
00265         splitLN('0') returns ('0', '', '')
00266         splitLN('0/') returns ('0', '', '')
00267         splitLN('something') returns ('something', '', '')
00268         splitLN('something/') returns ('something', '', '')    
00269     """
00270     # split it along the slashes
00271     parts = LN.split('/')
00272     # get the first part (before the first slash)
00273     rootguid = parts.pop(0)
00274     # try to get the last part, if there are more parts
00275     try:
00276         basename = parts.pop()
00277     except:
00278         basename = ''
00279     # put the slashes back between the rest items
00280     dirname = '/'.join(parts)
00281     return rootguid, dirname, basename

Here is the caller graph for this function:

def storage.common.upload_to_turl (   turl,
  protocol,
  fobj,
  size = None,
  ssl_config = {} 
)
docstring for upload_to_turl

Definition at line 21 of file common.py.

00021 
00022 def upload_to_turl(turl, protocol, fobj, size = None, ssl_config = {}):
00023     """docstring for upload_to_turl"""
00024     if protocol not in common_supported_protocols:
00025         raise Exception, 'Unsupported protocol'
00026     if protocol == 'byteio':
00027         from storage.client import ByteIOClient
00028         return ByteIOClient(turl, ssl_config = ssl_config).write(fobj)
00029     elif protocol == 'external':
00030         return 
00031     elif protocol == 'http':
00032         import arc
00033         from arcom import datapoint_from_url
00034         src = datapoint_from_url(fobj.name)
00035         dst = datapoint_from_url(turl, ssl_config)
00036         mover = arc.DataMover()
00037         mover.verbose(False)
00038         mover.retry(False)
00039         status = mover.Transfer(src, dst, arc.FileCache(), arc.URLMap())
00040         return str(status)

Here is the call graph for this function:

Here is the caller graph for this function:


Variable Documentation

Definition at line 8 of file common.py.

string storage.common.ALIVE = 'alive'

Definition at line 10 of file common.py.

Definition at line 19 of file common.py.

list storage.common.common_supported_protocols = ['http', 'byteio','external']

Definition at line 18 of file common.py.

string storage.common.CREATING = 'creating'

Definition at line 11 of file common.py.

string storage.common.DELETED = 'deleted'

Definition at line 14 of file common.py.

Definition at line 4 of file common.py.

string storage.common.INVALID = 'invalid'

Definition at line 13 of file common.py.

string storage.common.OFFLINE = 'offline'

Definition at line 16 of file common.py.

Definition at line 6 of file common.py.

string storage.common.STALLED = 'stalled'

Definition at line 12 of file common.py.

string storage.common.THIRDWHEEL = 'thirdwheel'

Definition at line 15 of file common.py.