Back to index

nordugrid-arc-nox  1.1.0~rc6
bartender.py
Go to the documentation of this file.
00001 import urlparse
00002 import httplib
00003 import arc
00004 import random
00005 import time
00006 import traceback
00007 import os
00008 import base64
00009 import threading
00010 from arcom import import_class_from_string, get_child_nodes, get_child_values_by_name
00011 from arcom.service import librarian_uri, bartender_uri, librarian_servicetype, gateway_uri, true, parse_node, create_response, node_to_data, bartender_servicetype
00012 from arcom.xmltree import XMLTree
00013 from storage.client import LibrarianClient, ShepherdClient, ISISClient
00014 from storage.common import parse_metadata, create_metadata, splitLN, remove_trailing_slash, global_root_guid, serialize_ids, deserialize_ids, sestore_guid, make_decision_metadata
00015 import traceback
00016 
00017 from storage.common import ALIVE, CREATING, THIRDWHEEL
00018 
00019 
00020 from arcom.logger import Logger
00021 log = Logger(arc.Logger(arc.Logger_getRootLogger(), 'Storage.Bartender'))
00022 
00023 class Bartender:
00024 
00025     def __init__(self, cfg, ssl_config, service_state):
00026         """ Constructor of the Bartender business-logic class.
00027         
00028         """
00029         self.service_state = service_state
00030         gatewayclass = str(cfg.Get('GatewayClass'))
00031         self.gateway = None
00032         if gatewayclass :
00033             cl = import_class_from_string(gatewayclass) 
00034             gatewaycfg = cfg.Get('GatewayCfg')
00035             self.gateway = cl(gatewaycfg)
00036             log.msg(arc.INFO, 'accessing gateway: %s', gatewayclass)
00037         else:
00038             log.msg(arc.INFO, 'This bartender does not support gateway') 
00039             log.msg(arc.INFO, 'cannot connect to gateway. Access of third party store required gateway.')
00040         self.ssl_config = ssl_config
00041         
00042         # get the URLs of the Librarians from the config file
00043         librarian_urls =  get_child_values_by_name(cfg, 'LibrarianURL')
00044         if librarian_urls:
00045             log.msg(arc.INFO,'Got Librarian URLs from the config:', librarian_urls)
00046             self.librarian = LibrarianClient(librarian_urls, ssl_config = self.ssl_config)
00047             self.service_state.running = True
00048         else:
00049             isis_urls = get_child_values_by_name(cfg, 'ISISURL')
00050             if not isis_urls:
00051                 log.msg(arc.ERROR, "Librarian URL or ISIS URL not found in the configuration.")
00052                 raise Exception, 'Bartender cannot run with no Librarian'
00053             log.msg(arc.INFO,'Got ISIS URL, starting initThread')
00054             threading.Thread(target = self.initThread, args = [isis_urls]).start()
00055 
00056     def initThread(self, isis_urls):
00057         librarian_found = False
00058         while not librarian_found:
00059             try:
00060                 log.msg(arc.INFO,'Getting Librarians from ISISes')
00061                 for isis_url in isis_urls:
00062                     log.msg(arc.INFO,'Trying to get Librarian from', isis_url)
00063                     isis = ISISClient(isis_url, ssl_config = self.ssl_config)
00064                     try:
00065                         librarian_urls = isis.getServiceURLs(librarian_servicetype)
00066                         log.msg(arc.INFO,'Got Librarian from ISIS:', librarian_urls)
00067                         if librarian_urls:
00068                             self.librarian = LibrarianClient(librarian_urls, ssl_config = self.ssl_config)
00069                             librarian_found = True
00070                     except:
00071                         log.msg(arc.VERBOSE, 'Error connecting to ISIS %{iu}s, reason: %{r}s' % {'iu' : isis_url, 'r' : traceback.format_exc()})
00072                 time.sleep(3)
00073             except Exception, e:
00074                 log.msg(arc.WARNING, 'Error in initThread: %s' % e)                
00075         log.msg(arc.INFO,'initThread finished, starting isisThread')
00076         self.service_state.running = True
00077         threading.Thread(target = self.isisThread, args = [isis_urls]).start()
00078             
00079     def isisThread(self, isis_urls):
00080         while self.service_state.running:
00081             try:
00082                 time.sleep(30)
00083                 log.msg(arc.INFO,'Getting Librarians from ISISes')
00084                 for isis_url in isis_urls:
00085                     if not self.service_state.running:
00086                         return
00087                     log.msg(arc.INFO,'Trying to get Librarian from', isis_url)
00088                     isis = ISISClient(isis_url, ssl_config = self.ssl_config)        
00089                     librarian_urls = isis.getServiceURLs(librarian_servicetype)
00090                     log.msg(arc.INFO, 'Got Librarian from ISIS:', librarian_urls)
00091                     if librarian_urls:
00092                         self.librarian = LibrarianClient(librarian_urls, ssl_config = self.ssl_config)
00093                         break
00094             except Exception, e:
00095                 log.msg(arc.WARNING, 'Error in isisThread: %s' % e)                
00096 
00097     def stat(self, auth, requests):
00098         """ Returns stat information about entries.
00099         
00100         stat(auth, requests)
00101         
00102         auth is an AuthRequest object containing information about the user's identity
00103         requests is a dictionary with requestIDs as keys, and Logical Names as values.
00104     
00105         Returns a dictionary with requestIDs as keys, and metadata as values.
00106         The 'metadata' is a dictionary with (section, property) pairs as keys.
00107         """
00108         response = {}
00109         # get the information from the librarian
00110         requests, traverse_response = self._traverse(requests)
00111         # we are only interested in the metadata and if the traversing was complete or not
00112         for requestID, (metadata, _, _, _, wasComplete, _) in traverse_response.items():
00113             if wasComplete: # if it was complete, then we found the entry and got the metadata
00114                 try:
00115                     decision = make_decision_metadata(metadata, auth.get_request('read'))
00116                     if decision != arc.DECISION_PERMIT:
00117                         metadata = {('error','permission denied') : 'you are not allowed to read'}
00118                 except:
00119                     #log.msg()
00120                     metadata = {('error', 'error checking permission') : traceback.format_exc()}
00121                 response[requestID] = metadata
00122             else: # if it was not complete, then we didn't found the entry, so metadata will be empty
00123                 response[requestID] = {}
00124         return response
00125    
00126     def delFile(self, auth, requests):
00127         """ Delete a file from the storage: initiate the process.
00128         
00129         delFile(requests)
00130         
00131         requests is a dictionary with requestID as key and (Logical Name, child metadata, protocols) as value
00132         """
00133         auth_request = auth.get_request('delete')
00134         import time
00135         response = {}
00136         # get the information from the librarian
00137         #requests, traverse_response = self._traverse(requests)
00138         traverse_response = self.librarian.traverseLN(requests)
00139         cat_rem_requests = {}
00140         cat_mod_requests = {}
00141         check_again = []
00142         for requestID, (metadata, GUID, LN, restLN, wasComplete, traversedList) in traverse_response.items():
00143             decision = make_decision_metadata(metadata, auth_request)
00144             if decision != arc.DECISION_PERMIT:
00145                 response[requestID] = 'denied'
00146             elif wasComplete and metadata[('entry', 'type')]=='file': # if it was complete, then we found the entry and got the metadata
00147                 # remove the file
00148                 cat_rem_requests[requestID] = GUID
00149                 # if this entry has a parent:
00150                 parents = [p for (s,p) in metadata.keys() if s == 'parents']
00151                 for parent in parents:
00152                     parent_GUID, child_name = parent.split('/')
00153                     cat_mod_requests[requestID + '-' + parent] = (parent_GUID, 'unset', 'entries', child_name, GUID)
00154                     cat_mod_requests[requestID + '-' + parent + '-closed?'] = (parent_GUID, 'setifvalue=yes', 'states', 'closed', 'broken')
00155                 response[requestID] = 'deleted'
00156             elif metadata.get(('entry', 'type'), '') == 'mountpoint' :
00157                 url = metadata[('mountpoint', 'externalURL')]+'/'+restLN
00158                 #print url
00159                 res = self._externalStore(auth ,url,'delFile')
00160                 #print res
00161                 #print res[url]['status']  
00162                 if 'successfully' in res[url]['status']:
00163                     response[requestID] = 'deleted'
00164                 else:
00165                     response[requestID] = 'nosuchLN'       
00166             else: # if it was not complete, then we didn't find the entry, so metadata will be empty
00167                 response[requestID] = 'nosuchLN'
00168         #print cat_rem_requests
00169         #print cat_mod_requests
00170         success = self.librarian.remove(cat_rem_requests)
00171         modify_success = self.librarian.modifyMetadata(cat_mod_requests)
00172         #print success
00173         #print modify_success 
00174         #print response
00175         return response
00176 
00177     def _traverse(self, requests):
00178         """ Helper method which connects the librarian, and traverses the LNs of the requests.
00179         
00180         _traverse(requests)
00181         
00182         Removes the trailing slash from  all the LNs in the request.
00183         Returns the requests and the traverse response.
00184         """
00185         # in each request the requestID is the key and the value is a list
00186         # the first item of the list is the Logical Name, we want to remove the trailing slash, and
00187         # leave the other items intact
00188         requests = [(rID, [remove_trailing_slash(data[0])] + list(data[1:])) for rID, data in requests.items()]
00189         log.msg(arc.VERBOSE, '//// _traverse request trailing slash removed:', dict(requests))
00190         # then we do the traversing. a traverse request contains a requestID and the Logical Name
00191         # so we just need the key (which is the request ID) and the first item of the value (which is the LN)
00192         traverse_request = dict([(rID, data[0]) for rID, data in requests])
00193         # call the librarian service
00194         traverse_response = self.librarian.traverseLN(traverse_request)
00195         if not traverse_response:
00196             raise Exception, 'Empty response from the Librarian'
00197         # return the requests as list (without the trailing slashes) and the traverse response from the librarian
00198         return requests, traverse_response
00199 
00200 
00201     def _new(self, auth, child_metadata, child_name = None, parent_GUID = None, parent_metadata = {}):
00202         """ Helper method which create a new entry in the librarian.
00203         
00204         _new(child_metadata, child_name = None, parent_GUID = None)
00205         
00206         child_metadata is a dictionary with {(section, property) : values} containing the metadata of the new entry
00207         child_name is the name of the new entry 
00208         parent_GUID is the GUID of the parent of the new entry
00209         
00210         This method creates a new librarian-entry with the given metadata.
00211         If child_name and parent_GUID are both given, then this method adds a new entry to the parent collection.
00212         """
00213         try:
00214             # set creation time stamp
00215             child_metadata[('timestamps', 'created')] = str(time.time())
00216             if child_name and parent_GUID:
00217                 child_metadata[('parents', '%s/%s' % (parent_GUID, child_name))] = 'parent' # this 'parent' string is never used
00218             # call the new method of the librarian with the child's metadata (requestID is '_new')
00219             new_response = self.librarian.new({'_new' : child_metadata})
00220             # we can access the response with the requestID, so we get the GUID of the newly created entry
00221             (child_GUID, new_success) = new_response['_new']
00222             # if the new method was not successful
00223             if new_success != 'success':
00224                 return 'failed to create new librarian entry', child_GUID
00225             else:
00226                 # if it was successful and we have a parent collection
00227                 if child_name and parent_GUID:
00228                     decision = make_decision_metadata(parent_metadata, auth.get_request('addEntry'))
00229                     if decision == arc.DECISION_PERMIT:
00230                         # we need to add the newly created librarian-entry to the parent collection
00231                         log.msg(arc.VERBOSE, 'adding', child_GUID, 'to parent', parent_GUID)
00232                         # this modifyMetadata request adds a new (('entries',  child_name) : child_GUID) element to the parent collection
00233                         modify_response = self.librarian.modifyMetadata({'_new' : (parent_GUID, 'add', 'entries', child_name, child_GUID),
00234                                                                         '_new_closed?' : (parent_GUID, 'setifvalue=yes', 'states', 'closed', 'broken')})
00235                         log.msg(arc.VERBOSE, 'modifyMetadata response', modify_response)
00236                         # get the 'success' value
00237                         modify_success = modify_response['_new']
00238                     else:
00239                         modify_success = 'denied'
00240                     # if the new element was not set, we have a problem
00241                     if modify_success != 'set':
00242                         log.msg(arc.VERBOSE, 'modifyMetadata failed, removing the new librarian entry', child_GUID)
00243                         # remove the newly created librarian-entry
00244                         self.librarian.remove({'_new' : child_GUID})
00245                         return 'failed to add child to parent', child_GUID
00246                     else:
00247                         return 'done', child_GUID
00248                 else: # no parent given, skip the 'adding child to parent' part
00249                     return 'done', child_GUID
00250         except Exception, e:
00251             log.msg(arc.ERROR, "Error creating new entry in Librarian: %s" % e)
00252             return 'internal error', None
00253         
00254     def _externalStore(self, auth, url, flag=''):
00255         """ This method calles the gateway backend class to get/check the full URL of the externally stored file"""
00256         response = {}
00257         if self.gateway != None:
00258             if flag == 'list':      
00259                 response = self.gateway.list(auth, url, flag)
00260             elif flag == 'getFile':
00261                 response = self.gateway.get(auth, url, flag)
00262             elif flag == 'delFile':
00263                 response = self.gateway.remove(auth, url, flag)
00264             elif flag == 'putFile': 
00265                  response = self.gateway.put(auth, url, flag)
00266         else:
00267             if flag == 'list':
00268                 response[url] = {'list': '', 'status': 'gateway is not configured. Bartender does not support mount points', 'protocol':''}            
00269             else:
00270                 response[url]={'turl': '' ,'status': 'gateway is not configured. Bartender does not support mount points', 'protocol':''}
00271         log.msg(arc.VERBOSE, '//// response from the external store:', response)
00272         return response
00273 
00274     def getFile(self, auth, requests):
00275         """ Get files from the storage.
00276         
00277         getFile(requests)
00278         
00279         requests is a dictionary with requestID as key, and (Logical Name, protocol list) as value
00280         """
00281         auth_request = auth.get_request('read')
00282         # call the _traverse helper method the get the information about the requested Logical Names
00283         requests, traverse_response = self._traverse(requests)
00284         response = {}
00285         #print traverse_response
00286         # for each requested LN
00287         for rID, (LN, protocols) in requests:
00288             turl = ''
00289             protocol = ''
00290             success = 'unknown'
00291             try:
00292                 log.msg(arc.VERBOSE, traverse_response[rID])
00293                 # split the traverse response
00294                 metadata, GUID, traversedLN, restLN, wasComplete, traversedList = traverse_response[rID]
00295                 # wasComplete is true if the given LN was found, so it could have been fully traversed
00296                 if not wasComplete:
00297                     if metadata.get(('entry', 'type'), '') == 'mountpoint':
00298                         url = metadata[('mountpoint', 'externalURL')]
00299                         res = self._externalStore(auth ,url+'/'+restLN,'getFile')
00300                         if res:
00301                             for key in res.keys():
00302                                 turl = res[key]['turl']
00303                                 protocol = res[key]['protocol'] 
00304                                 if res[key]['status'] == 'successful':
00305                                     success = 'done'
00306                                 else:
00307                                     success = res[key]['status']  
00308                         else:   
00309                             success = 'not found'
00310                     else:       
00311                         success = 'not found'
00312                 else:
00313                     # metadata contains all the metadata of the given entry
00314                     # ('entry', 'type') is the type of the entry: file, collection, etc.
00315                     decision = make_decision_metadata(metadata, auth_request)
00316                     if decision != arc.DECISION_PERMIT:
00317                         success = 'denied'
00318                     else:
00319                         type = metadata[('entry', 'type')]
00320                         if type != 'file':
00321                             success = 'is not a file'
00322                         else:
00323                             # if it is a file,  then we need all the locations where it is stored and alive
00324                             # this means all the metadata entries with in the 'locations' sections whose value is ALIVE
00325                             # the location itself serialized from the ID of the service and the ID of the replica within the service
00326                             # so the location needs to be deserialized into two ID with deserialize_ids()
00327                             # valid_locations will contain a list if (serviceID, referenceID, state)
00328                             valid_locations = [deserialize_ids(location) + [state] for (section, location), state in metadata.items() if section == 'locations' and state == ALIVE]
00329                             # if this list is empty
00330                             if not valid_locations:
00331                                 success = 'file has no valid replica'
00332                             else:
00333                                 ok = False
00334                                 while not ok and len(valid_locations) > 0:
00335                                     # if there are more valid_locations, randomly select one
00336                                     location = valid_locations.pop(random.choice(range(len(valid_locations))))
00337                                     log.msg(arc.VERBOSE, 'location chosen:', location)
00338                                     # split it to serviceID, referenceID - serviceID currently is just a plain URL of the service
00339                                     url, referenceID, _ = location
00340                                     # create an ShepherdClient with this URL, then send a get request with the referenceID
00341                                     try:
00342                                         get_response = dict(ShepherdClient(url, ssl_config = self.ssl_config).get({'getFile' :
00343                                             [('referenceID', referenceID)] + [('protocol', proto) for proto in protocols]})['getFile'])
00344                                     except:
00345                                         get_response = {'error' : traceback.format_exc()}
00346                                     # get_response is a dictionary with keys such as 'TURL', 'protocol' or 'error'
00347                                     if get_response.has_key('error'):
00348                                         # if there was an error
00349                                         log.msg(arc.VERBOSE, 'ERROR from the chosen Shepherd', get_response['error'])
00350                                         success = 'error while getting TURL (%s)' % get_response['error']
00351                                     else:
00352                                         # get the TURL and the choosen protocol, these will be set as reply for this requestID
00353                                         turl = get_response['TURL']
00354                                         protocol = get_response['protocol']
00355                                         success = 'done'
00356                                         ok = True
00357             except:
00358                 success = 'internal error (%s)' % traceback.format_exc()
00359             # set the success, turl, protocol for this request
00360             response[rID] = (success, turl, protocol)
00361         return response
00362    
00363     def addReplica(self, auth, requests, protocols):
00364         """ This method initiates the addition of a new replica to a file.
00365         
00366         addReplica(requests, protocols)
00367         
00368         requests is a dictionary with requestID-GUID pairs
00369         protocols is a list of supported protocols
00370         """
00371         # currently we permit everyone to upload a new replica,
00372         #   if it not matches the checksum or the size, then it will be removed
00373         # get the size and checksum information about all the requested GUIDs (these are in the 'states' section)
00374         #   the second argument of the get method specifies that we only need metadata from the 'states' section
00375         data = self.librarian.get(requests.values(), [('states',''),('locations',''),('policy','')])
00376         response = {}
00377         for rID, GUID in requests.items():
00378             # for each requested GUID
00379             metadata = data[GUID]
00380             log.msg(arc.VERBOSE, 'addReplica', 'requestID', rID, 'GUID', GUID, 'metadata', metadata, 'protocols', protocols)
00381             # get the size and checksum information of the file
00382             size = metadata[('states','size')]
00383             checksumType = metadata[('states','checksumType')]
00384             checksum = metadata[('states','checksum')]
00385             # list of shepherds with an alive or creating replica of this file (to avoid using one shepherd twice)
00386             exceptedSEs = [deserialize_ids(location)[0] 
00387                            for (section, location), status in metadata.items() if section == 'locations' and status in [ALIVE, CREATING, THIRDWHEEL]]
00388             # initiate replica addition of this file with the given protocols 
00389             success, turl, protocol = self._add_replica(size, checksumType, checksum, GUID, protocols, exceptedSEs)
00390             # set the response of this request
00391             response[rID] = (success, turl, protocol)
00392         return response
00393 
00394     def _find_alive_ses(self, except_these=[]):
00395         """  Get the list of currently alive Shepherds.
00396         
00397         _find_alive_ses()
00398         """
00399         # sestore_guid is the GUID of the librarian entry which the list of Shepherds registered by the Librarian
00400         SEs = self.librarian.get([sestore_guid])[sestore_guid]
00401         # SEs contains entries such as {(serviceID, 'nextHeartbeat') : timestamp} which indicates
00402         #   when a specific Shepherd service should report next
00403         #   if this timestamp is not a positive number, that means the Shepherd have not reported in time, probably it is not alive
00404         log.msg(arc.VERBOSE, 'Registered Shepherds in Librarian', SEs)
00405         # get all the Shepherds which has a positiv nextHeartbeat timestamp and which has not already been used
00406         alive_SEs = [s for (s, p), v in SEs.items() if p == 'nextHeartbeat' and int(v) > 0 and not s in except_these]
00407         log.msg(arc.VERBOSE, 'Alive Shepherds:', alive_SEs)
00408         response = []
00409         for se in alive_SEs:
00410             response.append(ShepherdClient(se, ssl_config = self.ssl_config))
00411         return response
00412 
00413     def _add_replica(self, size, checksumType, checksum, GUID, protocols, exceptedSEs=[]):
00414         """ Helper method to initiate addition of a replica to a file.
00415         
00416         _add_replica(size, checksumType, checksum, GUID, protocols)
00417         
00418         size is the size of the file
00419         checksumType indicates the type of the checksum
00420         checksum is the checksum itself
00421         GUID is the GUID of the file
00422         protocols is a list of protocols
00423         """
00424         turl = ''
00425         protocol = ''
00426         # prepare the 'put' request for the shepherd
00427         put_request = [('size', size), ('checksumType', checksumType),
00428             ('checksum', checksum), ('GUID', GUID)] + \
00429             [('protocol', protocol) for protocol in protocols]
00430         # find an alive Shepherd
00431         shepherds = self._find_alive_ses(exceptedSEs)
00432         random.shuffle(shepherds)
00433         while len(shepherds) > 0:
00434             shepherd = shepherds.pop()
00435             # call the SE's put method with the prepared request
00436             try:
00437                 put_response = dict(shepherd.put({'putFile': put_request})['putFile'])
00438             except:
00439                 put_response = {'error' : traceback.format_exc()}
00440             if put_response.has_key('error'):
00441                 log.msg(arc.VERBOSE, 'ERROR from the chosen Shepherd', put_response['error'])
00442             else:
00443                 # if the put request was successful then we have a transfer URL, a choosen protocol and the referenceID of the file
00444                 turl = put_response['TURL']
00445                 protocol = put_response['protocol']
00446                 return 'done', turl, protocol
00447         return 'no suitable shepherd found', turl, protocol
00448             
00449     def putFile(self, auth, requests):
00450         """ Put a new file to the storage: initiate the process.
00451         
00452         putFile(requests)
00453         
00454         requests is a dictionary with requestID as key and (Logical Name, child metadata, protocols) as value
00455         """
00456         # get all the information about the requested Logical Names from the librarian
00457         requests, traverse_response = self._traverse(requests)
00458         response = {}
00459         for rID, (LN, child_metadata, protocols) in requests:
00460             # for each request
00461             turl = ''
00462             protocol = ''
00463             metadata_ok = False
00464             try:
00465                 # get the size and checksum of the new file
00466                 log.msg(arc.VERBOSE, protocols)
00467                 size = child_metadata[('states', 'size')]
00468                 checksum = child_metadata[('states', 'checksum')]
00469                 checksumType = child_metadata[('states', 'checksumType')]
00470                 # need neededReplicas to see if we should call _add_replica
00471                 neededReplicas = child_metadata[('states', 'neededReplicas')]
00472                 metadata_ok = True
00473             except Exception, e:
00474                 success = 'missing metadata ' + str(e)
00475             if metadata_ok:
00476                 # if the metadata of the new file is OK
00477                 child_metadata[('entry','type')] = 'file'
00478                 child_metadata[('entry','owner')] = auth.get_identity()
00479                 try:
00480                     # split the Logical Name, rootguid will be the GUID of the root collection of this LN,
00481                     #   child_name is the name of the new file withing the parent collection
00482                     rootguid, _, child_name = splitLN(LN)
00483                     log.msg(arc.VERBOSE, 'LN', LN, 'rootguid', rootguid, 'child_name', child_name, 'real rootguid', rootguid or global_root_guid)
00484                     # get the traverse response corresponding to this request
00485                     #   metadata is the metadata of the last element which could been traversed, e.g. the parent of the new file
00486                     #   GUID is the GUID of the same
00487                     #   traversedLN indicates which part of the requested LN could have been traversed
00488                     #   restLN is the non-traversed part of the Logical Name, e.g. the filename of a non-existent file whose parent does exist
00489                     #   wasComplete indicates if the traverse was complete or not, if it was complete means that this LN exists
00490                     #   traversedlist contains the GUID and metadata of each element along the path of the LN
00491                     metadata, GUID, traversedLN, restLN, wasComplete, traversedlist = traverse_response[rID]
00492                     log.msg(arc.VERBOSE, 'metadata', metadata, 'GUID', GUID, 'traversedLN', traversedLN, 'restLN', restLN, 'wasComplete',wasComplete, 'traversedlist', traversedlist)
00493                     # if the traversing stopped at a mount point:
00494                     if metadata.get(('entry','type'), '') == 'mountpoint':
00495                         url = metadata[('mountpoint','externalURL')] + '/' + restLN                
00496                         res = self._externalStore(auth, url, 'putFile') 
00497                         response[rID] = (res[url]['status'],res[url]['turl'],res[url]['protocol']) 
00498                         return response    
00499                     if wasComplete: # this means the LN already exists, so we couldn't put a new file there
00500                         success = 'LN exists'
00501                     elif child_name == '': # this only can happen if the LN was a single GUID
00502                         # this means that the new file will have no parent
00503                         # we don't want to allow this:
00504                         success = 'cannot create a file without a parent collection'
00505                         # # set the type and GUID of the new file
00506                         # child_metadata[('entry','GUID')] = rootguid or global_root_guid
00507                         # # create the new entry
00508                         # success, GUID = self._new(auth, child_metadata)
00509                     elif restLN != child_name or GUID == '':
00510                         # if the non-traversed part of the Logical Name is not actully the name of the new file
00511                         #   or we have no parent guid
00512                         success = 'parent does not exist'
00513                     else:
00514                         # if everything is OK, then we create the new entry
00515                         success, GUID = self._new(auth, child_metadata, child_name, GUID, metadata)
00516                     if success == 'done':
00517                         # if the file was successfully created, it still has no replica, so we initiate creating one
00518                         # if neededReplicas is 0, we do nothing
00519                         if int(neededReplicas) > 0: # this will call shepherd.put()
00520                             success, turl, protocol = self._add_replica(size, checksumType, checksum, GUID, protocols)
00521                 except:
00522                     success = 'internal error (%s)' % traceback.format_exc()
00523             response[rID] = (success, turl, protocol)
00524         return response
00525         
00526     def unlink(self, auth, requests):
00527         """docstring for unlink"""
00528         auth_request = auth.get_request('removeEntry')
00529         requests, traverse_response = self._traverse(requests)
00530         response = {}
00531         for rID, [LN] in requests:
00532             metadata, GUID, traversedLN, restLN, wasComplete, traversedlist = traverse_response[rID]
00533             #print 'metadata', metadata, 'GUID', GUID, 'traversedLN', traversedLN, 'restLN', restLN, 'wasComplete',wasComplete, 'traversedlist', traversedlist
00534             if not wasComplete:
00535                 success = 'no such LN'
00536             else:
00537                 if len(traversedlist) < 2:
00538                     success = 'nothing to unlink'
00539                 else:
00540                     parent_GUID = traversedlist[-2][1]
00541                     child_name = traversedlist[-1][0]
00542                     parent_metadata = self.librarian.get([parent_GUID])[parent_GUID]
00543                     decision = make_decision_metadata(parent_metadata, auth_request)
00544                     if decision != arc.DECISION_PERMIT:
00545                         success = 'denied'
00546                     else:
00547                         mod_requests = {'unlink' : (parent_GUID, 'unset', 'entries', child_name, ''),
00548                                         'unlink-closed?' : (parent_GUID, 'setifvalue=yes', 'states', 'closed', 'broken')}
00549                         mod_response = self.librarian.modifyMetadata(mod_requests)
00550                         success = mod_response['unlink']
00551             response[rID] = success
00552         return response
00553 
00554     def unmakeCollection(self, auth, requests):
00555         """docstring for unmakeCollection"""
00556         auth_request = auth.get_request('delete')
00557         requests, traverse_response = self._traverse(requests)
00558         response = {}
00559         for rID, [LN] in requests:
00560             metadata, GUID, traversedLN, restLN, wasComplete, traversedlist = traverse_response[rID]
00561             log.msg(arc.VERBOSE, 'metadata', metadata, 'GUID', GUID, 'traversedLN', traversedLN, 'restLN', restLN, 'wasComplete',wasComplete, 'traversedlist', traversedlist)
00562             if not wasComplete:
00563                 success = 'no such LN'
00564             else:
00565                 decision = make_decision_metadata(metadata, auth_request)
00566                 if decision != arc.DECISION_PERMIT:
00567                     success = 'denied'
00568                 else:
00569                     number_of_entries = len([section for (section, _), _ in metadata.items() if section == 'entries'])
00570                     if number_of_entries > 0:
00571                         success = 'collection is not empty'
00572                     else:
00573                         try:
00574                             parentLN, parentGUID = traversedlist[-2]
00575                             # TODO: get the metadata of the parent, and check if the user has permission to removeEntry from it
00576                             mod_requests = {'unmake' : (parentGUID, 'unset', 'entries', traversedlist[-1][0], ''),
00577                                             'unmake-closed?' : (parentGUID, 'setifvalue=yes', 'states', 'closed', 'broken')}
00578                             mod_response = self.librarian.modifyMetadata(mod_requests)
00579                             success = mod_response['unmake']
00580                         except IndexError:
00581                             # it has no parent
00582                             success = 'unset'
00583                         if success == 'unset':
00584                             # TODO: handle hardlinks to collections
00585                             success = self.librarian.remove({'unmake' : GUID})['unmake']
00586             response[rID] = success
00587         return response
00588 
00589     def makeCollection(self, auth, requests):
00590         """ Create a new collection.
00591         
00592         makeCollection(requests)
00593         
00594         requests is dictionary with requestID as key and (Logical Name, metadata) as value
00595         """
00596         # do traverse all the requests
00597         requests, traverse_response = self._traverse(requests)
00598         response = {}
00599         for rID, (LN, child_metadata) in requests:
00600             # for each request first split the Logical Name
00601             rootguid, _, child_name = splitLN(LN)
00602             metadata, GUID, traversedLN, restLN, wasComplete, traversedlist = traverse_response[rID]
00603             #print metadata[('entry', 'type')]
00604             if metadata.get(('entry', 'type'), '') == 'mountpoint':
00605                 success = 'cannot create collection in mountpoint'      
00606                 response[rID] = success    
00607                 return response
00608             log.msg(arc.VERBOSE, 'metadata', metadata, 'GUID', GUID, 'traversedLN', traversedLN, 'restLN', restLN, 'wasComplete',wasComplete, 'traversedlist', traversedlist)
00609             child_metadata[('entry','owner')] = auth.get_identity()
00610             child_metadata[('entry','type')] = 'collection'
00611             if wasComplete: # this means the LN exists
00612                 success = 'LN exists'
00613             elif child_name == '': # this only can happen if the LN was a single GUID
00614                 # this means the collection has no parent
00615                 # we only allow this for the global root
00616                 if rootguid != global_root_guid and rootguid != '':
00617                     success = 'cannot create collection without a parent collection'
00618                 else:
00619                     child_metadata[('entry','GUID')] = global_root_guid
00620                     success, _ = self._new(auth, child_metadata)
00621             elif restLN != child_name or GUID == '':
00622                 success = 'parent does not exist'
00623             else:
00624                 # if everything is OK, create the new collection
00625                 #   here GUID is of the parent collection
00626                 success, _ = self._new(auth, child_metadata, child_name, GUID, metadata)
00627             response[rID] = success
00628         return response
00629 
00630     ### Created by Salman Toor ###
00631     
00632     def unmakeMountpoint(self, auth, requests):        
00633         """docstring for unmakeMountpoint"""  
00634         auth_request = auth.get_request('delete')      
00635         requests, traverse_response = self._traverse(requests)
00636         response = {}
00637         for rID, [LN] in requests:
00638             metadata, GUID, traversedLN, restLN, wasComplete, traversedlist = traverse_response[rID]
00639             log.msg(arc.VERBOSE, 'metadata', metadata, 'GUID', GUID, 'traversedLN', traversedLN, 'restLN', restLN, 'wasComplete',wasComplete, 'traversedlist', traversedlist)
00640             if not wasComplete:
00641                 success = 'no such LN'
00642             else:
00643                 decision = make_decision_metadata(metadata, auth_request)
00644                 if decision != arc.DECISION_PERMIT:
00645                     success = 'denied'
00646                 else:
00647                     try:
00648                         parentLN, parentGUID = traversedlist[-2]
00649                         # TODO: get the metadata of the parent, and check if the user has permission to removeEntry from it
00650                         mod_requests = {'unmake' : (parentGUID, 'unset', 'entries', traversedlist[-1][0], '')}
00651                         mod_response = self.librarian.modifyMetadata(mod_requests)
00652                         success = mod_response['unmake']
00653                     except IndexError:
00654                         # it has no parent
00655                         success = 'unset'
00656                     if success == 'unset':
00657                         success = self.librarian.remove({'unmake' : GUID})['unmake']
00658             response[rID] = success
00659         return response
00660 
00661     def makeMountpoint(self, auth, requests):
00662         """ Create a new Mountpoint.
00663                 makeMountpoint(requests)
00664                 requests is dictionary with requestID as key and (Logical Name, metadata) as value        """
00665         # do traverse all the requests
00666         requests, traverse_response = self._traverse(requests)
00667         response = {}
00668         
00669         for rID, (LN, child_metadata, URL) in requests:
00670             # for each request first split the Logical Name
00671             rootguid, _, child_name = splitLN(LN)
00672             metadata, GUID, traversedLN, restLN, wasComplete, traversedlist = traverse_response[rID]
00673             #print metadata[('entry', 'type')]
00674             if metadata.get(('entry', 'type'), '') == 'mountpoint':
00675                 success = 'cannot create anything in mountpoint'
00676                 response[rID] = success
00677                 return response
00678 
00679             log.msg(arc.VERBOSE, 'LN', LN, 'URL', URL, 'metadata', metadata, 'GUID', GUID, 'traversedLN', traversedLN, 'restLN', restLN, 'wasComplete',wasComplete, 'traversedlist', traversedlist)
00680             child_metadata[('entry','owner')] = auth.get_identity()
00681             child_metadata[('entry','type')] = 'mountpoint'
00682             child_metadata[('mountpoint','externalURL')] = URL 
00683             if wasComplete: # this means the LN exists
00684                 success = 'LN exists'
00685             elif child_name == '': # this only can happen if the LN was a single GUID
00686                 # this means the collection has no parent
00687                 child_metadata[('entry','GUID')] = rootguid or global_root_guid
00688                 success, _ = self._new(auth, child_metadata)
00689             elif restLN != child_name or GUID == '':
00690                 success = 'parent does not exist'
00691             else:
00692                 # if everything is OK, create the new collection
00693                 #   here GUID is of the parent collection
00694                 success, _ = self._new(auth, child_metadata, child_name, GUID, metadata)
00695             response[rID] = success
00696         return response
00697 
00698     ###   ###
00699 
00700 
00701     def list(self, auth, requests, neededMetadata = []):
00702         """ List the contents of a collection.
00703         
00704         list(requests, neededMetadata = [])
00705         
00706         requests is a dictionary with requestID as key and Logical Name as value
00707         neededMetadata is a list of (section, property) where property could be empty which means all properties of that section
00708             if neededMetadata is empty it means we need everything
00709         """
00710         auth_request = auth.get_request('read')
00711         #print 'ID: '+auth.get_identity()
00712         # do traverse the requested Logical Names
00713         requests, traverse_response = self._traverse(requests)
00714         response = {}
00715         for requestID, [LN] in requests:
00716             try:
00717                 # for each LN
00718                 metadata, GUID, traversedLN, restLN, wasComplete, traversedlist = traverse_response[requestID]
00719                 #print 'metadata'
00720                 #print metadata
00721                 if wasComplete:
00722                     # this means the LN exists
00723                     decision = make_decision_metadata(metadata, auth_request)
00724                     if decision != arc.DECISION_PERMIT:
00725                         entries = {}
00726                         status = 'denied'
00727                     else:
00728                         # let's get the type
00729                         type = metadata[('entry', 'type')]
00730                         if type == 'file': # files have no contents, we do not list them
00731                             status = 'is a file'
00732                             entries = {}
00733                         elif type == 'mountpoint':
00734                             url = metadata[('mountpoint', 'externalURL')]
00735                             res = self._externalStore(auth, url, 'list')[url]
00736                             status = res['status']
00737                             entries = dict([(name, ('', {})) for name in res['list']])
00738                         else: #if it is not a file, it must be a collection (currently there is no other type)
00739                             status = 'found'
00740                             # get all the properties and values from the 'entries' metadata section of the collection
00741                             #   these are the names and GUIDs: the contents of the collection
00742                             GUIDs = dict([(name, GUID)
00743                                 for (section, name), GUID in metadata.items() if section == 'entries'])
00744                             # get the needed metadata of all the entries
00745                             metadata = self.librarian.get(GUIDs.values(), neededMetadata)
00746                             # create a dictionary with the name of the entry as key and (GUID, metadata) as value
00747                             entries = dict([(name, (GUID, metadata[GUID])) for name, GUID in GUIDs.items()])
00748                 elif metadata.get(('entry', 'type'), '') == 'mountpoint':
00749                     url = metadata[('mountpoint', 'externalURL')] + '/' + restLN
00750                     res = self._externalStore(auth, url, 'list')[url]
00751                     status = res['status']
00752                     entries = dict([(name, ('', {})) for name in res['list']])
00753                 else:
00754                     entries = {}
00755                     status = 'not found'
00756             except Exception, e:
00757                 entries = {}
00758                 status = 'internal error (%s)' % traceback.format_exc()
00759             response[requestID] = (entries, status)
00760         return response
00761 
00762     def move(self, auth, requests):
00763         """ Move a file or collection within the global namespace.
00764         
00765         move(requests)
00766         
00767         requests is a dictionary with requestID as key and
00768             (sourceLN, targetLN, preserverOriginal) as value
00769         if preserverOriginal is true this method creates a hard link instead of moving
00770         """
00771         traverse_request = {}
00772         # create a traverse request, each move request needs two traversing: source and target
00773         for requestID, (sourceLN, targetLN, _) in requests.items():
00774             # from one requestID we create two: one for the source and one for the target
00775             traverse_request[requestID + 'source'] = remove_trailing_slash(sourceLN)
00776             traverse_request[requestID + 'target'] = targetLN
00777         traverse_response = self.librarian.traverseLN(traverse_request)
00778         log.msg(arc.VERBOSE, '\/\/', traverse_response)
00779         response = {}
00780         for requestID, (sourceLN, targetLN, preserveOriginal) in requests.items():
00781             sourceLN = remove_trailing_slash(sourceLN)
00782             # for each request
00783             log.msg(arc.VERBOSE, requestID, sourceLN, targetLN, preserveOriginal)
00784             # get the old and the new name of the entry, these are the last elements of the Logical Names
00785             _, _, old_child_name = splitLN(sourceLN)
00786             _, _, new_child_name = splitLN(targetLN)
00787             # get the GUID of the source LN from the traverse response
00788             _, sourceGUID, _, _, sourceWasComplete, sourceTraversedList \
00789                 = traverse_response[requestID + 'source']
00790             # get the GUID form the target's traverse response, this should be the parent of the target LN
00791             targetMetadata, targetGUID, _, targetRestLN, targetWasComplete, targetTraversedList \
00792                 = traverse_response[requestID + 'target']
00793             if targetMetadata.get(('entry', 'type'), '') == 'mountpoint':
00794                 success = 'moving into a mountpoint is not supported'
00795                 response[requestID] = success
00796                 return response
00797             # if the source traverse was not complete: the source LN does not exist
00798             if not sourceWasComplete:
00799                 success = 'nosuchLN'
00800             # if the target traverse was complete: the target LN already exists
00801             #   but if the new_child_name was empty, then the target is considered to be a collection, so it is OK
00802             elif targetWasComplete and new_child_name != '':
00803                 success = 'targetexists'
00804             # if the target traverse was not complete, and the non-traversed part is not just the new name
00805             # (which means that the parent collection does not exist)
00806             # or the target's traversed list is empty (which means that it has no parent: it's just a single GUID)
00807             elif not targetWasComplete and (targetRestLN != new_child_name or len(targetTraversedList) == 0):
00808                 success = 'invalidtarget'
00809             elif sourceGUID in [guid for (_, guid) in targetTraversedList]:
00810                 # if the the target is within the source's subtree, we cannot move it
00811                 success = 'invalidtarget'
00812             else:
00813                 # if the new child name is empty that means that the target LN has a trailing slash
00814                 #   so we just put the old name after it
00815                 if new_child_name == '':
00816                     new_child_name = old_child_name
00817                 decision = make_decision_metadata(targetMetadata, auth.get_request('addEntry'))
00818                 if decision != arc.DECISION_PERMIT:
00819                     success = 'adding child to parent denied'
00820                 else:
00821                     log.msg(arc.VERBOSE, 'adding', sourceGUID, 'to parent', targetGUID)
00822                     # adding the entry to the new parent
00823                     mm_resp = self.librarian.modifyMetadata(
00824                         {'move-target' : (targetGUID, 'add', 'entries', new_child_name, sourceGUID),
00825                         'move-target-parent' : (sourceGUID, 'set', 'parents', '%s/%s' % (targetGUID, new_child_name), 'parent'),
00826                         'move-target-closed?' : (targetGUID, 'setifvalue=yes', 'states', 'closed', 'broken')})
00827                     mm_succ = mm_resp['move-target']
00828                     if mm_succ != 'set':
00829                         success = 'failed adding child to parent'
00830                     else:
00831                         if preserveOriginal == true:
00832                             success = 'moved'
00833                         else:
00834                             # then we need to remove the source LN
00835                             # get the parent of the source: the source traverse has a list of the GUIDs of all the element along the path
00836                             source_parent_guid = sourceTraversedList[-2][1]
00837                             try:
00838                                 source_parent_metadata = self.librarian.get([source_parent_guid])[source_parent_guid]
00839                                 decision = make_decision_metadata(source_parent_metadata, auth.get_request('removeEntry'))
00840                                 if decision == arc.DECISION_PERMIT:
00841                                     log.msg(arc.VERBOSE, 'removing', sourceGUID, 'from parent', source_parent_guid)
00842                                     # delete the entry from the source parent
00843                                     mm_resp = self.librarian.modifyMetadata(
00844                                         {'move-source' : (source_parent_guid, 'unset', 'entries', old_child_name, ''),
00845                                         'move-source-parent' : (sourceGUID, 'unset', 'parents', '%s/%s' % (source_parent_guid, old_child_name), ''),
00846                                         'move-source-closed?' : (source_parent_guid, 'setifvalue=yes', 'states', 'closed', 'broken')})
00847                                     mm_succ = mm_resp['move-source']
00848                                     if mm_succ != 'unset':
00849                                         success = 'failed to remove original link'
00850                                         # TODO: need some handling; remove the new entry or something
00851                                     else:
00852                                         success = 'moved'
00853                                 else:
00854                                     success = 'denied to remove original link'
00855                             except:
00856                                 success = 'error while removing original link'
00857             response[requestID] = success
00858         return response
00859 
00860     def modify(self, auth, requests):
00861         requests, traverse_response = self._traverse(requests)
00862         librarian_requests = {}
00863         not_found = []
00864         denied = []
00865         for changeID, (LN, changeType, section, property, value) in requests:
00866             metadata, GUID, _, _, wasComplete, _ = traverse_response[changeID]
00867             if wasComplete:
00868                 if section == 'states':
00869                     decision = make_decision_metadata(metadata, auth.get_request('modifyStates'))
00870                     # TODO: do this instead with conditions in the Librarian
00871                     if property == 'closed':
00872                         current_state = metadata.get(('states','closed'),'no')
00873                         if value == 'no':
00874                             if current_state != 'no':
00875                                 decision = arc.DECISION_DENY
00876                         elif value == 'yes':
00877                             if current_state not in ['no', 'yes']:
00878                                 decision = arc.DECISION_DENY
00879                         else:
00880                             decision = arc.DECISION_DENY
00881                 elif section == 'metadata':
00882                     decision = make_decision_metadata(metadata, auth.get_request('modifyMetadata'))
00883                 elif section == 'policy':
00884                     decision = make_decision_metadata(metadata, auth.get_request('modifyPolicy'))
00885                 else:
00886                     decision = arc.DECISION_DENY
00887                 if decision == arc.DECISION_PERMIT:
00888                     librarian_requests[changeID] = (GUID, changeType, section, property, value)
00889                 else:
00890                     denied.append(changeID)
00891             else:
00892                 not_found.append(changeID)
00893         response = self.librarian.modifyMetadata(librarian_requests)
00894         for changeID in not_found:
00895             response[changeID] = 'no such LN'
00896         for changeID in denied:
00897             response[changeID] = 'denied'
00898         return response
00899 
00900 from arcom.service import Service
00901 
00902 class BartenderService(Service):
00903 
00904     def __init__(self, cfg):
00905         self.service_name = 'Bartender'
00906         #bar_request_names is the list of the names of the provided methods
00907         bar_request_names = ['stat','makeMountpoint','unmakeMountpoint', 'unmakeCollection', 'makeCollection', 'list', 'move', 'putFile', 'getFile', 'addReplica', 'delFile', 'modify', 'unlink', 'removeCredentials']
00908         # bartender_uri is the URI of the Bartender service namespace, and 'bar' is the prefix we want to use for this namespace
00909         bar_request_type = {'request_names' : bar_request_names,
00910             'namespace_prefix': 'bar', 'namespace_uri': bartender_uri} 
00911         # deleg_request_names contains the methods for delegation
00912         deleg_request_names = ['DelegateCredentialsInit', 'UpdateCredentials']
00913         # 'deleg' is the prefix for the delegation
00914         deleg_request_type = {'request_names' : deleg_request_names,
00915             'namespace_prefix': 'deleg', 'namespace_uri': 'http://www.nordugrid.org/schemas/delegation'}
00916         request_config = [deleg_request_type, bar_request_type]
00917         # call the Service's constructor
00918         Service.__init__(self, request_config, cfg, start_service = False)
00919         # get the path to proxy store
00920         self.proxy_store = str(cfg.Get('ProxyStore'))
00921         try:
00922             if not os.path.exists(self.proxy_store):
00923                 os.mkdir(self.proxy_store)
00924             log.msg(arc.VERBOSE, 'Proxy store:', self.proxy_store)
00925         except:
00926             self.proxy_store = ''
00927         if not self.proxy_store:
00928             log.msg(arc.ERROR, 'The directory for storing proxies is not available. Proxy delegation disabled.')
00929         self.bartender = Bartender(cfg, self.ssl_config, self.state)
00930         
00931     def stat(self, inpayload):
00932         # incoming SOAP message example:
00933         #
00934         #   <bar:stat>
00935         #       <bar:statRequestList>
00936         #           <bar:statRequestElement>
00937         #               <bar:requestID>0</bar:requestID>
00938         #               <bar:LN>/</bar:LN>
00939         #           </bar:statRequestElement>
00940         #       </bar:statRequestList>
00941         #   </bar:stat>
00942         #
00943         # outgoing SOAP message example:
00944         #
00945         #   <bar:statResponse>
00946         #       <bar:statResponseList>
00947         #           <bar:statResponseElement>
00948         #              <bar:requestID>0</bar:requestID>
00949         #               <bar:metadataList>
00950         #                   <bar:metadata>
00951         #                       <bar:section>states</bar:section>
00952         #                       <bar:property>closed</bar:property>
00953         #                       <bar:value>0</bar:value>
00954         #                   </bar:metadata>
00955         #                   <bar:metadata>
00956         #                       <bar:section>entries</bar:section>
00957         #                       <bar:property>testfile</bar:property>
00958         #                       <bar:value>cf05727b-73f3-4318-8454-16eaf10f302c</bar:value>
00959         #                   </bar:metadata>
00960         #                   <bar:metadata>
00961         #                       <bar:section>entry</bar:section>
00962         #                       <bar:property>type</bar:property>
00963         #                       <bar:value>collection</bar:value>
00964         #                   </bar:metadata>
00965         #               </bar:metadataList>
00966         #           </bar:statResponseElement>
00967         #       </bar:statResponseList>
00968         #   </bar:statResponse>
00969 
00970         # get all the requests
00971         request_nodes = get_child_nodes(inpayload.Child().Child())
00972         # get the requestID and LN of each request and create a dictionary where the requestID is the key and the LN is the value
00973         requests = dict([(
00974             str(request_node.Get('requestID')),
00975             [str(request_node.Get('LN'))]
00976             ) for request_node in request_nodes
00977         ])
00978         # call the Bartender class
00979         response = self.bartender.stat(inpayload.auth, requests)
00980         # create the metadata XML structure of each request
00981         for requestID, metadata in response.items():
00982             response[requestID] = create_metadata(metadata, 'bar')
00983         # create the response message with the requestID and the metadata for each request
00984         return create_response('bar:stat',
00985             ['bar:requestID', 'bar:metadataList'], response, self._new_soap_payload(), single = True)
00986 
00987     def delFile(self, inpayload):
00988         # incoming SOAP message example:
00989         #
00990         #   <bar:delFile>
00991         #       <bar:delFileRequestList>
00992         #           <bar:delFileRequestElement>
00993         #               <bar:requestID>0</bar:requestID>
00994         #               <bar:LN>/</bar:LN>
00995         #           </bar:delFileRequestElement>
00996         #       </bar:delFileRequestList>
00997         #   </bar:delFile>
00998         #
00999         # outgoing SOAP message example:
01000         #
01001         #   <soap-env:Envelope>
01002         #       <soap-env:Body>
01003         #           <bar:delFileResponse>
01004         #               <bar:delFileResponseList>
01005         #                   <bar:delFileResponseElement>
01006         #                       <bar:requestID>0</bar:requestID>
01007         #                       <bar:success>deleted</bar:success>
01008         #                   </bar:delFileResponseElement>
01009         #               </bar:delFileResponseList>
01010         #           </bar:delFileResponse>
01011         #       </soap-env:Body>
01012         #   </soap-env:Envelope>
01013 
01014         request_nodes = get_child_nodes(inpayload.Child().Child())
01015         # get the requestID and LN of each request and create a dictionary where the requestID is the key and the LN is the value
01016         requests = dict([
01017             (str(request_node.Get('requestID')), str(request_node.Get('LN')))
01018                 for request_node in request_nodes
01019         ])
01020         response = self.bartender.delFile(inpayload.auth, requests)
01021         return create_response('bar:delFile',
01022             ['bar:requestID', 'bar:success'], response, self._new_soap_payload(), single=True)
01023 
01024 
01025     def getFile(self, inpayload):
01026         # incoming SOAP message example:
01027         #
01028         #   <soap-env:Body>
01029         #       <bar:getFile>
01030         #           <bar:getFileRequestList>
01031         #               <bar:getFileRequestElement>
01032         #                   <bar:requestID>0</bar:requestID>
01033         #                   <bar:LN>/testfile</bar:LN>
01034         #                   <bar:protocol>byteio</bar:protocol>
01035         #               </bar:getFileRequestElement>
01036         #           </bar:getFileRequestList>
01037         #       </bar:getFile>
01038         #   </soap-env:Body>
01039         #
01040         # outgoing SOAP message example:
01041         #
01042         #   <soap-env:Envelope>
01043         #       <soap-env:Body>
01044         #           <bar:getFileResponse>
01045         #               <bar:getFileResponseList>
01046         #                   <bar:getFileResponseElement>
01047         #                       <bar:requestID>0</bar:requestID>
01048         #                       <bar:success>done</bar:success>
01049         #                       <bar:TURL>http://localhost:60000/byteio/12d86ba3-99d5-408e-91d1-35f7c47774e4</bar:TURL>
01050         #                       <bar:protocol>byteio</bar:protocol>
01051         #                   </bar:getFileResponseElement>
01052         #               </bar:getFileResponseList>
01053         #           </bar:getFileResponse>
01054         #       </soap-env:Body>
01055         #   </soap-env:Envelope>
01056 
01057         request_nodes = get_child_nodes(inpayload.Child().Child())
01058         requests = dict([
01059             (
01060                 str(request_node.Get('requestID')), 
01061                 ( # get the LN and all the protocols for each request and put them in a list
01062                     str(request_node.Get('LN')),
01063                     [str(node) for node in request_node.XPathLookup('//bar:protocol', self.ns)]
01064                 )
01065             ) for request_node in request_nodes
01066         ])
01067         response = self.bartender.getFile(inpayload.auth, requests)
01068         return create_response('bar:getFile',
01069             ['bar:requestID', 'bar:success', 'bar:TURL', 'bar:protocol'], response, self._new_soap_payload())
01070     
01071     def addReplica(self, inpayload):
01072         # incoming SOAP message example:
01073         #
01074         #   <soap-env:Body>
01075         #       <bar:addReplica>
01076         #           <bar:addReplicaRequestList>
01077         #               <bar:putReplicaRequestElement>
01078         #                   <bar:requestID>0</bar:requestID>
01079         #                   <bar:GUID>cf05727b-73f3-4318-8454-16eaf10f302c</bar:GUID>
01080         #               </bar:putReplicaRequestElement>
01081         #           </bar:addReplicaRequestList>
01082         #           <bar:protocol>byteio</bar:protocol>
01083         #       </bar:addReplica>
01084         #   </soap-env:Body>
01085         #
01086         # outgoing SOAP message example:
01087         #
01088         #   <soap-env:Envelope>
01089         #       <soap-env:Body>
01090         #           <bar:addReplicaResponse>
01091         #               <bar:addReplicaResponseList>
01092         #                   <bar:addReplicaResponseElement>
01093         #                       <bar:requestID>0</bar:requestID>
01094         #                       <bar:success>done</bar:success>
01095         #                       <bar:TURL>http://localhost:60000/byteio/f568be18-26ae-4925-ae0c-68fe023ef1a5</bar:TURL>
01096         #                       <bar:protocol>byteio</bar:protocol>
01097         #                   </bar:addReplicaResponseElement>
01098         #               </bar:addReplicaResponseList>
01099         #           </bar:addReplicaResponse>
01100         #       </soap-env:Body>
01101         #   </soap-env:Envelope>
01102         
01103         # get the list of supported protocols
01104         protocols = [str(node) for node in inpayload.XPathLookup('//bar:protocol', self.ns)]
01105         # get the GUID of each request
01106         request_nodes = get_child_nodes(inpayload.Child().Get('addReplicaRequestList'))
01107         requests = dict([(str(request_node.Get('requestID')), str(request_node.Get('GUID')))
01108                 for request_node in request_nodes])
01109         response = self.bartender.addReplica(inpayload.auth, requests, protocols)
01110         return create_response('bar:addReplica',
01111             ['bar:requestID', 'bar:success', 'bar:TURL', 'bar:protocol'], response, self._new_soap_payload())
01112     
01113     def putFile(self, inpayload):
01114         # incoming SOAP message example:
01115         #
01116         #   <soap-env:Body>
01117         #       <bar:putFile>
01118         #           <bar:putFileRequestList>
01119         #               <bar:putFileRequestElement>
01120         #                   <bar:requestID>0</bar:requestID>
01121         #                   <bar:LN>/testfile2</bar:LN>
01122         #                   <bar:metadataList>
01123         #                       <bar:metadata>
01124         #                           <bar:section>states</bar:section>
01125         #                           <bar:property>neededReplicas</bar:property>
01126         #                           <bar:value>2</bar:value>
01127         #                       </bar:metadata>
01128         #                       <bar:metadata>
01129         #                           <bar:section>states</bar:section>
01130         #                           <bar:property>size</bar:property>
01131         #                           <bar:value>11</bar:value>
01132         #                       </bar:metadata>
01133         #                   </bar:metadataList>
01134         #                   <bar:protocol>byteio</bar:protocol>
01135         #               </bar:putFileRequestElement>
01136         #           </bar:putFileRequestList>
01137         #       </bar:putFile>
01138         #   </soap-env:Body>
01139         #
01140         # outgoing SOAP message example:
01141         #
01142         #   <soap-env:Envelope>
01143         #       <soap-env:Body>
01144         #           <bar:putFileResponse>
01145         #               <bar:putFileResponseList>
01146         #                   <bar:putFileResponseElement>
01147         #                       <bar:requestID>0</bar:requestID>
01148         #                       <bar:success>done</bar:success>
01149         #                       <bar:TURL>http://localhost:60000/byteio/b8f2987b-a718-47b3-82bb-e838470b7e00</bar:TURL>
01150         #                       <bar:protocol>byteio</bar:protocol>
01151         #                   </bar:putFileResponseElement>
01152         #               </bar:putFileResponseList>
01153         #           </bar:putFileResponse>
01154         #       </soap-env:Body>
01155         #   </soap-env:Envelope>
01156 
01157         request_nodes = get_child_nodes(inpayload.Child().Child())
01158         requests = dict([
01159             (
01160                 str(request_node.Get('requestID')), 
01161                 (
01162                     str(request_node.Get('LN')),
01163                     parse_metadata(request_node.Get('metadataList')),
01164                     [str(node) for node in request_node.XPathLookup('//bar:protocol', self.ns)]
01165                 )
01166             ) for request_node in request_nodes
01167         ])
01168         response = self.bartender.putFile(inpayload.auth, requests)
01169         return create_response('bar:putFile',
01170             ['bar:requestID', 'bar:success', 'bar:TURL', 'bar:protocol'], response, self._new_soap_payload())
01171 
01172     def unlink(self, inpayload):
01173         """docstring for unlink"""
01174         request_nodes = get_child_nodes(inpayload.Child().Child())
01175         requests = dict([(
01176                 str(request_node.Get('requestID')),
01177                 [str(request_node.Get('LN'))]
01178             ) for request_node in request_nodes
01179         ])
01180         response = self.bartender.unlink(inpayload.auth, requests)
01181         return create_response('bar:unlink',
01182             ['bar:requestID', 'bar:success'], response, self._new_soap_payload(), single = True)
01183     
01184 
01185     def unmakeCollection(self, inpayload):
01186         request_nodes = get_child_nodes(inpayload.Child().Child())
01187         requests = dict([(
01188                 str(request_node.Get('requestID')),
01189                 [str(request_node.Get('LN'))]
01190             ) for request_node in request_nodes
01191         ])
01192         response = self.bartender.unmakeCollection(inpayload.auth, requests)
01193         return create_response('bar:unmakeCollection',
01194             ['bar:requestID', 'bar:success'], response, self._new_soap_payload(), single = True)
01195 
01196     def makeCollection(self, inpayload):
01197         # incoming SOAP message example:
01198         # 
01199         #   <soap-env:Body>
01200         #       <bar:makeCollection>
01201         #           <bar:makeCollectionRequestList>
01202         #               <bar:makeCollectionRequestElement>
01203         #                   <bar:requestID>0</bar:requestID>
01204         #                   <bar:LN>/testdir</bar:LN>
01205         #                   <bar:metadataList>
01206         #                       <bar:metadata>
01207         #                           <bar:section>states</bar:section>
01208         #                           <bar:property>closed</bar:property>
01209         #                           <bar:value>no</bar:value>
01210         #                       </bar:metadata>
01211         #                   </bar:metadataList>
01212         #               </bar:makeCollectionRequestElement>
01213         #           </bar:makeCollectionRequestList>
01214         #       </bar:makeCollection>
01215         #   </soap-env:Body>
01216         #
01217         # outgoing SOAP message example
01218         #
01219         #   <soap-env:Envelope>
01220         #       <soap-env:Body>
01221         #           <bar:makeCollectionResponse>
01222         #               <bar:makeCollectionResponseList>
01223         #                   <bar:makeCollectionResponseElement>
01224         #                       <bar:requestID>0</bar:requestID>
01225         #                       <bar:success>done</bar:success>
01226         #                   </bar:makeCollectionResponseElement>
01227         #               </bar:makeCollectionResponseList>
01228         #           </bar:makeCollectionResponse>
01229         #       </soap-env:Body>
01230         #   </soap-env:Envelope>
01231 
01232         request_nodes = get_child_nodes(inpayload.Child().Child())
01233         requests = dict([
01234             (str(request_node.Get('requestID')), 
01235                 (str(request_node.Get('LN')), parse_metadata(request_node.Get('metadataList')))
01236             ) for request_node in request_nodes
01237         ])
01238         response = self.bartender.makeCollection(inpayload.auth, requests)
01239         return create_response('bar:makeCollection',
01240             ['bar:requestID', 'bar:success'], response, self._new_soap_payload(), single = True)
01241 
01242     ### Created by Salman Toor. ###
01243     
01244     def unmakeMountpoint(self, inpayload):
01245         request_nodes = get_child_nodes(inpayload.Child().Child())
01246         requests = dict([(
01247                 str(request_node.Get('requestID')),
01248                 [str(request_node.Get('LN'))]
01249             ) for request_node in request_nodes
01250         ])
01251         response = self.bartender.unmakeMountpoint(inpayload.auth, requests)
01252         return create_response('bar:unmakeMountpoint',
01253             ['bar:requestID', 'bar:success'], response, self._new_soap_payload(), single = True)
01254 
01255     def makeMountpoint(self, inpayload):
01256         # incoming SOAP message example:
01257         # 
01258         #   <soap-env:Body>
01259         #       <bar:makeMountpoint>
01260         #           <bar:makeMountpointRequestList>
01261         #               <bar:makeMountpointRequestElement>
01262         #                   <bar:requestID>0</bar:requestID>
01263         #                   <bar:LN>/testdir</bar:LN>
01264         #                   <bar:URL>URL</bar:URL>
01265         #                   <bar:metadataList>
01266         #                       <bar:metadata>
01267         #                           <bar:section>any</bar:section>
01268         #                           <bar:property>additional</bar:property>
01269         #                           <bar:value>metadata</bar:value>
01270         #                       </bar:metadata>
01271         #                   </bar:metadataList>
01272         #               </bar:makeMountpointRequestElement>
01273         #           </bar:makeMountpointRequestList>
01274         #       </bar:makeMountpoint>
01275         #   </soap-env:Body>
01276         #
01277         # outgoing SOAP message example
01278         #
01279         #   <soap-env:Envelope>
01280         #       <soap-env:Body>
01281         #           <bar:makeMountpointResponse>
01282         #               <bar:makeMountpointResponseList>
01283         #                   <bar:makeMountpointResponseElement>
01284         #                       <bar:requestID>0</bar:requestID>
01285         #                       <bar:success>done</bar:success>
01286         #                   </bar:makeMountpointResponseElement>
01287         #               </bar:makeMountpointResponseList>
01288         #           </bar:makeMountpointResponse>
01289         #       </soap-env:Body>
01290         #   </soap-env:Envelope>
01291 
01292         request_nodes = get_child_nodes(inpayload.Child().Child())
01293         requests = dict([
01294             (str(request_node.Get('requestID')),
01295                 (str(request_node.Get('LN')), parse_metadata(request_node.Get('metadataList')), str(request_node.Get('URL')))
01296             ) for request_node in request_nodes
01297         ])
01298         response = self.bartender.makeMountpoint(inpayload.auth, requests)
01299         return create_response('bar:makeMountpoint',
01300             ['bar:requestID', 'bar:success'], response, self._new_soap_payload(), single = True)
01301         
01302     ###     ####        
01303 
01304     def list(self, inpayload):
01305         # incoming SOAP message example:
01306         #
01307         #   <soap-env:Body>
01308         #       <bar:list>
01309         #           <bar:listRequestList>
01310         #               <bar:listRequestElement>
01311         #                   <bar:requestID>0</bar:requestID>
01312         #                   <bar:LN>/</bar:LN>
01313         #               </bar:listRequestElement>
01314         #           </bar:listRequestList>
01315         #           <bar:neededMetadataList>
01316         #               <bar:neededMetadataElement>
01317         #                   <bar:section>entry</bar:section>
01318         #                   <bar:property></bar:property>
01319         #               </bar:neededMetadataElement>
01320         #           </bar:neededMetadataList>
01321         #       </bar:list>
01322         #   </soap-env:Body>
01323         #
01324         # outgoing SOAP message example:
01325         #
01326         #   <soap-env:Envelope>
01327         #       <soap-env:Body>
01328         #           <bar:listResponse>
01329         #               <bar:listResponseList>
01330         #                   <bar:listResponseElement>
01331         #                       <bar:requestID>0</bar:requestID>
01332         #                       <bar:entries>
01333         #                           <bar:entry>
01334         #                               <bar:name>testfile</bar:name>
01335         #                               <bar:GUID>cf05727b-73f3-4318-8454-16eaf10f302c</bar:GUID>
01336         #                               <bar:metadataList>
01337         #                                   <bar:metadata>
01338         #                                       <bar:section>entry</bar:section>
01339         #                                       <bar:property>type</bar:property>
01340         #                                       <bar:value>file</bar:value>
01341         #                                   </bar:metadata>
01342         #                               </bar:metadataList>
01343         #                           </bar:entry>
01344         #                           <bar:entry>
01345         #                               <bar:name>testdir</bar:name>
01346         #                               <bar:GUID>4cabc8cb-599d-488c-a253-165f71d4e180</bar:GUID>
01347         #                               <bar:metadataList>
01348         #                                   <bar:metadata>
01349         #                                       <bar:section>entry</bar:section>
01350         #                                       <bar:property>type</bar:property>
01351         #                                       <bar:value>collection</bar:value>
01352         #                                   </bar:metadata>
01353         #                               </bar:metadataList>
01354         #                           </bar:entry>
01355         #                       </bar:entries>
01356         #                       <bar:status>found</bar:status>
01357         #                   </bar:listResponseElement>
01358         #               </bar:listResponseList>
01359         #           </bar:listResponse>
01360         #       </soap-env:Body>
01361         #   </soap-env:Envelope>
01362         
01363         requests = parse_node(inpayload.Child().Get('listRequestList'),
01364             ['requestID', 'LN'], single = False)
01365         neededMetadata = [
01366             node_to_data(node, ['section', 'property'], single = True)
01367                 for node in get_child_nodes(inpayload.Child().Get('neededMetadataList'))
01368         ]
01369         response0 = self.bartender.list(inpayload.auth, requests, neededMetadata)
01370         response = dict([
01371             (requestID,
01372             ([('bar:entry', [
01373                 ('bar:name', name),
01374                 ('bar:GUID', GUID),
01375                 ('bar:metadataList', create_metadata(metadata))
01376             ]) for name, (GUID, metadata) in entries.items()],
01377             status)
01378         ) for requestID, (entries, status) in response0.items()])
01379         return create_response('bar:list',
01380             ['bar:requestID', 'bar:entries', 'bar:status'], response, self._new_soap_payload())
01381 
01382     def move(self, inpayload):
01383         # incoming SOAP message example:
01384         #   <soap-env:Body>
01385         #       <bar:move>
01386         #           <bar:moveRequestList>
01387         #               <bar:moveRequestElement>
01388         #                   <bar:requestID>0</bar:requestID>
01389         #                   <bar:sourceLN>/testfile2</bar:sourceLN>
01390         #                   <bar:targetLN>/testdir/</bar:targetLN>
01391         #                   <bar:preserveOriginal>0</bar:preserveOriginal>
01392         #               </bar:moveRequestElement>
01393         #           </bar:moveRequestList>
01394         #       </bar:move>
01395         #   </soap-env:Body>
01396         #
01397         # outgoing SOAP message example:
01398         #
01399         #   <soap-env:Envelope>
01400         #       <soap-env:Body>
01401         #           <bar:moveResponse>
01402         #               <bar:moveResponseList>
01403         #                   <bar:moveResponseElement>
01404         #                       <bar:requestID>0</bar:requestID>
01405         #                       <bar:status>moved</bar:status>
01406         #                   </bar:moveResponseElement>
01407         #               </bar:moveResponseList>
01408         #           </bar:moveResponse>
01409         #       </soap-env:Body>
01410         #   </soap-env:Envelope>
01411 
01412         requests = parse_node(inpayload.Child().Child(),
01413             ['requestID', 'sourceLN', 'targetLN', 'preserveOriginal'])
01414         response = self.bartender.move(inpayload.auth, requests)
01415         return create_response('bar:move',
01416             ['bar:requestID', 'bar:status'], response, self._new_soap_payload(), single = True)
01417 
01418     def modify(self, inpayload):
01419         requests = parse_node(inpayload.Child().Child(), ['bar:changeID',
01420             'bar:LN', 'bar:changeType', 'bar:section', 'bar:property', 'bar:value'])
01421         response = self.bartender.modify(inpayload.auth, requests)
01422         return create_response('bar:modify', ['bar:changeID', 'bar:success'],
01423             response, self._new_soap_payload(), single = True)
01424 
01425     def DelegateCredentialsInit(self,inpayload):
01426         ns = arc.NS('delegation','http://www.nordugrid.org/schemas/delegation')
01427         outpayload = arc.PayloadSOAP(ns)
01428         if self.proxy_store:
01429             #print inpayload.GetXML()
01430             # Delegation Credentials(NEED TO FIX IT)  
01431             self.delegSOAP = arc.DelegationContainerSOAP()
01432             self.delegSOAP.DelegateCredentialsInit(inpayload,outpayload)
01433             #print "\n outpayload"
01434             #print outpayload.GetXML()
01435         return outpayload
01436         
01437     def UpdateCredentials(self,inpayload):
01438         #print inpayload.GetXML()
01439         ns = arc.NS('delegation','http://www.nordugrid.org/schemas/delegation')
01440         outpayload = arc.PayloadSOAP(ns)
01441         credAndid = self.delegSOAP.UpdateCredentials(inpayload,outpayload)
01442         #print credAndid
01443         if credAndid[0] == True:
01444             #print "\n ---Delegated Credentials--- "
01445             #print credAndid[1]
01446             log.msg(arc.INFO,'Delegation status: ', credAndid[0])
01447             if (os.path.isdir(self.proxy_store)):
01448                 #print "ProxyStore: "+self.proxy_store 
01449                 filePath = self.proxy_store+'/'+base64.b64encode(inpayload.auth.get_identity())+'.proxy'
01450                 log.msg(arc.VERBOSE,'creating proxy file : ', filePath)
01451                 proxyfile = open(filePath, 'w') 
01452                 proxyfile.write(credAndid[1])
01453                 proxyfile.close()
01454                 log.msg(arc.VERBOSE,'created successfully, ID: %s',credAndid[2]) 
01455                 os.system('chmod 600 '+filePath)
01456             else:
01457                 log.msg(arc.VERBOSE,'cannot access proxy_store, Check the configuration file (service.xml)\n Need to have a <ProxyStore>')
01458         else:
01459             log.msg(arc.INFO,'Delegation failed: ')
01460         return outpayload
01461         
01462     def removeCredentials(self, inpayload):
01463         response = {}
01464         if self.proxy_store:
01465             log.msg(arc.INFO, 'ID: '+inpayload.auth.get_identity())
01466             message = ''
01467             if (os.path.isdir(self.proxy_store)):
01468                 log.msg(arc.VERBOSE, 'ProxyStore: %s',self.proxy_store)
01469                 filePath = self.proxy_store+'/'+base64.b64encode(inpayload.auth.get_identity())+'.proxy'
01470                 if (os.path.isfile(filePath)):    
01471                     os.system('rm '+filePath)
01472                     message = 'Credential removed successfully'
01473                     status = 'successful' 
01474                 else:
01475                     message =  'cannot access proxy file: '+filePath
01476                     status = 'failed'
01477             else:
01478                 message = 'cannot access proxy_store, Check the configuration file (service.xml)\n Need to have a <ProxyStore>'
01479                 status = 'failed'
01480         else: 
01481             message = 'cannot access proxy_store, Check the configuration file (service.xml)\n Need to have a <ProxyStore>'                
01482             status = 'failed'
01483         #print message
01484         response['message'] = message
01485         response['status'] = status
01486         log.msg(arc.VERBOSE, 'removeCredentials: %s', message)
01487         log.msg(arc.VERBOSE, 'removeCredentials: %s', status)  
01488         return create_response('bar:removeCredentials', ['bar:message','bar:status'],response, self._new_soap_payload(), single = True) 
01489 
01490     def RegistrationCollector(self, doc):
01491         regentry = arc.XMLNode('<RegEntry />')
01492         regentry.NewChild('SrcAdv').NewChild('Type').Set(bartender_servicetype)
01493         #Place the document into the doc attribute
01494         doc.Replace(regentry)
01495         return True
01496 
01497     def GetAdditionalLocalInformation(self, service_node):
01498         service_node.NewChild('Type').Set(bartender_servicetype)