Back to index

nordugrid-arc-nox  1.1.0~rc6
service.py
Go to the documentation of this file.
00001 # namespace URIs of the storage services
00002 ahash_uri = 'http://www.nordugrid.org/schemas/ahash'
00003 librarian_uri = 'http://www.nordugrid.org/schemas/librarian'
00004 bartender_uri = 'http://www.nordugrid.org/schemas/bartender'
00005 shepherd_uri = 'http://www.nordugrid.org/schemas/shepherd'
00006 gateway_uri = 'http://www.nordugrid.org/schemas/gateway'
00007 delegation_uri = 'http://www.nordugrid.org/schemas/delegation'
00008 rbyteio_uri = 'http://schemas.ggf.org/byteio/2005/10/random-access'
00009 
00010 wsrf_rp_uri = 'http://docs.oasis-open.org/wsrf/rp-2'
00011 
00012 
00013 # service type names
00014 ahash_servicetype = 'org.nordugrid.storage.ahash'
00015 librarian_servicetype = 'org.nordugrid.storage.librarian'
00016 bartender_servicetype = 'org.nordugrid.storage.bartender'
00017 shepherd_servicetype = 'org.nordugrid.storage.shepherd'
00018 
00019 # URI for the simple transfer mechanism of ByteIO
00020 byteio_simple_uri = 'http://schemas.ggf.org/byteio/2005/10/transfer-mechanisms/simple'
00021 # True and False values used in the XML representation
00022 true = '1'
00023 false = '0'
00024 # the defaults for TrustManager
00025 default_checking_interval = 600
00026 default_ahash_id = '3'
00027 
00028 import arc
00029 import inspect
00030 import time
00031 import sys
00032 import threading
00033 import random
00034 from arcom import get_child_nodes
00035 from arcom.security import AuthRequest, parse_ssl_config
00036 from arcom.logger import Logger
00037 log = Logger(arc.Logger(arc.Logger_getRootLogger(), 'Storage.Service'))
00038 
00039 class ServiceState:
00040     def __init__(self, running = True):
00041         self.running = running
00042 
00043 class Service:
00044     
00045     def __init__(self, request_config, cfg = None, start_service = True):
00046         self._trust_manager = []
00047         self.ssl_config = {}
00048         self._force_trust = False
00049         self.state = ServiceState(start_service)
00050         if cfg:
00051             self.ssl_config = parse_ssl_config(cfg)
00052             trust_manager_node = cfg.Get('TrustManager')
00053             fromFile = str(trust_manager_node.Attribute('FromFile'))
00054             if fromFile:
00055                 try:
00056                     xml_string = file(fromFile).read()
00057                     trust_manager_node = arc.XMLNode(xml_string)
00058                 except:
00059                     log.msg()
00060                     pass
00061             self._force_trust = str(trust_manager_node.Attribute('Force')) not in ['no', 'No', 'NO']
00062             entries = get_child_nodes(trust_manager_node)
00063             for entry in entries:
00064                 name = entry.Name()
00065                 if name in ['DN', 'CA']:
00066                     self._trust_manager.append({'type': name, 'DNs': [str(entry)]})
00067                 if name in ['DNsFromAHash']:
00068                     try:
00069                         checking_interval = int(str(entry.Attribute('CheckingInterval')))
00070                     except:
00071                         checking_interval = None
00072                     if not checking_interval:
00073                         checking_interval = default_checking_interval
00074                     ahash_id = str(entry.Attribute('ID'))
00075                     if not ahash_id:
00076                         ahash_id = default_ahash_id
00077                     ahashes = get_child_nodes(entry)
00078                     ahash_urls = []
00079                     for ahash in ahashes:
00080                         if ahash.Name() == 'AHashURL':
00081                             ahash_urls.append(str(ahash))
00082                     data = {'type': name, 'DNs' : [], 'URLs' : ahash_urls,
00083                         'checking_interval' : checking_interval, 'ahash_id' : ahash_id}
00084                     self._trust_manager.append(data)
00085                     threading.Thread(target = self._get_dns_from_ahash, args = [data]).start()
00086         if not hasattr(self,'service_name'):
00087             self.service_name = 'Python Service With No Name'
00088         #if self._trust_manager:
00089         #    print self.service_name, "TrustManager:", self._force_trust and 'force' or 'don\'t force', self._trust_manager
00090         log.msg(arc.INFO, "Starting:", self.service_name)
00091         self.request_config = request_config
00092         self.ns = arc.NS(dict([(request_type['namespace_prefix'], request_type['namespace_uri'])
00093             for request_type in self.request_config]))
00094         
00095     def __del__(self):
00096         try:
00097             self.state.running = False
00098         except:
00099             pass
00100         log.msg(arc.INFO, "Stopping:", self.service_name)
00101     
00102     def _get_dns_from_ahash(self, data):
00103         try:
00104             from storage.client import AHashClient
00105         except:
00106             log.msg()
00107         # first just wait a few seconds
00108         time.sleep(10)
00109         while True:
00110             try:
00111                 #print "Start getting a list of DNs from an AHash"
00112                 ahash_url = random.choice(data['URLs'])
00113                 #print "Chosen AHash:", ahash_url
00114                 ahash = AHashClient(ahash_url, ssl_config = self.ssl_config)
00115                 results = ahash.get([data['ahash_id']])[data['ahash_id']]
00116                 data['DNs'] = [DN for (_, DN) in results.keys()]
00117                 #print "data", data
00118                 #print "Done, waiting for %d seconds" % data['checking_interval']
00119                 time.sleep(data['checking_interval'])
00120             except:
00121                 log.msg()
00122                 time.sleep(1)
00123     
00124     def _is_trusted(self, DN, CA):
00125         if not self._trust_manager:
00126             return True
00127         #print '_is_trusted called with', DN, CA
00128         trusted = False
00129         for entry in self._trust_manager:
00130             if entry['type'] == 'DN':
00131                 if DN in entry['DNs']:
00132                     #print DN, 'is listed as trusted'
00133                     trusted = True
00134             if entry['type'] == 'CA':
00135                 if CA in entry['DNs']:
00136                     #print DN, 'has a CA which is listed as trusted'
00137                     trusted = True
00138             if entry['type'] == 'DNsFromAHash':
00139                 if DN in entry['DNs']:
00140                     #print DN, 'is listed as trusted in these AHashes:', ', '.join(entry['URLs'])
00141                     trusted = True
00142         return trusted
00143         
00144     def _get_trusted_dns(self):
00145         dns = []
00146         for entry in self._trust_manager:
00147             if entry['type'] in ['DN' or 'DNsFromAHash']:
00148                 dns.extend(entry['DNs'])
00149         return dns
00150     
00151     def _new_soap_payload(self):
00152         return arc.PayloadSOAP(self.ns)
00153     
00154     def _call_request(self, request_name, inmsg):
00155         inpayload = inmsg.Payload()
00156         auth = AuthRequest(inmsg)
00157         inpayload.auth = auth
00158         if self._force_trust and not self._is_trusted(*auth.get_identity_and_ca()):
00159             raise Exception, 'client is not trusted'
00160         return getattr(self,request_name)(inpayload)
00161     
00162     def GetLocalInformation(self):
00163         ns = arc.NS({'':'http://schemas.ogf.org/glue/2008/05/spec_2.0_d41_r01'})
00164         info = arc.XMLNode(ns,'Domains')
00165         service_node = info.NewChild('AdminDomain').NewChild('Services').NewChild('Service')
00166         endpoint_node = service_node.NewChild('Endpoint')
00167         endpoint_node.NewChild('HealthState').Set('ok')
00168         if self.state.running:
00169             serving_state = 'production'
00170         else:
00171             serving_state = 'closed'
00172         endpoint_node.NewChild('ServingState').Set(serving_state)
00173         try:
00174             self.GetAdditionalLocalInformation(service_node)
00175         except:
00176             pass
00177         return info
00178     
00179     def process(self, inmsg, outmsg):
00180         """ Method to process incoming message and create outgoing one. """
00181         # gets the payload from the incoming message
00182         inpayload = inmsg.Payload()
00183         try:
00184             # the first child of the payload should be the name of the request
00185             request_node = inpayload.Child()
00186             # get the namespace of the request node
00187             request_namespace = request_node.Namespace()
00188             matched_request_types = [request_type for request_type in self.request_config if request_type['namespace_uri'] == request_namespace]
00189             if len(matched_request_types) == 0:
00190                 # check if it is a LIDI request:
00191                 if request_namespace == wsrf_rp_uri:
00192                     outpayload = arc.PayloadSOAP(arc.NS({'wsrf-rp':wsrf_rp_uri}))
00193                     outpayload.NewChild('wsrf-rp:GetResourcePropertyDocumentResponse').NewChild(self.GetLocalInformation())
00194                     outmsg.Payload(outpayload)
00195                     return arc.MCC_Status(arc.STATUS_OK)
00196                 raise Exception, 'wrong namespace. expected: %s' % ', '.join([request_type['namespace_uri'] for request_type in self.request_config])
00197             current_request_type = matched_request_types[0]
00198             # get the name of the request without the namespace prefix
00199             request_name = request_node.Name()
00200             if request_name not in current_request_type['request_names']:
00201                 # if the name of the request is not in the list of supported request names
00202                 raise Exception, 'wrong request (%s)' % request_name
00203             log.msg(arc.VERBOSE,'%(sn)s.%(rn)s called' % {'sn':self.service_name, 'rn':request_name})
00204             if not self.state.running:
00205                 outpayload = arc.PayloadSOAP(self.ns, True)
00206                 fault = outpayload.Fault()
00207                 fault.Reason('%s service is inactive (not initialized yet or shutting down)' % self.service_name) 
00208                 outmsg.Payload(outpayload)
00209                 return arc.MCC_Status(arc.STATUS_OK)
00210             # if the request name is in the supported names,
00211             # then this class should have a method with this name
00212             # the 'getattr' method returns this method
00213             # which then we could call with the incoming payload
00214             # and which will return the response payload
00215             log.msg(arc.DEBUG, inpayload.GetXML())
00216             outpayload = self._call_request(request_name, inmsg)
00217             # sets the payload of the outgoing message
00218             outmsg.Payload(outpayload)
00219             # return with the STATUS_OK status
00220             return arc.MCC_Status(arc.STATUS_OK)
00221         except:
00222             # if there is any exception, print it
00223             msg = log.msg()
00224             outpayload = arc.PayloadSOAP(self.ns, True)
00225             fault = outpayload.Fault()
00226             fault.Reason('%s service raised a %s' % (self.service_name, msg))
00227             outmsg.Payload(outpayload)
00228             return arc.MCC_Status(arc.STATUS_OK)
00229 
00230 def parse_node(node, names, single = False, string = True):
00231     """ Call node_to_data() for each child of the given node.
00232     
00233     parse_node(node, names, single = False, string = True)
00234     
00235     node is the XMLNode whose children we want to convert
00236     names is a list of tag names which will be returned in the specified order
00237     single indicates that we need only one value beside the key, do not put it into a list
00238     string indicates that we need the string data of the nodes, not the nodes itself.
00239     
00240     Example:
00241     
00242         xml = XMLNode('''
00243             <statRequestList>
00244                 <statRequestElement>
00245                     <requestID>0</requestID>
00246                     <LN>/</LN>
00247                 </statRequestElement>
00248                 <statRequestElement>
00249                     <requestID>1</requestID>
00250                     <LN>/testfile</LN>
00251                 </statRequestElement>
00252             </statRequestList>
00253         ''')
00254       
00255     parse_node(xml, ['requestID','LN']) returns:
00256         
00257         {'0': ['/'], '1': ['/testfile']}
00258 
00259 
00260     parse_node(xml, ['requestID','LN'], single = True) returns:
00261         
00262         {'0': '/', '1': '/testfile'}
00263 
00264 
00265     parse_node(xml, ['LN','requestID'], True) returns:
00266         
00267         {'/': '0', '/testfile': '1'}
00268 
00269 
00270     parse_node(xml, ['requestID','LN','LN']) returns:
00271         
00272         {'0': ['/', '/'], '1': ['/testfile', '/testfile']}
00273 
00274 
00275     """
00276     return dict([
00277         node_to_data(n, names, single, string)
00278             for n in get_child_nodes(node)
00279     ])
00280 
00281 def parse_to_dict(node, names):
00282     """ Convert the children of the node to a dictionary of dictionaries.
00283     
00284     parse_to_dict(node, names)
00285     
00286     node is the XMLNode whose children we want to convert
00287     names is a list of tag names, for each child only these names will be included in the dictionary
00288     
00289     Example:
00290     
00291         <statResponseList>
00292             <statResponseElement>
00293                 <requestID>123</requestID>
00294                 <referenceID>abdad</referenceID>
00295                 <state>alive</state>
00296                 <size>871432</size>
00297             </statResponseElement>
00298             <statResponseElement>
00299                 <requestID>456</requestID>
00300                 <referenceID>fefeg</referenceID>
00301                 <state>alive</state>
00302                 <size>945</size>
00303             </statResponseElement>
00304         </statResponseList>
00305         
00306     parse_to_dict(xml, ['requestID', 'state', 'size']) returns:
00307 
00308         {'123': {'size': '871432', 'state': 'alive'},
00309          '456': {'size': '945', 'state': 'alive'}}
00310 
00311 
00312     parse_to_dict(xml, ['referenceID','requestID', 'state', 'size']) returns:
00313     
00314         {'abdad': {'requestID': '123', 'size': '871432', 'state': 'alive'},
00315          'fefeg': {'requestID': '456', 'size': '945', 'state': 'alive'}}
00316     """
00317     return dict([(str(n.Get(names[0])), dict([(name, str(n.Get(name))) for name in names[1:]]))
00318         for n in get_child_nodes(node)])
00319 
00320 def create_response(method_name, tag_names, elements, payload, single = False):
00321     """ Creates an XMLNode payload from a dictionary of tag names and list of values.
00322     
00323     create_response(method_name, tag_names, elements, payload, single = False)
00324     
00325     method_name is the name of the method which will be used as a prefix in the name of the 'Response' tag
00326     tag_names is a list of names which will be used in the specified order as tag names
00327     elements is a dictionary where the key will be tagged as the first tag name,
00328         and the value is a list whose items will be tagged in the order of the tag_names list
00329     payload is an XMLNode, the response will be added to that
00330     single indicates if there is only one value per key
00331     
00332     Example:
00333     
00334         elements = {'123': ['alive', '871432'], '456': ['alive', '945']}
00335         tag_names = ['requestID', 'state', 'size']
00336         method_name = 'stat'
00337         payload = arc.PayloadSOAP(arc.NS())
00338 
00339     after create_response(method_name, tag_names, elements, payload, single = False) payload will contain:
00340     
00341         <statResponse>
00342             <statResponseList>
00343                 <statResponseElement>
00344                     <requestID>123</requestID>
00345                     <state>alive</state>
00346                     <size>871432</size>
00347                 </statResponseElement>
00348                 <statResponseElement>
00349                     <requestID>456</requestID>
00350                     <state>alive</state>
00351                     <size>945</size>
00352                 </statResponseElement>
00353             </statResponseList>
00354         </statResponse>
00355 
00356     The method_name used to prefix the 'Response' the 'ResponseList' and 'ResponseElement' node names.
00357     We could say
00358         method_name = 'ns:stat'
00359         tag_names = ['ns:requestID', 'ns:state', 'ns:size']
00360     if we want namespace prefixes.
00361     """
00362     # first create an XMLTree, then add it to the payload XMLNode
00363     from arcom.xmltree import XMLTree
00364     if single:
00365         # if there is only a single value for each key
00366         tree = XMLTree(from_tree =
00367             (method_name + 'ResponseList', [
00368                 (method_name + 'ResponseElement', [
00369                     (tag_names[0], key),
00370                     (tag_names[1], value)
00371                 ]) for key, value in elements.items()
00372             ])
00373         )
00374     else:
00375         # if there is more values for a key
00376         tree = XMLTree(from_tree =
00377             (method_name + 'ResponseList', [
00378                 (method_name + 'ResponseElement', [
00379                     (tag_names[0], key) # tag the key with the first item in tag_names
00380                 ] + [ # for each item in the values list pick the next name from tag_names
00381                     (tag_names[i + 1], values[i]) for i in range(len(values))
00382                 ]) for key, values in elements.items()
00383             ])
00384         )
00385     # create a <method_name>Response child node in the payload
00386     response_node = payload.NewChild(method_name + 'Response')
00387     # add the XMLTree to this newly created node
00388     tree.add_to_node(response_node)
00389     # return the payload XMLNode
00390     return payload
00391 
00392 def node_to_data(node, names, single = False, string = True):
00393     """ Get some children of an XMLNode and return them in a list in the specified order using the first one as a key.
00394     
00395     node_to_data(node, names, single = False, string = True)
00396     
00397     node is an XMLNode which has some children
00398     names is a list of strings, the names of the children we want to extract, the first name always will be a key
00399     single is a boolean indicating if we want only a single value thus do not put it in a list
00400     string is a boolean indicating if we want the string values of the nodes or the nodes itself
00401     
00402     Example:
00403     
00404         node:
00405             <changeRequest>
00406                 <changeID>0</changeID>
00407                 <ID>123</ID>
00408                 <section>states</section>
00409                 <property>neededReplicas</property>
00410                 <value>3</value>
00411                 <somethingElse>not interesting</somethingElse>
00412                 <changeType>set</changeType>
00413             </changeRequest>
00414             
00415         names: ['changeID', 'ID', 'changeType', 'section', 'property', 'value']
00416         
00417         here changeID will be the key, and all the other names will be in a list in the specified order
00418         
00419         so it returns ('0', ['123', 'set', 'states', 'neededReplicas', '3'])
00420         
00421             ('somethingElse' is not returned)
00422     
00423     Example:
00424     
00425         node:
00426             <getRequest>
00427                 <GUID>11</GUID>
00428                 <requestID>99</requestID>
00429             </getRequest>
00430             
00431         names: ['requestID', 'GUID']
00432         single: True
00433         
00434         here requestID will be the key, and GUID is the single value which won't be in a list
00435         
00436         so it returns ('99', '11')
00437         
00438             (instead of '99', ['11'])
00439     """
00440     if string:
00441         # if we need the strings
00442         # for each name get the string data of the child with that name,
00443         data = [str(node.Get(name)) for name in names]
00444     else:
00445         # for each name get the child node itself
00446         data = [node.Get(name) for name in names]
00447     if single:
00448         # return the first item as a key and the second item as a single value
00449         return data[0], data[1]
00450     else:
00451         # return the first item as a key, and all the rest items as a list
00452         return data[0], data[1:]
00453 
00454 def get_data_node(node):
00455     return node.Get('Body').Child().Child()
00456