Back to index

nordugrid-arc-nox  1.1.0~rc6
JobLogFile.cpp
Go to the documentation of this file.
00001 #include "JobLogFile.h"
00002 
00003 #include <sys/stat.h>
00004 #include <sys/types.h>
00005 
00006 #include <errno.h>
00007 #include <stdio.h>
00008 #include <string.h>
00009 #include <list>
00010 
00011 #include "arc/Logger.h"
00012 
00013 // Needed to redefine mkdir on mingw
00014 #ifdef WIN32
00015 #include <arc/win32.h>
00016 #endif
00017 
00018 namespace Arc
00019 {
00020 
00021   int JobLogFile::parse(const std::string& _filename)
00022   {
00023     int count=0;  //number of parsed values
00024     clear();
00025     filename=_filename;
00026     if (!exists()) return -1;
00027 
00028     std::ifstream logfile(filename.c_str(),std::ios::in);
00029     std::string line;
00030     while (logfile.good())
00031       {
00032        std::getline(logfile,line);
00033        size_type e=line.find('=');
00034        if (e!=std::string::npos)
00035          {
00036            count++;
00037            std::string key=line.substr(0, e),
00038              value=line.substr(e+1, std::string::npos);
00039            (*this)[key]=value;
00040          }
00041       }
00042     logfile.close();
00043 
00044     //Parse jobreport_options string!
00045 
00046     std::string jobreport_opts=(*this)["accounting_options"];
00047     std::string option;
00048     size_type pcomma=jobreport_opts.find(',');
00049     size_type pcolon;
00050     while (pcomma!=std::string::npos)
00051       {
00052        option=jobreport_opts.substr(0,pcomma);
00053        // separate opt_name:value pair
00054        pcolon=option.find(':');
00055        if (pcolon!=std::string::npos)
00056          {
00057            std::string key=option.substr(0, pcolon), 
00058              value=option.substr(pcolon+1, std::string::npos);
00059 
00060            (*this)[std::string("jobreport_option_")+key]=value;
00061          }
00062        
00063        //next:
00064        jobreport_opts=jobreport_opts.substr(pcomma+1, std::string::npos);
00065        pcomma=jobreport_opts.find(',');
00066       }
00067     option=jobreport_opts;
00068     pcolon=option.find(':');
00069     if (pcolon!=std::string::npos)
00070       {
00071        std::string key=option.substr(0, pcolon), 
00072          value=option.substr(pcolon+1, std::string::npos);
00073        
00074        (*this)[std::string("jobreport_option_")+key]=value;
00075       }
00076 
00077 
00078 
00079     return count;
00080   }
00081 
00082   void JobLogFile::createUsageRecord(Arc::XMLNode &usagerecord,
00083                                  const char *recordid_prefix)
00084   {
00085     //***
00086     //If archiving is enabled: first try to load archived UR
00087     std::string archive_fn=getArchivingPath();
00088     if (!archive_fn.empty())
00089       {
00090        errno=0;
00091        if (usagerecord.ReadFromFile(archive_fn))
00092          {
00093            Arc::Logger::rootLogger.msg(Arc::VERBOSE,
00094               "Read archive file %s",
00095               archive_fn.c_str());
00096            return;
00097          }
00098        Arc::Logger::rootLogger.msg(Arc::VERBOSE,
00099           "Could not read archive file %s for job log file %s (%s), generating new UR",
00100           archive_fn.c_str(),
00101           filename.c_str(),
00102           strerror(errno));
00103       }
00104     //Otherwise go on and create new UR
00105     //***
00106 
00107     Arc::NS ns_ur;
00108     
00109     //Namespaces defined by OGF
00110     ns_ur[""]="http://schema.ogf.org/urf/2003/09/urf";
00111     ns_ur["urf"]="http://schema.ogf.org/urf/2003/09/urf";
00112     ns_ur["xsd"]="http://www.w3.org/2001/XMLSchema";
00113     ns_ur["xsi"]="http://www.w3.org/2001/XMLSchema-instance";
00114     ns_ur["ds"]="http://www.w3.org/2000/09/xmldsig#";
00115     ns_ur["arc"]="http://www.nordugrid.org/ws/schemas/ur-arc";
00116 
00117     //Get node names
00118     std::list<std::string> nodenames;
00119     std::string mainnode;
00120     std::string nodestr=(*this)["nodename"];
00121     size_type pcolon=nodestr.find(':');
00122     while (pcolon!=std::string::npos)
00123       {
00124        nodenames.push_back(nodestr.substr(0,pcolon));
00125        nodestr=nodestr.substr(pcolon+1,std::string::npos);
00126        pcolon=nodestr.find(':');
00127       }
00128     if (!nodestr.empty()) nodenames.push_back(nodestr);
00129     if (!nodenames.empty()) mainnode=*(nodenames.begin());
00130 
00131     //Get runtime environments
00132     std::list<std::string> rtes;
00133     std::string rtestr=(*this)["runtimeenvironment"];
00134     size_type pspace=rtestr.find(" ");
00135     while (pspace!=std::string::npos)
00136       {
00137        std::string rte=rtestr.substr(0,pspace);
00138        if (!rte.empty()) rtes.push_back(rte);
00139        rtestr=rtestr.substr(pspace+1,std::string::npos);
00140        pspace=rtestr.find(" ");
00141       }
00142     if (!rtestr.empty()) rtes.push_back(rtestr);
00143     
00144     //Fill this Usage Record
00145     Arc::XMLNode ur(ns_ur,"JobUsageRecord");
00146     
00147     //RecordIdentity, GlobalJobId, LocalJobId
00148     if (find("ngjobid")!=end())
00149       {
00150        // Timestamp for record, required
00151        std::string nowstamp=Arc::Time().str(Arc::UTCTime);
00152        
00153        Arc::XMLNode rid=ur.NewChild("RecordIdentity");
00154        rid.NewAttribute("createTime")=nowstamp;
00155        //NOTE! Current LUTS also sets a "creationTime"[sic!] for each record
00156        
00157        // ID for record
00158        if (!mainnode.empty())
00159          rid.NewAttribute("recordId")=
00160            std::string(recordid_prefix) + mainnode + 
00161            '-' + (*this)["ngjobid"];
00162        else
00163          rid.NewAttribute("recordId")=
00164            std::string(recordid_prefix) + (*this)["ngjobid"];
00165        
00166        ur.NewChild("JobIdentity").NewChild("GlobalJobId")=
00167          (*this)["globalid"];
00168        
00169        if (find("localjobid")!=end())
00170          ur["JobIdentity"].NewChild("LocalJobId")=
00171            (*this)["localjobid"];
00172       }
00173     else
00174     {
00175       //TODO what if not valid?
00176       Arc::Logger::rootLogger.msg(Arc::VERBOSE,
00177                     "Missing required UR element \"RecordIdentity\", in job log file %s",
00178                     filename.c_str());
00179       usagerecord.Destroy();
00180       return;
00181     }
00182     
00183     //ProcessId?
00184     
00185     //GlobalUser[Nn]ame, LocalUserId
00186     //TODO clarify case
00187     //NOTE! original JARM used "GlobalUserId"
00188     if (find("usersn")!=end())
00189       {
00190        ur.NewChild("UserIdentity").NewChild("GlobalUserName")=
00191          (*this)["usersn"];
00192        
00193        if (find("localuser")!=end())
00194          ur["UserIdentity"].NewChild("LocalUserId")=
00195            (*this)["localuser"];
00196       }
00197     
00198     //JobName
00199     if (find("jobname")!=end())
00200       {
00201        ur.NewChild("JobName")=(*this)["jobname"];
00202       }
00203     
00204     //Charge?
00205     
00206     //Status
00207     if (find("status")!=end())
00208       {
00209        ur.NewChild("Status")=(*this)["status"];  //TODO convert?
00210       }
00211     else
00212       {
00213        //TODO what if not valid?
00214        Arc::Logger::rootLogger.msg(Arc::VERBOSE,
00215                      "Missing required element \"Status\" in job log file %s",
00216                      filename.c_str());
00217        usagerecord.Destroy();
00218        return;
00219       }
00220     
00221     //---
00222     //Network?
00223 
00224     //Disk?
00225 
00226     //Memory
00227     if (find("usedmemory")!=end())
00228       {
00229        Arc::XMLNode memn=ur.NewChild("Memory")=(*this)["usedmemory"];
00230        memn.NewAttribute("storageUnit")="kB";
00231        memn.NewAttribute("metric")="average";
00232        memn.NewAttribute("type")="virtual";
00233       }
00234 
00235     if (find("usedmaxresident")!=end())
00236       {
00237        Arc::XMLNode memn=ur.NewChild("Memory")=(*this)["usedmaxresident"];
00238        memn.NewAttribute("storageUnit")="kB";
00239        memn.NewAttribute("metric")="max";
00240        memn.NewAttribute("type")="physical";
00241       }
00242 
00243     if (find("usedaverageresident")!=end())
00244       {
00245        Arc::XMLNode memn=ur.NewChild("Memory")=(*this)["usedaverageresident"];
00246        memn.NewAttribute("storageUnit")="kB";
00247        memn.NewAttribute("metric")="average";
00248        memn.NewAttribute("type")="physical";
00249       }
00250     
00251     //Swap?
00252 
00253     //TimeDuration, TimeInstant, ServiceLevel?
00254 
00255     //---
00256     //WallDuration
00257     if (find("usedwalltime")!=end())
00258       {
00259        Arc::Period walldur((*this)["usedwalltime"],Arc::PeriodSeconds);
00260        ur.NewChild("WallDuration")=(std::string)walldur;
00261       }
00262     
00263     //CpuDuration
00264 
00265     if (find("usedusercputime")!=end() && find("usedkernelcputime")!=end())
00266       {
00267        Arc::Period udur((*this)["usedusercputime"],Arc::PeriodSeconds);
00268        Arc::Period kdur((*this)["usedkernelcputime"],Arc::PeriodSeconds);
00269 
00270        Arc::XMLNode udurn=ur.NewChild("CpuDuration")=(std::string)udur;
00271        udurn.NewAttribute("usageType")="user";
00272 
00273        Arc::XMLNode kdurn=ur.NewChild("CpuDuration")=(std::string)kdur;
00274        kdurn.NewAttribute("usageType")="kernel";
00275       }
00276     else
00277     if (find("usedcputime")!=end())
00278       {
00279        Arc::Period cpudur((*this)["usedcputime"],Arc::PeriodSeconds);
00280        ur.NewChild("CpuDuration")=(std::string)cpudur;
00281       }
00282     
00283     //StartTime
00284     if (find("submissiontime")!=end())
00285       {
00286        Arc::Time starttime((*this)["submissiontime"]);
00287        ur.NewChild("StartTime")=starttime.str(Arc::UTCTime);
00288       }
00289     
00290     //EndTime
00291     if (find("endtime")!=end())
00292       {
00293        Arc::Time endtime((*this)["endtime"]);
00294        ur.NewChild("EndTime")=endtime.str(Arc::UTCTime);
00295       }
00296     
00297     //MachineName
00298     if (find("nodename")!=end())
00299       {
00300        ur.NewChild("MachineName")=mainnode;
00301       }
00302     
00303     //Host
00304     if (!mainnode.empty())
00305       {
00306        Arc::XMLNode primary_node=ur.NewChild("Host");
00307        primary_node=mainnode;
00308        primary_node.NewAttribute("primary")="true";
00309        std::list<std::string>::iterator it=nodenames.begin();
00310        ++it;
00311        while (it!=nodenames.end())
00312          {
00313            ur.NewChild("Host")=*it;
00314            ++it;
00315          }    
00316       }
00317     
00318     //SubmitHost
00319     if (find("clienthost")!=end())
00320       {
00321        // Chop port no.
00322        std::string hostport=(*this)["clienthost"], host;
00323        size_type clnp=hostport.find(":");
00324        if (clnp==std::string::npos)
00325          host=hostport;
00326        else
00327          host=hostport.substr(0,clnp);
00328 
00329        ur.NewChild("SubmitHost")=host;
00330       }
00331     
00332     //Queue
00333     if (find("lrms")!=end())
00334       {
00335        ur.NewChild("Queue")=(*this)["lrms"];  //OK for Queue?
00336       }
00337     
00338     //ProjectName
00339     if (find("projectname")!=end())
00340       {
00341        ur.NewChild("ProjectName")=(*this)["projectname"];
00342       }
00343 
00344     
00345     //NodeCount
00346     if (find("nodecount")!=end())
00347       {
00348        ur.NewChild("NodeCount")=(*this)["nodecount"];
00349       }
00350 
00351     //Processors?
00352 
00353     //Extra:
00354     //RunTimeEnvironment
00355 
00356     for(std::list<std::string>::iterator jt=rtes.begin();
00357        jt!=rtes.end();
00358        ++jt)
00359       {
00360        ur.NewChild("arc:RunTimeEnvironment")=*jt;
00361       }
00362     
00363     //TODO user id info
00364 
00365     //***
00366     //Archiving if enabled:
00367     if (!archive_fn.empty())
00368       {
00369        struct stat st;
00370        std::string dir_name=(*this)["jobreport_option_archiving"];
00371        if (stat(dir_name.c_str(),&st)!=0)
00372          {
00373            Arc::Logger::rootLogger.msg(Arc::VERBOSE,
00374                                     "Creating directory %s",
00375                                     dir_name.c_str());
00376            errno=0;
00377            if (mkdir(dir_name.c_str(),S_IRWXU)!=0)
00378              {
00379               Arc::Logger::rootLogger.msg(Arc::ERROR,
00380                   "Failed to create archive directory %s: %s",
00381                   dir_name.c_str(),
00382                   strerror(errno));
00383              }
00384          }
00385        
00386        Arc::Logger::rootLogger.msg(Arc::VERBOSE,
00387                                 "Archiving UR to file %s",
00388                                 archive_fn.c_str());
00389        errno=0;
00390        if (!ur.SaveToFile(archive_fn.c_str()))
00391          {
00392            Arc::Logger::rootLogger.msg(Arc::ERROR,
00393                                    "Failed to write file %s: %s",
00394                                    archive_fn.c_str(),
00395                                    strerror(errno));
00396          }
00397       }
00398     //***
00399 
00400     usagerecord.Replace(ur);
00401   }
00402 
00403   bool JobLogFile::exists()
00404   {
00405     //TODO cross-platform?
00406     struct stat s;
00407     return (0==stat(filename.c_str(),&s));
00408   }
00409 
00410   bool JobLogFile::olderThan(time_t age)
00411   {
00412     struct stat s;
00413     return ( ( 0==stat(filename.c_str(),&s) ) &&
00414             ( (time(NULL)-s.st_mtime) > age ) 
00415             );
00416   }
00417 
00418   void JobLogFile::remove()
00419   {
00420     if (!allow_remove) return;
00421     errno=0;
00422     int e = ::remove(filename.c_str());
00423     if (e)
00424       Arc::Logger::rootLogger.msg(Arc::ERROR,"Failed to delete file %s:%s",
00425                     filename.c_str(),
00426                     strerror(errno));
00427   }
00428 
00429   std::string JobLogFile::getArchivingPath()
00430   {
00431     //no archiving dir set
00432     if ((*this)["jobreport_option_archiving"].empty()) return std::string();
00433 
00434     //if set, archive file name corresponds to original job log file
00435     std::string base_fn;
00436     size_type seppos=filename.rfind('/');
00437     if (seppos==std::string::npos)
00438       base_fn=filename;
00439     else
00440       base_fn=filename.substr(seppos+1,std::string::npos);
00441 
00442     return (*this)["jobreport_option_archiving"]+"/usagerecord."+base_fn;
00443   }
00444   
00445 } // namespace