Back to index

nordugrid-arc-nox  1.1.0~rc6
arcsub.cpp
Go to the documentation of this file.
00001 // -*- indent-tabs-mode: nil -*-
00002 
00003 #ifdef HAVE_CONFIG_H
00004 #include <config.h>
00005 #endif
00006 
00007 #include <fstream>
00008 #include <iostream>
00009 #include <list>
00010 #include <string>
00011 #include <sys/types.h>
00012 #include <sys/stat.h>
00013 #include <unistd.h>
00014 
00015 #include <arc/ArcConfig.h>
00016 #include <arc/ArcLocation.h>
00017 #include <arc/DateTime.h>
00018 #include <arc/FileLock.h>
00019 #include <arc/IString.h>
00020 #include <arc/Logger.h>
00021 #include <arc/OptionParser.h>
00022 #include <arc/StringConv.h>
00023 #include <arc/URL.h>
00024 #include <arc/Utils.h>
00025 #include <arc/XMLNode.h>
00026 #include <arc/client/Submitter.h>
00027 #include <arc/client/TargetGenerator.h>
00028 #include <arc/client/JobDescription.h>
00029 #include <arc/UserConfig.h>
00030 #include <arc/client/Broker.h>
00031 
00032 int main(int argc, char **argv) {
00033 
00034   setlocale(LC_ALL, "");
00035 
00036   Arc::Logger logger(Arc::Logger::getRootLogger(), "arcsub");
00037   Arc::LogStream logcerr(std::cerr);
00038   logcerr.setFormat(Arc::ShortFormat);
00039   Arc::Logger::getRootLogger().addDestination(logcerr);
00040   Arc::Logger::getRootLogger().setThreshold(Arc::WARNING);
00041 
00042   Arc::ArcLocation::Init(argv[0]);
00043 
00044   Arc::OptionParser options(istring("[filename ...]"),
00045                             istring("The arcsub command is used for "
00046                                     "submitting jobs to grid enabled "
00047                                     "computing\nresources."),
00048                             istring("Argument to -i has the format "
00049                                     "Flavour:URL e.g.\n"
00050                                     "ARC0:ldap://grid.tsl.uu.se:2135/"
00051                                     "mds-vo-name=sweden,O=grid\n"
00052                                     "CREAM:ldap://cream.grid.upjs.sk:2170/"
00053                                     "o=grid\n"
00054                                     "\n"
00055                                     "Argument to -c has the format "
00056                                     "Flavour:URL e.g.\n"
00057                                     "ARC0:ldap://grid.tsl.uu.se:2135/"
00058                                     "nordugrid-cluster-name=grid.tsl.uu.se,"
00059                                     "Mds-Vo-name=local,o=grid"));
00060 
00061   std::list<std::string> clusters;
00062   options.AddOption('c', "cluster",
00063                     istring("explicity select or reject a specific cluster"),
00064                     istring("[-]name"),
00065                     clusters);
00066 
00067   std::list<std::string> indexurls;
00068   options.AddOption('i', "index",
00069                     istring("explicity select or reject an index server"),
00070                     istring("[-]name"),
00071                     indexurls);
00072 
00073   std::list<std::string> jobdescriptionstrings;
00074   options.AddOption('e', "jobdescrstring",
00075                     istring("jobdescription string describing the job to "
00076                             "be submitted"),
00077                     istring("string"),
00078                     jobdescriptionstrings);
00079 
00080   std::list<std::string> jobdescriptionfiles;
00081   options.AddOption('f', "jobdescrfile",
00082                     istring("jobdescription file describing the job to "
00083                             "be submitted"),
00084                     istring("string"),
00085                     jobdescriptionfiles);
00086 
00087   std::string joblist;
00088   options.AddOption('j', "joblist",
00089                     istring("file where the jobs will be stored"),
00090                     istring("filename"),
00091                     joblist);
00092 
00093   /*
00094      bool dryrun = false;
00095      options.AddOption('D', "dryrun", istring("add dryrun option"),
00096                     dryrun);
00097 
00098    */
00099   bool dumpdescription = false;
00100   options.AddOption('x', "dumpdescription",
00101                     istring("do not submit - dump job description "
00102                             "in the language accepted by the target"),
00103                     dumpdescription);
00104 
00105   int timeout = -1;
00106   options.AddOption('t', "timeout", istring("timeout in seconds (default 20)"),
00107                     istring("seconds"), timeout);
00108 
00109   std::string conffile;
00110   options.AddOption('z', "conffile",
00111                     istring("configuration file (default ~/.arc/client.conf)"),
00112                     istring("filename"), conffile);
00113 
00114   std::string debug;
00115   options.AddOption('d', "debug",
00116                     istring("FATAL, ERROR, WARNING, INFO, VERBOSE or DEBUG"),
00117                     istring("debuglevel"), debug);
00118 
00119   std::string broker;
00120   options.AddOption('b', "broker",
00121                     istring("select broker method (Random (default), FastestQueue, or custom)"),
00122                     istring("broker"), broker);
00123 
00132   bool version = false;
00133   options.AddOption('v', "version", istring("print version information"),
00134                     version);
00135 
00136   std::list<std::string> params = options.Parse(argc, argv);
00137 
00138   // If debug is specified as argument, it should be set before loading the configuration.
00139   if (!debug.empty())
00140     Arc::Logger::getRootLogger().setThreshold(Arc::string_to_level(debug));
00141 
00142   Arc::UserConfig usercfg(conffile, joblist);
00143   if (!usercfg) {
00144     logger.msg(Arc::ERROR, "Failed configuration initialization");
00145     return 1;
00146   }
00147 
00148   if (debug.empty() && !usercfg.Verbosity().empty())
00149     Arc::Logger::getRootLogger().setThreshold(Arc::string_to_level(usercfg.Verbosity()));
00150 
00151   if (timeout > 0)
00152     usercfg.Timeout(timeout);
00153 
00154   if (!broker.empty())
00155     usercfg.Broker(broker);
00156 
00157   if (version) {
00158     std::cout << Arc::IString("%s version %s", "arcsub", VERSION)
00159               << std::endl;
00160     return 0;
00161   }
00162 
00163   if (!clusters.empty() || !indexurls.empty())
00164     usercfg.ClearSelectedServices();
00165 
00166   if (!clusters.empty())
00167     usercfg.AddServices(clusters, Arc::COMPUTING);
00168 
00169   if (!indexurls.empty())
00170     usercfg.AddServices(indexurls, Arc::INDEX);
00171 
00172   jobdescriptionfiles.insert(jobdescriptionfiles.end(),
00173                              params.begin(), params.end());
00174 
00175   if (jobdescriptionfiles.empty() && jobdescriptionstrings.empty()) {
00176     logger.msg(Arc::ERROR, "No job description input specified");
00177     return 1;
00178   }
00179 
00180   std::list<Arc::JobDescription> jobdescriptionlist;
00181 
00182   //Loop over input job description files
00183   for (std::list<std::string>::iterator it = jobdescriptionfiles.begin();
00184        it != jobdescriptionfiles.end(); it++) {
00185 
00186     std::ifstream descriptionfile(it->c_str());
00187 
00188     if (!descriptionfile) {
00189       logger.msg(Arc::ERROR, "Can not open job description file: %s", *it);
00190       return 1;
00191     }
00192 
00193     descriptionfile.seekg(0, std::ios::end);
00194     std::streamsize length = descriptionfile.tellg();
00195     descriptionfile.seekg(0, std::ios::beg);
00196 
00197     char *buffer = new char[length + 1];
00198     descriptionfile.read(buffer, length);
00199     descriptionfile.close();
00200 
00201     buffer[length] = '\0';
00202     Arc::JobDescription jobdesc;
00203     jobdesc.Parse((std::string)buffer);
00204 
00205     if (jobdesc)
00206       jobdescriptionlist.push_back(jobdesc);
00207     else {
00208       logger.msg(Arc::ERROR, "Invalid JobDescription:");
00209       std::cout << buffer << std::endl;
00210       delete[] buffer;
00211       return 1;
00212     }
00213     delete[] buffer;
00214   }
00215 
00216   //Loop over job description input strings
00217   for (std::list<std::string>::iterator it = jobdescriptionstrings.begin();
00218        it != jobdescriptionstrings.end(); it++) {
00219 
00220     Arc::JobDescription jobdesc;
00221 
00222     jobdesc.Parse(*it);
00223 
00224     if (jobdesc)
00225       jobdescriptionlist.push_back(jobdesc);
00226     else {
00227       logger.msg(Arc::ERROR, "Invalid JobDescription:");
00228       std::cout << *it << std::endl;
00229       return 1;
00230     }
00231   }
00232 
00233   Arc::TargetGenerator targen(usercfg);
00234   targen.GetTargets(0, 1);
00235 
00236   if (targen.FoundTargets().empty()) {
00237     std::cout << Arc::IString("Job submission aborted because no clusters returned any information") << std::endl;
00238     return 1;
00239   }
00240 
00241   std::map<int, std::string> notsubmitted;
00242 
00243   int jobnr = 1;
00244   std::list<std::string> jobids;
00245 
00246   Arc::BrokerLoader loader;
00247   Arc::Broker *ChosenBroker = loader.load(usercfg.Broker().first, usercfg);
00248   if (!ChosenBroker) {
00249     logger.msg(Arc::ERROR, "Unable to load broker %s", usercfg.Broker().first);
00250     return 1;
00251   }
00252   logger.msg(Arc::INFO, "Broker %s loaded", usercfg.Broker().first);
00253 
00254   for (std::list<Arc::JobDescription>::iterator it =
00255          jobdescriptionlist.begin(); it != jobdescriptionlist.end();
00256        it++, jobnr++) {
00257 
00258     ChosenBroker->PreFilterTargets(targen.ModifyFoundTargets(), *it);
00259 
00260     while (true) {
00261       const Arc::ExecutionTarget* target = ChosenBroker->GetBestTarget();
00262 
00263       if (!target) {
00264         if (dumpdescription) {
00265           std::cout << Arc::IString("Unable to print job description: No target found.") << std::endl;
00266           return 1;
00267         }
00268         std::cout << Arc::IString("Job submission failed, no more possible targets") << std::endl;
00269         break;
00270       }
00271 
00272       Arc::Submitter *submitter = target->GetSubmitter(usercfg);
00273 
00274       if (dumpdescription) {
00275         Arc::JobDescription jobdescdump(*it);
00276         if (!submitter->ModifyJobDescription(jobdescdump, *target)) {
00277           std::cout << "Unable to modify job description according to needs for target cluster." << std::endl;
00278           return 1;
00279         }
00280 
00281         std::string jobdesclang = "ARCJSDL";
00282         if (target->GridFlavour == "ARC0")
00283           jobdesclang = "XRSL";
00284         else if (target->GridFlavour == "CREAM")
00285           jobdesclang = "JDL";
00286         const std::string jobdesc = jobdescdump.UnParse(jobdesclang);
00287         if (jobdesc.empty()) {
00288           std::cout << Arc::IString("An error occurred during the generation of the job description output.") << std::endl;
00289           return 1;
00290         }
00291 
00292         std::cout << Arc::IString("Job description to be send to ") << target->Cluster.str() << ":" << std::endl;
00293         std::cout << jobdesc << std::endl;
00294         return 0;
00295       }
00296 
00297 
00298       //submit the job
00299       Arc::URL jobid = submitter->Submit(*it, *target);
00300       if (!jobid) {
00301         std::cout << Arc::IString("Submission to %s failed, trying next target", target->url.str()) << std::endl;
00302         continue;
00303       }
00304 
00305       ChosenBroker->RegisterJobsubmission();
00306       std::cout << Arc::IString("Job submitted with jobid: %s", jobid.str())
00307                 << std::endl;
00308 
00309       break;
00310     } //end loop over all possible targets
00311   } //end loop over all job descriptions
00312 
00313   if (jobdescriptionlist.size() > 1) {
00314     std::cout << std::endl << Arc::IString("Job submission summary:")
00315               << std::endl;
00316     std::cout << "-----------------------" << std::endl;
00317     std::cout << Arc::IString("%d of %d jobs were submitted",
00318                               jobdescriptionlist.size() - notsubmitted.size(),
00319                               jobdescriptionlist.size()) << std::endl;
00320     if (notsubmitted.size())
00321       std::cout << Arc::IString("The following %d were not submitted",
00322                                 notsubmitted.size()) << std::endl;
00323     /*
00324        std::map<int, std::string>::iterator it;
00325        for (it = notsubmitted.begin(); it != notsubmitted.end(); it++) {
00326        std::cout << _("Job nr.") << " " << it->first;
00327        if (it->second.size()>0) std::cout << ": " << it->second;
00328        std::cout << std::endl;
00329        }
00330      */
00331   }
00332 
00333   return 0;
00334 }