Back to index

salome-kernel  6.5.0
Launcher.cxx
Go to the documentation of this file.
00001 // Copyright (C) 2007-2012  CEA/DEN, EDF R&D, OPEN CASCADE
00002 //
00003 // Copyright (C) 2003-2007  OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN,
00004 // CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS
00005 //
00006 // This library is free software; you can redistribute it and/or
00007 // modify it under the terms of the GNU Lesser General Public
00008 // License as published by the Free Software Foundation; either
00009 // version 2.1 of the License.
00010 //
00011 // This library is distributed in the hope that it will be useful,
00012 // but WITHOUT ANY WARRANTY; without even the implied warranty of
00013 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00014 // Lesser General Public License for more details.
00015 //
00016 // You should have received a copy of the GNU Lesser General Public
00017 // License along with this library; if not, write to the Free Software
00018 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
00019 //
00020 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
00021 //
00022 
00023 #ifdef WITH_LIBBATCH
00024 #include <Batch/Batch_Date.hxx>
00025 #include <Batch/Batch_BatchManagerCatalog.hxx>
00026 #include <Batch/Batch_FactBatchManager_eClient.hxx>
00027 #include <Batch/Batch_BatchManager_eClient.hxx>
00028 #endif
00029 
00030 #include "Basics_Utils.hxx"
00031 #include "Basics_DirUtils.hxx"
00032 #include "SALOME_Launcher_Handler.hxx"
00033 #include "Launcher.hxx"
00034 #include "Launcher_Job_Command.hxx"
00035 #include <iostream>
00036 #include <sstream>
00037 #include <sys/stat.h>
00038 #include <time.h>
00039 
00040 //=============================================================================
00047 //=============================================================================
00048 Launcher_cpp::Launcher_cpp()
00049 {
00050   LAUNCHER_MESSAGE("Launcher_cpp constructor");
00051   _job_cpt = 0;
00052   _job_cpt_mutex = new pthread_mutex_t();
00053   pthread_mutex_init(_job_cpt_mutex, NULL);
00054 }
00055 
00056 //=============================================================================
00060 //=============================================================================
00061 Launcher_cpp::~Launcher_cpp()
00062 {
00063   LAUNCHER_MESSAGE("Launcher_cpp destructor");
00064 #ifdef WITH_LIBBATCH
00065   std::map<int, Launcher::Job *>::const_iterator it_job;
00066   for(it_job = _launcher_job_map.begin(); it_job != _launcher_job_map.end(); it_job++)
00067     delete it_job->second;
00068   std::map <int, Batch::BatchManager_eClient * >::const_iterator it1;
00069   for(it1=_batchmap.begin();it1!=_batchmap.end();it1++)
00070     delete it1->second;
00071 #endif
00072 
00073   pthread_mutex_destroy(_job_cpt_mutex);
00074   delete _job_cpt_mutex;
00075 }
00076 
00077 #ifdef WITH_LIBBATCH
00078 
00079 //=============================================================================
00083 //=============================================================================
00084 void 
00085 Launcher_cpp::createJob(Launcher::Job * new_job)
00086 {
00087   LAUNCHER_MESSAGE("Creating a new job");
00088   // Add job to the jobs map
00089   pthread_mutex_lock(_job_cpt_mutex);
00090   new_job->setNumber(_job_cpt);
00091   _job_cpt++;
00092   pthread_mutex_unlock(_job_cpt_mutex);
00093   std::map<int, Launcher::Job *>::const_iterator it_job = _launcher_job_map.find(new_job->getNumber());
00094   if (it_job == _launcher_job_map.end())
00095     _launcher_job_map[new_job->getNumber()] = new_job;
00096   else
00097   {
00098     LAUNCHER_INFOS("A job as already the same id: " << new_job->getNumber());
00099     delete new_job;
00100     throw LauncherException("A job as already the same id - job is not created !");
00101   }
00102   LAUNCHER_MESSAGE("New Job created");
00103 }
00104 
00105 //=============================================================================
00109 //=============================================================================
00110 void 
00111 Launcher_cpp::launchJob(int job_id)
00112 {
00113   LAUNCHER_MESSAGE("Launch a job");
00114 
00115   // Check if job exist
00116   std::map<int, Launcher::Job *>::const_iterator it_job = _launcher_job_map.find(job_id);
00117   if (it_job == _launcher_job_map.end())
00118   {
00119     LAUNCHER_INFOS("Cannot find the job, is it created ? job number: " << job_id);
00120     throw LauncherException("Cannot find the job, is it created ?");
00121   }
00122 
00123   Launcher::Job * job = it_job->second;
00124 
00125   // Check job state (cannot launch a job already launched...)
00126   if (job->getState() != "CREATED")
00127   {
00128     LAUNCHER_INFOS("Bad state of the job: " << job->getState());
00129     throw LauncherException("Bad state of the job: " + job->getState());
00130   }
00131 
00132   // Third step search batch manager for the job into the map -> instanciate one if does not exist
00133 #ifdef WITH_LIBBATCH
00134   std::map<int, Batch::BatchManager_eClient *>::const_iterator it = _batchmap.find(job_id);
00135   if(it == _batchmap.end())
00136   {
00137     createBatchManagerForJob(job);
00138   }
00139 #endif
00140 
00141   try {
00142     Batch::JobId batch_manager_job_id = _batchmap[job_id]->submitJob(*(job->getBatchJob()));
00143     job->setBatchManagerJobId(batch_manager_job_id);
00144     job->setState("QUEUED");
00145   }
00146   catch(const Batch::EmulationException &ex)
00147   {
00148     LAUNCHER_INFOS("Job is not launched, exception in submitJob: " << ex.message);
00149     throw LauncherException(ex.message.c_str());
00150   }
00151   LAUNCHER_MESSAGE("Job launched");
00152 }
00153 
00154 //=============================================================================
00158 //=============================================================================
00159 const char *
00160 Launcher_cpp::getJobState(int job_id)
00161 {
00162   LAUNCHER_MESSAGE("Get job state");
00163 
00164   // Check if job exist
00165   std::map<int, Launcher::Job *>::const_iterator it_job = _launcher_job_map.find(job_id);
00166   if (it_job == _launcher_job_map.end())
00167   {
00168     LAUNCHER_INFOS("Cannot find the job, is it created ? job number: " << job_id);
00169     throw LauncherException("Cannot find the job, is it created ?");
00170   }
00171 
00172   Launcher::Job * job = it_job->second;
00173 
00174   std::string state;
00175   try
00176   {
00177     state = job->updateJobState();
00178   }
00179   catch(const Batch::EmulationException &ex)
00180   {
00181     LAUNCHER_INFOS("getJobState failed, exception: " << ex.message);
00182     throw LauncherException(ex.message.c_str());
00183   }
00184   catch(const Batch::RunTimeException &ex)
00185   {
00186     LAUNCHER_INFOS("getJobState failed, exception: " << ex.message);
00187     throw LauncherException(ex.message.c_str());
00188   }
00189 
00190   return state.c_str();
00191 }
00192 
00193 //=============================================================================
00197 //=============================================================================
00198 void
00199 Launcher_cpp::getJobResults(int job_id, std::string directory)
00200 {
00201   LAUNCHER_MESSAGE("Get Job results");
00202 
00203   // Check if job exist
00204   std::map<int, Launcher::Job *>::const_iterator it_job = _launcher_job_map.find(job_id);
00205   if (it_job == _launcher_job_map.end())
00206   {
00207     LAUNCHER_INFOS("Cannot find the job, is it created ? job number: " << job_id);
00208     throw LauncherException("Cannot find the job, is it created ?");
00209   }
00210 
00211   Launcher::Job * job = it_job->second;
00212   std::string resource_name = job->getResourceDefinition().Name;
00213   try 
00214   {
00215     if (directory != "")
00216       _batchmap[job_id]->importOutputFiles(*(job->getBatchJob()), directory);
00217     else
00218       _batchmap[job_id]->importOutputFiles(*(job->getBatchJob()), job->getResultDirectory());
00219   }
00220   catch(const Batch::EmulationException &ex)
00221   {
00222     LAUNCHER_INFOS("getJobResult is maybe incomplete, exception: " << ex.message);
00223     throw LauncherException(ex.message.c_str());
00224   }
00225   LAUNCHER_MESSAGE("getJobResult ended");
00226 }
00227 
00228 //=============================================================================
00232 //=============================================================================
00233 bool
00234 Launcher_cpp::getJobDumpState(int job_id, std::string directory)
00235 {
00236   bool rtn;
00237   LAUNCHER_MESSAGE("Get Job dump state");
00238 
00239   // Check if job exist
00240   std::map<int, Launcher::Job *>::const_iterator it_job = _launcher_job_map.find(job_id);
00241   if (it_job == _launcher_job_map.end())
00242   {
00243     LAUNCHER_INFOS("Cannot find the job, is it created ? job number: " << job_id);
00244     throw LauncherException("Cannot find the job, is it created ?");
00245   }
00246 
00247   Launcher::Job * job = it_job->second;
00248   std::string resource_name = job->getResourceDefinition().Name;
00249   try 
00250   {
00251     if (directory != "")
00252       rtn = _batchmap[job_id]->importDumpStateFile(*(job->getBatchJob()), directory);
00253     else
00254       rtn = _batchmap[job_id]->importDumpStateFile(*(job->getBatchJob()), job->getResultDirectory());
00255   }
00256   catch(const Batch::EmulationException &ex)
00257   {
00258     LAUNCHER_INFOS("getJobResult is maybe incomplete, exception: " << ex.message);
00259     throw LauncherException(ex.message.c_str());
00260   }
00261   LAUNCHER_MESSAGE("getJobResult ended");
00262   return rtn;
00263 }
00264 
00265 //=============================================================================
00269 //=============================================================================
00270 void
00271 Launcher_cpp::removeJob(int job_id)
00272 {
00273   LAUNCHER_MESSAGE("Remove Job");
00274 
00275   // Check if job exist
00276   std::map<int, Launcher::Job *>::iterator it_job = _launcher_job_map.find(job_id);
00277   if (it_job == _launcher_job_map.end())
00278   {
00279     LAUNCHER_INFOS("Cannot find the job, is it created ? job number: " << job_id);
00280     throw LauncherException("Cannot find the job, is it created ?");
00281   }
00282 
00283   it_job->second->removeJob();
00284   delete it_job->second;
00285   _launcher_job_map.erase(it_job);
00286 }
00287 
00288 //=============================================================================
00292 //=============================================================================
00293 void
00294 Launcher_cpp::stopJob(int job_id)
00295 {
00296   LAUNCHER_MESSAGE("Stop Job");
00297 
00298   // Check if job exist
00299   std::map<int, Launcher::Job *>::iterator it_job = _launcher_job_map.find(job_id);
00300   if (it_job == _launcher_job_map.end())
00301   {
00302     LAUNCHER_INFOS("Cannot find the job, is it created ? job number: " << job_id);
00303     throw LauncherException("Cannot find the job, is it created ?");
00304   }
00305 
00306   it_job->second->stopJob();
00307 }
00308 
00309 //=============================================================================
00314 //=============================================================================
00315 long 
00316 Launcher_cpp::createJobWithFile(const std::string xmlExecuteFile, 
00317                                 const std::string clusterName)
00318 {
00319   LAUNCHER_MESSAGE("Begin of Launcher_cpp::createJobWithFile");
00320 
00321   // Parsing xml file
00322   ParserLauncherType job_params = ParseXmlFile(xmlExecuteFile);
00323 
00324   // Creating a new job
00325   Launcher::Job_Command * new_job = new Launcher::Job_Command();
00326 
00327   std::string cmdFile = Kernel_Utils::GetTmpFileName();  
00328 #ifndef WIN32
00329   cmdFile += ".sh";
00330 #else
00331   cmdFile += ".bat";
00332 #endif
00333   std::ofstream os;
00334   os.open(cmdFile.c_str(), std::ofstream::out );
00335   os << "#! /bin/sh" << std::endl;
00336   os << job_params.Command;
00337   os.close();
00338 
00339   new_job->setJobFile(cmdFile);
00340   new_job->setLocalDirectory(job_params.RefDirectory);
00341   new_job->setWorkDirectory(job_params.MachinesList[clusterName].WorkDirectory);
00342   new_job->setEnvFile(job_params.MachinesList[clusterName].EnvFile);
00343 
00344   for(int i=0; i < job_params.InputFile.size(); i++)
00345     new_job->add_in_file(job_params.InputFile[i]);
00346   for(int i=0; i < job_params.OutputFile.size();i++)
00347     new_job->add_out_file(job_params.OutputFile[i]);
00348 
00349   resourceParams p;
00350   p.hostname = clusterName;
00351   p.name = "";
00352   p.OS = "";
00353   p.nb_proc = job_params.NbOfProcesses;
00354   p.nb_node = 0;
00355   p.nb_proc_per_node = 0;
00356   p.cpu_clock = 0;
00357   p.mem_mb = 0;
00358   new_job->setResourceRequiredParams(p);
00359 
00360   createJob(new_job);
00361   return new_job->getNumber();
00362 }
00363 
00364 //=============================================================================
00368 //=============================================================================
00369 Batch::BatchManager_eClient *
00370 Launcher_cpp::FactoryBatchManager(ParserResourcesType& params)
00371 {
00372   std::string mpi;
00373   Batch::CommunicationProtocolType protocol;
00374   Batch::FactBatchManager_eClient* fact;
00375 
00376   int nb_proc_per_node = params.DataForSort._nbOfProcPerNode;
00377   std::string hostname = params.HostName;
00378 
00379   switch(params.Protocol)
00380   {
00381     case rsh:
00382       protocol = Batch::RSH;
00383       break;
00384     case ssh:
00385       protocol = Batch::SSH;
00386       break;
00387     default:
00388       throw LauncherException("Unknown protocol for this resource");
00389       break;
00390   }
00391 
00392   switch(params.mpi)
00393   {
00394     case lam:
00395       mpi = "lam";
00396       break;
00397     case mpich1:
00398       mpi = "mpich1";
00399       break;
00400     case mpich2:
00401       mpi = "mpich2";
00402       break;
00403     case openmpi:
00404       mpi = "openmpi";
00405       break;
00406     case slurmmpi:
00407       mpi = "slurmmpi";
00408       break;
00409     case prun:
00410       mpi = "prun";
00411       break;
00412     default:
00413       mpi = "nompi";
00414   }
00415 
00416   const char * bmType;
00417   switch( params.Batch )
00418   {
00419     case pbs:
00420       bmType = "ePBS";
00421       break;
00422     case lsf:
00423       bmType = "eLSF";
00424       break;
00425     case sge:
00426       bmType = "eSGE";
00427       break;
00428     case ccc:
00429       bmType = "eCCC";
00430       break;
00431     case slurm:
00432       bmType = "eSLURM";
00433       break;
00434     case ssh_batch:
00435       bmType = "eSSH";
00436       break;
00437     case ll:
00438       bmType = "eLL";
00439       break;
00440     case vishnu:
00441       bmType = "eVISHNU";
00442       break;
00443     default:
00444       LAUNCHER_MESSAGE("Bad batch description of the resource: Batch = " << params.Batch);
00445       throw LauncherException("No batchmanager for that cluster - Bad batch description of the resource");
00446   }
00447   Batch::BatchManagerCatalog & cata = Batch::BatchManagerCatalog::getInstance();
00448   fact = dynamic_cast<Batch::FactBatchManager_eClient*>(cata(bmType));
00449   if (fact == NULL) {
00450     LAUNCHER_MESSAGE("Cannot find batch manager factory for " << bmType << ". Check your version of libBatch.");
00451     throw LauncherException("Cannot find batch manager factory");
00452   }
00453   LAUNCHER_MESSAGE("Instanciation of batch manager of type: " << bmType);
00454   Batch::BatchManager_eClient * batch_client = (*fact)(hostname.c_str(), params.UserName.c_str(),
00455                                                        protocol, mpi.c_str(), nb_proc_per_node);
00456   return batch_client;
00457 }
00458 
00459 //----------------------------------------------------------
00460 // Without LIBBATCH - Launcher_cpp do nothing...
00461 //----------------------------------------------------------
00462 #else
00463 
00464 void 
00465 Launcher_cpp::createJob(Launcher::Job * new_job)
00466 {
00467   LAUNCHER_INFOS("Launcher compiled without LIBBATCH - cannot create a job !!!");
00468   delete new_job;
00469   throw LauncherException("Method Launcher_cpp::createJob is not available "
00470                           "(libBatch was not present at compilation time)");
00471 }
00472 
00473 void 
00474 Launcher_cpp::launchJob(int job_id)
00475 {
00476   LAUNCHER_INFOS("Launcher compiled without LIBBATCH - cannot launch a job !!!");
00477   throw LauncherException("Method Launcher_cpp::launchJob is not available "
00478                           "(libBatch was not present at compilation time)");
00479 }
00480 
00481 const char *
00482 Launcher_cpp::getJobState(int job_id)
00483 {
00484   LAUNCHER_INFOS("Launcher compiled without LIBBATCH - cannot get job state!!!");
00485   throw LauncherException("Method Launcher_cpp::getJobState is not available "
00486                           "(libBatch was not present at compilation time)");
00487 }
00488 
00489 void
00490 Launcher_cpp::getJobResults(int job_id, std::string directory)
00491 {
00492   LAUNCHER_INFOS("Launcher compiled without LIBBATCH - cannot get job results!!!");
00493   throw LauncherException("Method Launcher_cpp::getJobResults is not available "
00494                           "(libBatch was not present at compilation time)");
00495 }
00496 
00497 bool
00498 Launcher_cpp::getJobDumpState(int job_id, std::string directory)
00499 {
00500   LAUNCHER_INFOS("Launcher compiled without LIBBATCH - cannot get job dump state!!!");
00501   throw LauncherException("Method Launcher_cpp::getJobDumpState is not available "
00502                           "(libBatch was not present at compilation time)");
00503 }
00504 
00505 void
00506 Launcher_cpp::removeJob(int job_id)
00507 {
00508   LAUNCHER_INFOS("Launcher compiled without LIBBATCH - cannot remove job!!!");
00509   throw LauncherException("Method Launcher_cpp::removeJob is not available "
00510                           "(libBatch was not present at compilation time)");
00511 }
00512 
00513 void
00514 Launcher_cpp::stopJob(int job_id)
00515 {
00516   throw LauncherException("Method Launcher_cpp::stopJob is not available "
00517                           "(libBatch was not present at compilation time)");
00518 }
00519 
00520 long 
00521 Launcher_cpp::createJobWithFile( const std::string xmlExecuteFile, std::string clusterName)
00522 {
00523   throw LauncherException("Method Launcher_cpp::createJobWithFile is not available "
00524                           "(libBatch was not present at compilation time)");
00525   return 0;
00526 }
00527 
00528 #endif
00529 
00530 ParserLauncherType 
00531 Launcher_cpp::ParseXmlFile(std::string xmlExecuteFile)
00532 {
00533   ParserLauncherType job_params;
00534   SALOME_Launcher_Handler * handler = new SALOME_Launcher_Handler(job_params);
00535 
00536   const char* aFilePath = xmlExecuteFile.c_str();
00537   FILE* aFile = fopen(aFilePath, "r");
00538   if (aFile != NULL)
00539   {
00540     xmlDocPtr aDoc = xmlReadFile(aFilePath, NULL, 0);
00541     if (aDoc != NULL)
00542       handler->ProcessXmlDocument(aDoc);
00543     else
00544     {
00545       std::string message = "ResourcesManager_cpp: could not parse file: " + xmlExecuteFile;
00546       LAUNCHER_MESSAGE(message);
00547       delete handler;
00548       throw LauncherException(message);
00549     }
00550     // Free the document
00551     xmlFreeDoc(aDoc);
00552     fclose(aFile);
00553   }
00554   else
00555   {
00556     std::string message = "ResourcesManager_cpp: file is not readable: " + xmlExecuteFile;
00557     LAUNCHER_MESSAGE(message);
00558     delete handler;
00559     throw LauncherException(message);
00560   }
00561 
00562   // Return
00563   delete handler;
00564   return job_params;
00565 }
00566 
00567 std::map<int, Launcher::Job *>
00568 Launcher_cpp::getJobs()
00569 {
00570   return _launcher_job_map;
00571 }
00572 
00573 void 
00574 Launcher_cpp::createBatchManagerForJob(Launcher::Job * job)
00575 {
00576   int job_id = job->getNumber();
00577 
00578   // Select a ressource for the job
00579   std::vector<std::string> ResourceList;
00580   resourceParams params = job->getResourceRequiredParams();
00581   try
00582   {
00583     ResourceList = _ResManager->GetFittingResources(params);
00584   }
00585   catch(const ResourcesException &ex)
00586   {
00587     throw LauncherException(ex.msg.c_str());
00588   }
00589   if (ResourceList.size() == 0)
00590   {
00591     LAUNCHER_INFOS("No adequate resource found for the job, number " << job->getNumber());
00592     job->setState("ERROR");
00593     throw LauncherException("No resource found the job");
00594   }
00595 
00596   // Configure the job with the resource selected - the first of the list
00597   ParserResourcesType resource_definition = _ResManager->GetResourcesDescr(ResourceList[0]);
00598 
00599   // Set resource definition to the job
00600   // The job will check if the definitions needed
00601   try
00602   {
00603     job->setResourceDefinition(resource_definition);
00604   }
00605   catch(const LauncherException &ex)
00606   {
00607     LAUNCHER_INFOS("Error in the definition of the resource, mess: " << ex.msg);
00608     job->setState("ERROR");
00609     throw ex;
00610   }
00611 
00612   // Step 2: We can now add a Factory if the resource is correctly define
00613 #ifdef WITH_LIBBATCH
00614   std::map<int, Batch::BatchManager_eClient *>::const_iterator it = _batchmap.find(job_id);
00615   if(it == _batchmap.end())
00616   {
00617     try
00618     {
00619       // Warning cannot write on one line like this, because map object is constructed before
00620       // the method is called...
00621       //_batchmap[job_id] = FactoryBatchManager(resource_definition);
00622       Batch::BatchManager_eClient * batch_client = FactoryBatchManager(resource_definition);
00623       _batchmap[job_id] = batch_client;
00624     }
00625     catch(const LauncherException &ex)
00626     {
00627       LAUNCHER_INFOS("Error during creation of the batch manager of the job, mess: " << ex.msg);
00628       throw ex;
00629     }
00630     catch(const Batch::EmulationException &ex)
00631     {
00632       LAUNCHER_INFOS("Error during creation of the batch manager of the job, mess: " << ex.message);
00633       throw LauncherException(ex.message);
00634     }
00635     catch(const Batch::InvalidArgumentException &ex)
00636     {
00637       LAUNCHER_INFOS("Error during creation of the batch manager of the job, mess: " << ex.message);
00638       throw LauncherException(ex.message);
00639     }
00640   }
00641 #endif
00642 }
00643 
00644 void 
00645 Launcher_cpp::addJobDirectlyToMap(Launcher::Job * new_job, const std::string job_reference)
00646 {
00647   // Step 0: Calculated job_id
00648   pthread_mutex_lock(_job_cpt_mutex);
00649   int job_id = _job_cpt;
00650   _job_cpt++;
00651   new_job->setNumber(job_id);
00652   pthread_mutex_unlock(_job_cpt_mutex);
00653 
00654   // Step 1: check if resource is already in the map
00655   createBatchManagerForJob(new_job);
00656 
00657   // Step 2: add the job to the batch manager
00658 #ifdef WITH_LIBBATCH
00659   try
00660   {
00661     Batch::JobId batch_manager_job_id = _batchmap[job_id]->addJob(*(new_job->getBatchJob()),
00662                                                                   job_reference);
00663     new_job->setBatchManagerJobId(batch_manager_job_id);
00664   }
00665   catch(const Batch::EmulationException &ex)
00666   {
00667     LAUNCHER_INFOS("Job cannot be added, exception in addJob: " << ex.message);
00668     throw LauncherException(ex.message.c_str());
00669   }
00670 
00671   // Step 3: add job to launcher map
00672   std::map<int, Launcher::Job *>::const_iterator it_job = _launcher_job_map.find(new_job->getNumber());
00673   if (it_job == _launcher_job_map.end())
00674     _launcher_job_map[new_job->getNumber()] = new_job;
00675   else
00676   {
00677     LAUNCHER_INFOS("A job as already the same id: " << new_job->getNumber());
00678     delete new_job;
00679     throw LauncherException("A job as already the same id - job is not created !");
00680   }
00681   LAUNCHER_MESSAGE("New job added");
00682 #endif
00683 }