SHUTTLE/AliShuttle.cxx

   1 /**************************************************************************
   2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
   3  *                                                                        *
   4  * Author: The ALICE Off-line Project.                                    *
   5  * Contributors are mentioned in the code where appropriate.              *
   6  *                                                                        *
   7  * Permission to use, copy, modify and distribute this software and its   *
   8  * documentation strictly for non-commercial purposes is hereby granted   *
   9  * without fee, provided that the above copyright notice appears in all   *
  10  * copies and that both the copyright notice and this permission notice   *
  11  * appear in the supporting documentation. The authors make no claims     *
  12  * about the suitability of this software for any purpose. It is          *
  13  * provided "as is" without express or implied warranty.                  *
  14  **************************************************************************/
  15
  16 /*
  17 $Log$
  18 Revision 1.18  2006/10/20 15:22:59  jgrosseo
  19 o) Adding time out to the execution of the preprocessors: The Shuttle forks and the parent process monitors the child
  20 o) Merging Collect, CollectAll, CollectNew function
  21 o) Removing implementation of empty copy constructors (declaration still there!)
  22
  23 Revision 1.17  2006/10/05 16:20:55  jgrosseo
  24 adapting to new CDB classes
  25
  26 Revision 1.16  2006/10/05 15:46:26  jgrosseo
  27 applying to the new interface
  28
  29 Revision 1.15  2006/10/02 16:38:39  jgrosseo
  30 update (alberto):
  31 fixed memory leaks
  32 storing of objects that failed to be stored to the grid before
  33 interfacing of shuttle status table in daq system
  34
  35 Revision 1.14  2006/08/29 09:16:05  jgrosseo
  36 small update
  37
  38 Revision 1.13  2006/08/15 10:50:00  jgrosseo
  39 effc++ corrections (alberto)
  40
  41 Revision 1.12  2006/08/08 14:19:29  jgrosseo
  42 Update to shuttle classes (Alberto)
  43
  44 - Possibility to set the full object's path in the Preprocessor's and
  45 Shuttle's  Store functions
  46 - Possibility to extend the object's run validity in the same classes
  47 ("startValidity" and "validityInfinite" parameters)
  48 - Implementation of the StoreReferenceData function to store reference
  49 data in a dedicated CDB storage.
  50
  51 Revision 1.11  2006/07/21 07:37:20  jgrosseo
  52 last run is stored after each run
  53
  54 Revision 1.10  2006/07/20 09:54:40  jgrosseo
  55 introducing status management: The processing per subdetector is divided into several steps,
  56 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
  57 can keep track of the number of failures and skips further processing after a certain threshold is
  58 exceeded. These thresholds can be configured in LDAP.
  59
  60 Revision 1.9  2006/07/19 10:09:55  jgrosseo
  61 new configuration, accesst to DAQ FES (Alberto)
  62
  63 Revision 1.8  2006/07/11 12:44:36  jgrosseo
  64 adding parameters for extended validity range of data produced by preprocessor
  65
  66 Revision 1.7  2006/07/10 14:37:09  jgrosseo
  67 small fix + todo comment
  68
  69 Revision 1.6  2006/07/10 13:01:41  jgrosseo
  70 enhanced storing of last sucessfully processed run (alberto)
  71
  72 Revision 1.5  2006/07/04 14:59:57  jgrosseo
  73 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
  74
  75 Revision 1.4  2006/06/12 09:11:16  jgrosseo
  76 coding conventions (Alberto)
  77
  78 Revision 1.3  2006/06/06 14:26:40  jgrosseo
  79 o) removed files that were moved to STEER
  80 o) shuttle updated to follow the new interface (Alberto)
  81
  82 Revision 1.2  2006/03/07 07:52:34  hristov
  83 New version (B.Yordanov)
  84
  85 Revision 1.6  2005/11/19 17:19:14  byordano
  86 RetrieveDATEEntries and RetrieveConditionsData added
  87
  88 Revision 1.5  2005/11/19 11:09:27  byordano
  89 AliShuttle declaration added
  90
  91 Revision 1.4  2005/11/17 17:47:34  byordano
  92 TList changed to TObjArray
  93
  94 Revision 1.3  2005/11/17 14:43:23  byordano
  95 import to local CVS
  96
  97 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
  98 Initial import as subdirectory in AliRoot
  99
 100 Revision 1.2  2005/09/13 08:41:15  byordano
 101 default startTime endTime added
 102
 103 Revision 1.4  2005/08/30 09:13:02  byordano
 104 some docs added
 105
 106 Revision 1.3  2005/08/29 21:15:47  byordano
 107 some docs added
 108
 109 */
 110
 111 //
 112 // This class is the main manager for AliShuttle.
 113 // It organizes the data retrieval from DCS and call the
 114 // interface methods of AliPreprocessor.
 115 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
 116 // data for its set of aliases is retrieved. If there is registered
 117 // AliPreprocessor for this detector then it will be used
 118 // accroding to the schema (see AliPreprocessor).
 119 // If there isn't registered AliPreprocessor than the retrieved
 120 // data is stored automatically to the undelying AliCDBStorage.
 121 // For detSpec is used the alias name.
 122 //
 123
 124 #include "AliShuttle.h"
 125
 126 #include "AliCDBManager.h"
 127 #include "AliCDBStorage.h"
 128 #include "AliCDBId.h"
 129 #include "AliCDBRunRange.h"
 130 #include "AliCDBPath.h"
 131 #include "AliCDBEntry.h"
 132 #include "AliShuttleConfig.h"
 133 #include "DCSClient/AliDCSClient.h"
 134 #include "AliLog.h"
 135 #include "AliPreprocessor.h"
 136 #include "AliShuttleStatus.h"
 137 #include "AliShuttleLogbookEntry.h"
 138
 139 #include <TSystem.h>
 140 #include <TObject.h>
 141 #include <TString.h>
 142 #include <TTimeStamp.h>
 143 #include <TObjString.h>
 144 #include <TSQLServer.h>
 145 #include <TSQLResult.h>
 146 #include <TSQLRow.h>
 147 #include <TMutex.h>
 148
 149 #include <fstream>
 150
 151 #include <sys/types.h>
 152 #include <sys/wait.h>
 153
 154 ClassImp(AliShuttle)
 155
 156 TString AliShuttle::fgkMainCDB("alien://folder=ShuttleCDB");
 157 TString AliShuttle::fgkLocalCDB("local://LocalShuttleCDB");
 158 TString AliShuttle::fgkMainRefStorage("alien://folder=ShuttleReference");
 159 TString AliShuttle::fgkLocalRefStorage("local://LocalReferenceStorage");
 160
 161 Bool_t AliShuttle::fgkProcessDCS(kTRUE);
 162
 163 const char* AliShuttle::fgkShuttleTempDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/temp");
 164 const char* AliShuttle::fgkShuttleLogDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/log");
 165
 166 //______________________________________________________________________________________________
 167 AliShuttle::AliShuttle(const AliShuttleConfig* config,
 168                 UInt_t timeout, Int_t retries):
 169 fConfig(config),
 170 fTimeout(timeout), fRetries(retries),
 171 fPreprocessorMap(),
 172 fLogbookEntry(0),
 173 fCurrentDetector(),
 174 fStatusEntry(0),
 175 fGridError(kFALSE),
 176 fMonitoringMutex(0),
 177 fLastActionTime(0),
 178 fLastAction()
 179 {
 180         //
 181         // config: AliShuttleConfig used
 182         // timeout: timeout used for AliDCSClient connection
 183         // retries: the number of retries in case of connection error.
 184         //
 185
 186         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
 187         for(int iSys=0;iSys<3;iSys++) {
 188                 fServer[iSys]=0;
 189                 fFESlist[iSys].SetOwner(kTRUE);
 190         }
 191         fPreprocessorMap.SetOwner(kTRUE);
 192
 193         fMonitoringMutex = new TMutex();
 194 }
 195
 196 //______________________________________________________________________________________________
 197 AliShuttle::~AliShuttle()
 198 {
 199 // destructor
 200
 201         fPreprocessorMap.DeleteAll();
 202         for(int iSys=0;iSys<3;iSys++)
 203                 if(fServer[iSys]) {
 204                         fServer[iSys]->Close();
 205                         delete fServer[iSys];
 206                         fServer[iSys] = 0;
 207                 }
 208
 209         if (fStatusEntry){
 210                 delete fStatusEntry;
 211                 fStatusEntry = 0;
 212         }
 213
 214         if (fMonitoringMutex)
 215         {
 216                 delete fMonitoringMutex;
 217                 fMonitoringMutex = 0;
 218         }
 219 }
 220
 221 //______________________________________________________________________________________________
 222 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
 223 {
 224         //
 225         // Registers new AliPreprocessor.
 226         // It uses GetName() for indentificator of the pre processor.
 227         // The pre processor is registered it there isn't any other
 228         // with the same identificator (GetName()).
 229         //
 230
 231         const char* detName = preprocessor->GetName();
 232         if(GetDetPos(detName) < 0)
 233                 AliFatal(Form("********** !!!!! Invalid detector name: %s !!!!! **********", detName));
 234
 235         if (fPreprocessorMap.GetValue(detName)) {
 236                 AliWarning(Form("AliPreprocessor %s is already registered!", detName));
 237                 return;
 238         }
 239
 240         fPreprocessorMap.Add(new TObjString(detName), preprocessor);
 241 }
 242 //______________________________________________________________________________________________
 243 UInt_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
 244                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
 245 {
 246   // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
 247   // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
 248   // using this function. Use StoreReferenceData instead!
 249   // It calls WriteToCDB function which perform actual storage
 250
 251         return WriteToCDB(fgkMainCDB, fgkLocalCDB, path, object,
 252                                 metaData, validityStart, validityInfinite);
 253
 254 }
 255
 256 //______________________________________________________________________________________________
 257 UInt_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData)
 258 {
 259   // Stores a CDB object in the storage for reference data. This objects will not be available during
 260   // offline reconstrunction. Use this function for reference data only!
 261   // It calls WriteToCDB function which perform actual storage
 262
 263         return WriteToCDB(fgkMainRefStorage, fgkLocalRefStorage, path, object, metaData);
 264
 265 }
 266
 267 //______________________________________________________________________________________________
 268 UInt_t AliShuttle::WriteToCDB(const char* mainUri, const char* localUri,
 269                         const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
 270                         Int_t validityStart, Bool_t validityInfinite)
 271 {
 272   // write object into the CDB. Parameters are passed by Store and StoreReferenceData functions.
 273   // The parameters are:
 274   //   1) Uri of the main storage (Grid)
 275   //   2) Uri of the backup storage (Local)
 276   //   3) the object's path.
 277   //   4) the object to be stored
 278   //   5) the metaData to be associated with the object
 279   //   6) the validity start run number w.r.t. the current run,
 280   //      if the data is valid only for this run leave the default 0
 281   //   7) specifies if the calibration data is valid for infinity (this means until updated),
 282   //      typical for calibration runs, the default is kFALSE
 283   //
 284   // returns 0 if fail
 285   //         1 if stored in main (Grid) storage
 286   //         2 if stored in backup (Local) storage
 287
 288         const char* cdbType = (mainUri == fgkMainCDB) ? "CDB" : "Reference";
 289
 290         Int_t firstRun = GetCurrentRun() - validityStart;
 291         if(firstRun < 0) {
 292                 AliError("First valid run happens to be less than 0! Setting it to 0.");
 293                 firstRun=0;
 294         }
 295
 296         Int_t lastRun = -1;
 297         if(validityInfinite) {
 298                 lastRun = AliCDBRunRange::Infinity();
 299         } else {
 300                 lastRun = GetCurrentRun();
 301         }
 302
 303         AliCDBId id(path, firstRun, lastRun, -1, -1);
 304
 305         if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
 306                 TObjString runUsed = Form("%d", GetCurrentRun());
 307                 metaData->SetProperty("RunUsed(TObjString)",&runUsed);
 308         }
 309
 310         UInt_t result = 0;
 311
 312         if (!(AliCDBManager::Instance()->GetStorage(mainUri))) {
 313                 AliError(Form("WriteToCDB - Cannot activate main %s storage", cdbType));
 314         } else {
 315                 result = (UInt_t) AliCDBManager::Instance()->GetStorage(mainUri)
 316                                         ->Put(object, id, metaData);
 317         }
 318
 319         if(!result) {
 320
 321                 Log(fCurrentDetector,
 322                         Form("WriteToCDB - Problem with main %s storage. Putting <%s> into backup storage",
 323                                 cdbType, path.GetPath().Data()));
 324
 325                 // Set Grid version to current run number, to ease retrieval later
 326                 id.SetVersion(GetCurrentRun());
 327
 328                 result = AliCDBManager::Instance()->GetStorage(localUri)
 329                                         ->Put(object, id, metaData);
 330
 331                 if(result) {
 332                         result = 2;
 333                         fGridError = kTRUE;
 334                 }else{
 335                         Log(fCurrentDetector, "WriteToCDB - Can't store data!");
 336                 }
 337         }
 338
 339         return result;
 340
 341 }
 342
 343 //______________________________________________________________________________________________
 344 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
 345 {
 346 // Reads the AliShuttleStatus from the CDB
 347
 348         if (fStatusEntry){
 349                 delete fStatusEntry;
 350                 fStatusEntry = 0;
 351         }
 352
 353         fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalCDB())
 354                 ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), GetCurrentRun());
 355
 356         if (!fStatusEntry) return 0;
 357         fStatusEntry->SetOwner(1);
 358
 359         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
 360         if (!status) {
 361                 AliError("Invalid object stored to CDB!");
 362                 return 0;
 363         }
 364
 365         return status;
 366 }
 367
 368 //______________________________________________________________________________________________
 369 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
 370 {
 371 // writes the status for one subdetector
 372
 373         if (fStatusEntry){
 374                 delete fStatusEntry;
 375                 fStatusEntry = 0;
 376         }
 377
 378         Int_t run = GetCurrentRun();
 379
 380         AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), run, run);
 381
 382         fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
 383         fStatusEntry->SetOwner(1);
 384
 385         UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
 386
 387         if (!result) {
 388                 AliError(Form("WriteShuttleStatus for %s, run %d failed", fCurrentDetector.Data(), run));
 389                 return kFALSE;
 390         }
 391
 392         return kTRUE;
 393 }
 394
 395 //______________________________________________________________________________________________
 396 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
 397 {
 398   // changes the AliShuttleStatus for the given detector and run to the given status
 399
 400         if (!fStatusEntry){
 401                 AliError("UNEXPECTED: fStatusEntry empty");
 402                 return;
 403         }
 404
 405         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
 406
 407         if (!status){
 408                 AliError("UNEXPECTED: status could not be read from current CDB entry");
 409                 return;
 410         }
 411
 412         TString actionStr = Form("UpdateShuttleStatus - %s: Changing state from %s to %s",
 413                                 fCurrentDetector.Data(),
 414                                 status->GetStatusName(),
 415                                 status->GetStatusName(newStatus));
 416         Log("SHUTTLE", actionStr);
 417         SetLastAction(actionStr);
 418
 419         status->SetStatus(newStatus);
 420         if (increaseCount) status->IncreaseCount();
 421
 422         AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
 423 }
 424 //______________________________________________________________________________________________
 425 Bool_t AliShuttle::ContinueProcessing()
 426 {
 427 // this function reads the AliShuttleStatus information from CDB and
 428 // checks if the processing should be continued
 429 // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
 430
 431         AliShuttleLogbookEntry::Status entryStatus =
 432                 fLogbookEntry->GetDetectorStatus(fCurrentDetector);
 433
 434         if(entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
 435                 Log("SHUTTLE", Form("ContinueProcessing - %s is %s",
 436                                 fCurrentDetector.Data(),
 437                                 fLogbookEntry->GetDetectorStatusName(entryStatus)));
 438                 return kFALSE;
 439         }
 440
 441         // if we get here, according to Shuttle logbook subdetector is in UNPROCESSED state
 442         AliShuttleStatus* status = ReadShuttleStatus();
 443         if (!status) {
 444                 // first time
 445                 Log("SHUTTLE", Form("ContinueProcessing - %s: Processing first time",
 446                                 fCurrentDetector.Data()));
 447                 status = new AliShuttleStatus(AliShuttleStatus::kStarted);
 448                 return WriteShuttleStatus(status);
 449         }
 450
 451         // The following two cases shouldn't happen if Shuttle Logbook was correctly updated.
 452         // If it happens it may mean Logbook updating failed... let's do it now!
 453         if (status->GetStatus() == AliShuttleStatus::kDone ||
 454             status->GetStatus() == AliShuttleStatus::kFailed){
 455                 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s. Updating Shuttle Logbook",
 456                                         fCurrentDetector.Data(),
 457                                         status->GetStatusName(status->GetStatus())));
 458                 UpdateShuttleLogbook(fCurrentDetector.Data(),
 459                                         status->GetStatusName(status->GetStatus()));
 460                 return kFALSE;
 461         }
 462
 463         if (status->GetStatus() == AliShuttleStatus::kStoreFailed) {
 464                 Log("SHUTTLE",
 465                         Form("ContinueProcessing - %s: Grid storage of one or more objects failed. Trying again now",
 466                                 fCurrentDetector.Data()));
 467                 if(TryToStoreAgain()){
 468                         Log(fCurrentDetector.Data(), "ContinueProcessing - All objects successfully stored into OCDB");
 469                         UpdateShuttleStatus(AliShuttleStatus::kDone);
 470                         UpdateShuttleLogbook(fCurrentDetector.Data(), "DONE");
 471                 } else {
 472                         Log("SHUTTLE",
 473                                 Form("ContinueProcessing - %s: Grid storage failed again",
 474                                         fCurrentDetector.Data()));
 475                 }
 476                 return kFALSE;
 477         }
 478
 479         // if we get here, there is a restart
 480
 481         // abort conditions
 482         if (status->GetCount() >= fConfig->GetMaxRetries()) {
 483                 Log("SHUTTLE",
 484                         Form("ContinueProcessing - %s failed %d times in status %s - Updating Shuttle Logbook",
 485                                 fCurrentDetector.Data(),
 486                                 status->GetCount(), status->GetStatusName()));
 487                 UpdateShuttleLogbook(fCurrentDetector.Data(), "FAILED");
 488                 return kFALSE;
 489         }
 490
 491         Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. Aborted before with %s. Retry number %d.",
 492                         fCurrentDetector.Data(),
 493                         status->GetStatusName(), status->GetCount()));
 494
 495         UpdateShuttleStatus(AliShuttleStatus::kStarted, kTRUE);
 496
 497         return kTRUE;
 498 }
 499
 500 //______________________________________________________________________________________________
 501 Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
 502 {
 503         //
 504         // Makes data retrieval for all detectors in the configuration.
 505         // entry: Shuttle logbook entry, contains run paramenters and status of detectors
 506         // (Unprocessed, Inactive, Failed or Done).
 507         // Returns kFALSE in case of error occured and kTRUE otherwise
 508         //
 509
 510         if(!entry) return kFALSE;
 511
 512         fLogbookEntry = entry;
 513
 514         if(fLogbookEntry->IsDone()){
 515                 Log("SHUTTLE","Process - Shuttle is already DONE. Updating logbook");
 516                 UpdateShuttleLogbook("shuttle_done");
 517                 fLogbookEntry = 0;
 518                 return kTRUE;
 519         }
 520
 521
 522         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^* \n",
 523                                         GetCurrentRun()));
 524
 525         fLogbookEntry->Print("all");
 526
 527         // Initialization
 528         Bool_t hasError = kFALSE;
 529         for(Int_t iSys=0;iSys<3;iSys++) fFESCalled[iSys]=kFALSE;
 530
 531         AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
 532         if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
 533         AliCDBStorage *mainRefSto = AliCDBManager::Instance()->GetStorage(fgkMainRefStorage);
 534         if(mainRefSto) mainRefSto->QueryCDB(GetCurrentRun());
 535
 536         // Loop on detectors in the configuration
 537         TIter iter(fConfig->GetDetectors());
 538         TObjString* aDetector = 0;
 539
 540         while ((aDetector = (TObjString*) iter.Next())) {
 541                 fCurrentDetector = aDetector->String();
 542
 543                 if (!fConfig->HostProcessDetector(fCurrentDetector)) continue;
 544
 545                 AliPreprocessor* aPreprocessor =
 546                         dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
 547                 if(!aPreprocessor){
 548                         Log("SHUTTLE",Form("Process - %s: no preprocessor registered. Skipping",
 549                                                         fCurrentDetector.Data()));
 550                         continue;
 551                 }
 552
 553                 if (ContinueProcessing() == kFALSE) continue;
 554
 555                 AliInfo(Form("\n\n \t\t\t****** run %d - %s: START  ******",
 556                                                 GetCurrentRun(), aDetector->GetName()));
 557
 558
 559     Int_t pid = fork();
 560
 561     if (pid < 0)
 562     {
 563       Log("SHUTTLE", "ERROR: Forking failed");
 564     }
 565     else if (pid > 0)
 566     {
 567       // parent
 568       AliInfo(Form("In parent process of %d - %s: Starting monitoring", GetCurrentRun(), aDetector->GetName()));
 569
 570       Long_t begin = time(0);
 571
 572       int status; // to be used with waitpid, on purpose an int (not Int_t)!
 573       while (waitpid(pid, &status, WNOHANG) == 0)
 574       {
 575         Long_t expiredTime = time(0) - begin;
 576
 577         if (expiredTime > fConfig->GetPPTimeOut())
 578         {
 579           Log("SHUTTLE", Form("Process time out. Run time: %d seconds. Killing...", expiredTime));
 580
 581           kill(pid, 9);
 582
 583           hasError = kTRUE;
 584
 585           gSystem->Sleep(1000);
 586         }
 587         else
 588         {
 589           if (expiredTime % 60 == 0)
 590             Log("SHUTTLE", Form("Checked process. Run time: %d seconds.", expiredTime));
 591
 592           gSystem->Sleep(1000);
 593         }
 594       }
 595
 596       AliInfo(Form("In parent process of %d - %s: Client has terminated.", GetCurrentRun(), aDetector->GetName()));
 597
 598       if (WIFEXITED(status))
 599       {
 600         Int_t returnCode = WEXITSTATUS(status);
 601
 602         Log("SHUTTLE", Form("The return code is %d", returnCode));
 603
 604         if (returnCode != 0)
 605           hasError = kTRUE;
 606       }
 607     }
 608     else if (pid == 0)
 609     {
 610       // client
 611       AliInfo(Form("In client process of %d - %s", GetCurrentRun(), aDetector->GetName()));
 612
 613       UInt_t result = ProcessCurrentDetector();
 614
 615       Int_t returnCode = 0; // will be set to 1 in case of an error
 616
 617       if (!result) {
 618         returnCode = 1;
 619         AliInfo(Form("\n \t\t\t****** run %d - %s: PREPROCESSOR ERROR ****** \n\n",
 620                 GetCurrentRun(), aDetector->GetName()));
 621       }
 622       else if(result == 2) {
 623         AliInfo(Form("\n \t\t\t****** run %d - %s: STORAGE ERROR ****** \n\n",
 624                 GetCurrentRun(), aDetector->GetName()));
 625       } else {
 626         AliInfo(Form("\n \t\t\t****** run %d - %s: DONE ****** \n\n",
 627                 GetCurrentRun(), aDetector->GetName()));
 628       }
 629
 630       if (result > 0)
 631       {
 632         // Process successful: Update time_processed field in FES logbooks!
 633         if(fFESCalled[kDAQ]) {
 634           if (UpdateDAQTable() == kFALSE)
 635             returnCode = 1;
 636           fFESlist[kDAQ].Clear();
 637         }
 638         //if(fFESCalled[kDCS]) {
 639         //  if (UpdateDCSTable(aDetector->GetName()) == kFALSE)
 640         //    returnCode = 1;
 641         //  fFESlist[kDCS].Clear();
 642         //}
 643         //if(fFESCalled[kHLT]) {
 644         //  if (UpdateHLTTable(aDetector->GetName()) == kFALSE)
 645         //    returnCode = 1;
 646         //      fFESlist[kHLT].Clear();
 647         //}
 648       }
 649
 650       AliInfo(Form("Client process of %d - %s is exiting now with %d.", GetCurrentRun(), aDetector->GetName(), returnCode));
 651
 652       // the client exits here
 653       gSystem->Exit(returnCode);
 654
 655       AliError("We should never get here!!!");
 656     }
 657         }
 658
 659         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^* \n",
 660                                                         GetCurrentRun()));
 661
 662         //check if shuttle is done for this run, if so update logbook
 663         TObjArray checkEntryArray;
 664         checkEntryArray.SetOwner(1);
 665         TString whereClause = Form("where run=%d",GetCurrentRun());
 666         if(QueryShuttleLogbook(whereClause.Data(), checkEntryArray)) {
 667
 668                 AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
 669                                                         (checkEntryArray.At(0));
 670
 671                 if(checkEntry && checkEntry->IsDone()){
 672                         Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
 673                         UpdateShuttleLogbook("shuttle_done");
 674                 }
 675         }
 676
 677         fLogbookEntry = 0;
 678
 679         return hasError == kFALSE;
 680 }
 681
 682 //______________________________________________________________________________________________
 683 UInt_t AliShuttle::ProcessCurrentDetector()
 684 {
 685         //
 686         // Makes data retrieval just for a specific detector (fCurrentDetector).
 687         // Threre should be a configuration for this detector.
 688
 689         AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), GetCurrentRun()));
 690
 691         UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
 692
 693         TString host(fConfig->GetDCSHost(fCurrentDetector));
 694         Int_t port = fConfig->GetDCSPort(fCurrentDetector);
 695
 696         TIter iter(fConfig->GetDCSAliases(fCurrentDetector));
 697         TObjString* anAlias;
 698         TMap aliasMap;
 699         aliasMap.SetOwner(1);
 700
 701         Bool_t aDCSError = kFALSE;
 702         fGridError = kFALSE;
 703
 704         while ((anAlias = (TObjString*) iter.Next())) {
 705                 TObjArray *valueSet = new TObjArray();
 706                 valueSet->SetOwner(1);
 707                 // TODO Test only... I've added a flag that allows to
 708                 // exclude DCS archive DB query
 709                 if(fgkProcessDCS){
 710                         AliInfo("Querying DCS archive DB data...");
 711                         aDCSError = (GetValueSet(host, port, anAlias->String(), valueSet) == 0);
 712                 } else {
 713                         AliInfo(Form("Skipping DCS processing. Port = %d",port));
 714                         aDCSError = kFALSE;
 715                 }
 716                 if(!aDCSError) {
 717                         aliasMap.Add(anAlias->Clone(), valueSet);
 718                 }else{
 719                         Log(fCurrentDetector, Form("ProcessCurrentDetector - Error while retrieving alias %s",
 720                                         anAlias->GetName()));
 721                         UpdateShuttleStatus(AliShuttleStatus::kDCSError, kTRUE);
 722                         aliasMap.DeleteAll();
 723                         return 0;
 724                 }
 725         }
 726
 727         // DCS Archive DB processing successful. Call Preprocessor!
 728         UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
 729
 730         AliPreprocessor* aPreprocessor =
 731                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
 732
 733         aPreprocessor->Initialize(GetCurrentRun(), GetCurrentStartTime(), GetCurrentEndTime());
 734         UInt_t aPPResult = aPreprocessor->Process(&aliasMap);
 735
 736         UInt_t returnValue = 0;
 737         if (aPPResult == 0) { // Preprocessor error
 738                 UpdateShuttleStatus(AliShuttleStatus::kPPError);
 739                 returnValue = 0;
 740         } else if (fGridError == kFALSE) { // process and Grid storage ok!
 741                 UpdateShuttleStatus(AliShuttleStatus::kDone);
 742                 UpdateShuttleLogbook(fCurrentDetector, "DONE");
 743                 Log(fCurrentDetector.Data(),
 744                         "ProcessCurrentDetector - Preprocessor and Grid storage ended successfully");
 745                 returnValue = 1;
 746         } else { // Grid storage error (process ok, but object put in local storage)
 747                 UpdateShuttleStatus(AliShuttleStatus::kStoreFailed);
 748                 returnValue = 2;
 749         }
 750
 751         aliasMap.DeleteAll();
 752
 753         return returnValue;
 754 }
 755
 756 //______________________________________________________________________________________________
 757 Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
 758                 TObjArray& entries)
 759 {
 760 // Query DAQ's Shuttle logbook and fills detector status object.
 761 // Call QueryRunParameters to query DAQ logbook for run parameters.
 762
 763         // check connection, in case connect
 764         if(!Connect(kDAQ)) return kFALSE;
 765
 766         TString sqlQuery;
 767         sqlQuery = Form("select * from logbook_shuttle %s order by run", whereClause);
 768
 769         TSQLResult* aResult = fServer[kDAQ]->Query(sqlQuery);
 770         if (!aResult) {
 771                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
 772                 return kFALSE;
 773         }
 774
 775         if(aResult->GetRowCount() == 0) {
 776                 if(sqlQuery.Contains("where shuttle_done=0")){
 777                         Log("SHUTTLE", "QueryShuttleLogbook - All runs in Shuttle Logbook are already DONE");
 778                         delete aResult;
 779                         return kTRUE;
 780                 } else {
 781                         AliError("No entries in Shuttle Logbook match request");
 782                         delete aResult;
 783                         return kFALSE;
 784                 }
 785         }
 786
 787         // TODO Check field count!
 788         const UInt_t nCols = 24;
 789         if (aResult->GetFieldCount() != (Int_t) nCols) {
 790                 AliError("Invalid SQL result field number!");
 791                 delete aResult;
 792                 return kFALSE;
 793         }
 794
 795         entries.SetOwner(1);
 796
 797         TSQLRow* aRow;
 798         while ((aRow = aResult->Next())) {
 799                 TString runString(aRow->GetField(0), aRow->GetFieldLength(0));
 800                 Int_t run = runString.Atoi();
 801
 802                 AliShuttleLogbookEntry *entry = QueryRunParameters(run);
 803                 if (!entry)
 804                         continue;
 805
 806                 // loop on detectors
 807                 for(UInt_t ii = 0; ii < nCols; ii++)
 808                         entry->SetDetectorStatus(aResult->GetFieldName(ii), aRow->GetField(ii));
 809
 810                 entries.AddLast(entry);
 811                 delete aRow;
 812         }
 813
 814         if(sqlQuery.Contains("where shuttle_done=0"))
 815                 Log("SHUTTLE", Form("QueryShuttleLogbook - Found %d unprocessed runs in Shuttle Logbook",
 816                                                         entries.GetEntriesFast()));
 817         delete aResult;
 818         return kTRUE;
 819 }
 820
 821 //______________________________________________________________________________________________
 822 AliShuttleLogbookEntry* AliShuttle::QueryRunParameters(Int_t run)
 823 {
 824         //
 825         // Retrieve run parameters written in the DAQ logbook and sets them into AliShuttleLogbookEntry object
 826         //
 827
 828         // check connection, in case connect
 829         if (!Connect(kDAQ))
 830                 return 0;
 831
 832         TString sqlQuery;
 833         sqlQuery.Form("select * from logbook where run=%d", run);
 834
 835         TSQLResult* aResult = fServer[kDAQ]->Query(sqlQuery);
 836         if (!aResult) {
 837                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
 838                 return 0;
 839         }
 840
 841         if (aResult->GetRowCount() == 0) {
 842                 Log("SHUTTLE", Form("QueryRunParameters - No entry in DAQ Logbook for run %d. Skipping", run));
 843                 delete aResult;
 844                 return 0;
 845         }
 846
 847         if (aResult->GetRowCount() > 1) {
 848                 AliError(Form("More than one entry in DAQ Logbook for run %d. Skipping", run));
 849                 delete aResult;
 850                 return 0;
 851         }
 852
 853         TSQLRow* aRow = aResult->Next();
 854         if (!aRow)
 855         {
 856                 AliError(Form("Could not retrieve row for run %d. Skipping", run));
 857                 delete aResult;
 858                 return 0;
 859         }
 860
 861         AliShuttleLogbookEntry* entry = new AliShuttleLogbookEntry(run);
 862
 863         for (Int_t ii = 0; ii < aResult->GetFieldCount(); ii++)
 864                 entry->SetRunParameter(aResult->GetFieldName(ii), aRow->GetField(ii));
 865
 866         UInt_t startTime = entry->GetStartTime();
 867         UInt_t endTime = entry->GetEndTime();
 868
 869         if (!startTime || !endTime || startTime > endTime) {
 870                 Log("SHUTTLE",
 871                         Form("QueryRunParameters - Invalid parameters for Run %d: startTime = %d, endTime = %d",
 872                                 run, startTime, endTime));
 873                 delete entry;
 874                 delete aRow;
 875                 delete aResult;
 876                 return 0;
 877         }
 878
 879         delete aRow;
 880         delete aResult;
 881
 882         return entry;
 883 }
 884
 885 //______________________________________________________________________________________________
 886 Bool_t AliShuttle::TryToStoreAgain()
 887 {
 888   // Called in case the detector failed to store the object in Grid OCDB
 889   // It tries to store the object again, if it does not find more recent and overlapping objects
 890   // Calls underlying TryToStoreAgain(const char*) function twice, for OCDB and Reference storage.
 891
 892         AliInfo("Trying to store OCDB data again...");
 893         Bool_t resultCDB = TryToStoreAgain(fgkMainCDB);
 894
 895         AliInfo("Trying to store reference data again...");
 896         Bool_t resultRef = TryToStoreAgain(fgkMainRefStorage);
 897
 898         return resultCDB && resultRef;
 899 }
 900
 901 //______________________________________________________________________________________________
 902 Bool_t AliShuttle::TryToStoreAgain(TString& gridURI)
 903 {
 904   // Called by TryToStoreAgain(), performs actual storage retry
 905
 906         TObjArray* gridIds=0;
 907
 908         Bool_t result = kTRUE;
 909
 910         const char* type = 0;
 911         TString backupURI;
 912         if(gridURI == fgkMainCDB) {
 913                 type = "OCDB";
 914                 backupURI = fgkLocalCDB;
 915         } else if(gridURI == fgkMainRefStorage) {
 916                 type = "reference";
 917                 backupURI = fgkLocalRefStorage;
 918         } else {
 919                 AliError(Form("Invalid storage URI: %s", gridURI.Data()));
 920                 return kFALSE;
 921         }
 922
 923         AliCDBManager* man = AliCDBManager::Instance();
 924
 925         AliCDBStorage *gridSto = man->GetStorage(gridURI);
 926         if(!gridSto) {
 927                 Log(fCurrentDetector.Data(),
 928                         Form("TryToStoreAgain - cannot activate main %s storage", type));
 929                 return kFALSE;
 930         }
 931
 932         gridIds = gridSto->GetQueryCDBList();
 933
 934         // get objects previously stored in local CDB
 935         AliCDBStorage *backupSto = man->GetStorage(backupURI);
 936         AliCDBPath aPath(GetOfflineDetName(fCurrentDetector.Data()),"*","*");
 937         // Local objects were stored with current run as Grid version!
 938         TList* localEntries = backupSto->GetAll(aPath.GetPath(), GetCurrentRun(), GetCurrentRun());
 939         localEntries->SetOwner(1);
 940
 941         // loop on local stored objects
 942         TIter localIter(localEntries);
 943         AliCDBEntry *aLocEntry = 0;
 944         while((aLocEntry = dynamic_cast<AliCDBEntry*> (localIter.Next()))){
 945                 aLocEntry->SetOwner(1);
 946                 AliCDBId aLocId = aLocEntry->GetId();
 947                 aLocEntry->SetVersion(-1);
 948                 aLocEntry->SetSubVersion(-1);
 949
 950                 // loop on Grid valid Id's
 951                 Bool_t store = kTRUE;
 952                 TIter gridIter(gridIds);
 953                 AliCDBId* aGridId = 0;
 954                 while((aGridId = dynamic_cast<AliCDBId*> (gridIter.Next()))){
 955                         // If local object is valid up to infinity we store it anyway
 956                         // TODO This does not work! It may hide more recent objects...
 957                         if(aLocId.GetLastRun() == AliCDBRunRange::Infinity()) {
 958                                 // TODO Check that it won't hide more recent files! how????
 959                                 break;
 960                         }
 961                         if(aGridId->GetPath() != aLocId.GetPath()) continue;
 962                         // skip all objects valid up to infinity
 963                         if(aGridId->GetLastRun() == AliCDBRunRange::Infinity()) continue;
 964                         // if we get here, it means there's already some more recent object stored on Grid!
 965                         store = kFALSE;
 966                         break;
 967                 }
 968
 969                 if(!store){
 970                         Log(fCurrentDetector.Data(),
 971                                 Form("TryToStoreAgain - A more recent object already exists in %s storage: <%s>",
 972                                         type, aGridId->ToString().Data()));
 973                         // removing local filename...
 974                         // TODO maybe it's better not to remove it, it was not copied to the Grid!
 975                         TString filename;
 976                         backupSto->IdToFilename(aLocId, filename);
 977                         AliInfo(Form("Removing local file %s", filename.Data()));
 978                         gSystem->Exec(Form("rm %s",filename.Data()));
 979                         continue;
 980                 }
 981
 982                 // If we get here, the file can be stored!
 983                 Bool_t storeOk = gridSto->Put(aLocEntry);
 984                 if(storeOk){
 985                         Log(fCurrentDetector.Data(),
 986                                 Form("TryToStoreAgain - Object <%s> successfully put into %s storage",
 987                                         aLocId.ToString().Data(), type));
 988
 989                         // removing local filename...
 990                         TString filename;
 991                         backupSto->IdToFilename(aLocId, filename);
 992                         AliInfo(Form("Removing local file %s", filename.Data()));
 993                         gSystem->Exec(Form("rm %s", filename.Data()));
 994                         continue;
 995                 } else  {
 996                         Log(fCurrentDetector.Data(),
 997                                 Form("TryToStoreAgain - Grid %s storage of object <%s> failed again",
 998                                         type, aLocId.ToString().Data()));
 999                         result = kFALSE;
1000                 }
1001         }
1002         localEntries->Clear();
1003
1004         return result;
1005 }
1006
1007 //______________________________________________________________________________________________
1008 Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* alias,
1009                                 TObjArray* valueSet)
1010 {
1011 // Retrieve all "alias" data points from the DCS server
1012 // host, port: TSocket connection parameters
1013 // alias: name of the alias
1014 // valueSet: array of retrieved AliDCSValue's
1015
1016         AliDCSClient client(host, port, fTimeout, fRetries);
1017         if (!client.IsConnected()) {
1018                 return kFALSE;
1019         }
1020
1021         Int_t result = client.GetAliasValues(alias,
1022                 GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
1023
1024         if (result < 0) {
1025                 Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get '%s'! Reason: %s",
1026                         alias, AliDCSClient::GetErrorString(result)));
1027
1028                 if (result == AliDCSClient::fgkServerError) {
1029                         Log(fCurrentDetector.Data(), Form("GetValueSet - Server error: %s",
1030                                 client.GetServerError().Data()));
1031                 }
1032
1033                 return kFALSE;
1034         }
1035
1036         return kTRUE;
1037 }
1038
1039 //______________________________________________________________________________________________
1040 const char* AliShuttle::GetFile(Int_t system, const char* detector,
1041                 const char* id, const char* source)
1042 {
1043 // Get calibration file from file exchange servers
1044 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
1045
1046         switch(system){
1047                 case kDAQ:
1048                         return GetDAQFileName(detector, id, source);
1049                         break;
1050                 case kDCS:
1051                         return GetDCSFileName(detector, id, source);
1052                         break;
1053                 case kHLT:
1054                         return GetHLTFileName(detector, id, source);
1055                         break;
1056                 default:
1057                         AliError(Form("No valid system index: %d",system));
1058         }
1059
1060         return 0;
1061 }
1062
1063 //______________________________________________________________________________________________
1064 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
1065 {
1066 // Get sources producing the condition file Id from file exchange servers
1067 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
1068
1069         switch(system){
1070                 case kDAQ:
1071                         return GetDAQFileSources(detector, id);
1072                         break;
1073                 case kDCS:
1074                         return GetDCSFileSources(detector, id);
1075                         break;
1076                 case kHLT:
1077                         return GetHLTFileSources(detector, id);
1078                         break;
1079                 default:
1080                         AliError(Form("No valid system index: %d",system));
1081         }
1082
1083         return NULL;
1084 }
1085
1086 //______________________________________________________________________________________________
1087 Bool_t AliShuttle::Connect(Int_t system)
1088 {
1089 // Connect to MySQL Server of the system's FES logbook
1090 // DAQ Logbook, Shuttle Logbook and DAQ FES Logbook are on the same host
1091
1092         // check connection: if already connected return
1093         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
1094
1095         TString aFESlbHost= Form("mysql://%s", fConfig->GetFESlbHost(system));
1096
1097         fServer[system] = TSQLServer::Connect(aFESlbHost,
1098                         fConfig->GetFESlbUser(system),
1099                         fConfig->GetFESlbPass(system));
1100         if (!fServer[system] || !fServer[system]->IsConnected()) {
1101                 AliError(Form("Can't establish connection to FES logbook for %s",
1102                                         AliShuttleInterface::GetSystemName(system)));
1103                 if(fServer[system]) delete fServer[system];
1104                 return kFALSE;
1105         }
1106
1107         // Get tables
1108         // TODO in the configuration should the table name be there too?
1109         TSQLResult* aResult=0;
1110         switch(system){
1111                 case kDAQ:
1112                         aResult = fServer[kDAQ]->GetTables("REFSYSLOG");
1113                         break;
1114                 case kDCS:
1115                         //aResult = fServer[kDCS]->GetTables("REFSYSLOG");
1116                         break;
1117                 case kHLT:
1118                         //aResult = fServer[kHLT]->GetTables("REFSYSLOG");
1119                         break;
1120                 default:
1121                         break;
1122         }
1123
1124         delete aResult;
1125         return kTRUE;
1126 }
1127
1128 //______________________________________________________________________________________________
1129 const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source)
1130 {
1131 // Retrieves a file from the DAQ FES.
1132 // First queris the DAQ logbook_fs for the DAQ file name, using the run, detector, id and source info
1133 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
1134 // run: current run being processed (given by Logbook entry fLogbookEntry)
1135 // detector: the Preprocessor name
1136 // id: provided as a parameter by the Preprocessor
1137 // source: provided by the Preprocessor through GetFileSources function
1138
1139         // check connection, in case connect
1140         if (!Connect(kDAQ))
1141         {
1142                 Log(detector, "GetDAQFileName - Couldn't connect to DAQ Logbook");
1143                 return 0;
1144         }
1145
1146         // Query preparation
1147         TString sqlQueryStart = "select filePath from logbook_fs where";
1148         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
1149                                 GetCurrentRun(), detector, id, source);
1150         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1151
1152         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1153
1154         // Query execution
1155         TSQLResult* aResult = 0;
1156         aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1157         if (!aResult) {
1158                 Log(detector, Form("GetDAQFileName - Can't execute SQL query for: id = %s, source = %s",
1159                                 id, source));
1160                 return 0;
1161         }
1162
1163         if(aResult->GetRowCount() == 0)
1164         {
1165                 Log(detector,
1166                         Form("GetDAQFileName - No entry in FES table for: id = %s, source = %s",
1167                                 id, source));
1168                 delete aResult;
1169                 return 0;
1170         }
1171
1172         if (aResult->GetRowCount() > 1) {
1173                 Log(detector,
1174                         Form("GetDAQFileName - More than one entry in FES table for: id = %s, source = %s",
1175                                 id, source));
1176                 delete aResult;
1177                 return 0;
1178         }
1179
1180         TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
1181
1182         if (!aRow){
1183                 Log(detector, Form("GetDAQFileName - Empty set result from query: id = %s, source = %s",
1184                                 id, source));
1185                 delete aResult;
1186                 return 0;
1187         }
1188
1189         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
1190
1191         delete aResult;
1192         delete aRow;
1193
1194         AliDebug(2, Form("filePath = %s",filePath.Data()));
1195
1196         // retrieved file is renamed to make it unique
1197         TString localFileName = Form("%s_%d_%s_%s.shuttle",
1198                                         detector, GetCurrentRun(), id, source);
1199
1200         // file retrieval from DAQ FES
1201         Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
1202         if(!result) {
1203                 Log(detector, Form("GetDAQFileName - Copy of file %s from DAQ FES failed", filePath.Data()));
1204                 return 0;
1205         } else {
1206                 AliInfo(Form("File %s copied from DAQ FES into %s/%s",
1207                         filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
1208         }
1209
1210
1211         fFESCalled[kDAQ]=kTRUE;
1212         TObjString *fileParams = new TObjString(Form("%s_!?!_%s", id, source));
1213         fFESlist[kDAQ].Add(fileParams);
1214
1215         return localFileName.Data();
1216
1217 }
1218
1219 //______________________________________________________________________________________________
1220 Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName)
1221 {
1222
1223         // check temp directory: trying to cd to temp; if it does not exist, create it
1224         AliDebug(2, Form("Copy file %s from DAQ FES into folder %s and rename it as %s",
1225                         daqFileName,fgkShuttleTempDir, localFileName));
1226
1227         void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
1228         if (dir == NULL) {
1229                 if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
1230                         AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
1231                         return kFALSE;
1232                 }
1233
1234         } else {
1235                 gSystem->FreeDirectory(dir);
1236         }
1237
1238         TString baseDAQFESFolder = "DAQ";
1239         TString command = Form("scp %s@%s:%s/%s %s/%s",
1240                 fConfig->GetFESUser(kDAQ),
1241                 fConfig->GetFESHost(kDAQ),
1242                 baseDAQFESFolder.Data(),
1243                 daqFileName,
1244                 fgkShuttleTempDir,
1245                 localFileName);
1246
1247         AliDebug(2, Form("%s",command.Data()));
1248
1249         UInt_t nRetries = 0;
1250         UInt_t maxRetries = 3;
1251
1252         // copy!! if successful TSystem::Exec returns 0
1253         while(nRetries++ < maxRetries) {
1254                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
1255                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
1256         }
1257
1258         return kFALSE;
1259
1260 }
1261
1262 //______________________________________________________________________________________________
1263 TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id)
1264 {
1265 // Retrieves a file from the DCS FES.
1266
1267         // check connection, in case connect
1268         if(!Connect(kDAQ)){
1269                 Log(detector, "GetDAQFileSources - Couldn't connect to DAQ Logbook");
1270                 return 0;
1271         }
1272
1273         // Query preparation
1274         TString sqlQueryStart = "select DAQsource from logbook_fs where";
1275         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
1276                                 GetCurrentRun(), detector, id);
1277         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1278
1279         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1280
1281         // Query execution
1282         TSQLResult* aResult;
1283         aResult = fServer[kDAQ]->Query(sqlQuery);
1284         if (!aResult) {
1285                 Log(detector, Form("GetDAQFileSources - Can't execute SQL query for id: %s", id));
1286                 return 0;
1287         }
1288
1289         if (aResult->GetRowCount() == 0) {
1290                 Log(detector,
1291                         Form("GetDAQFileSources - No entry in FES table for id: %s", id));
1292                 delete aResult;
1293                 return 0;
1294         }
1295
1296         TSQLRow* aRow;
1297         TList *list = new TList();
1298         list->SetOwner(1);
1299
1300         while((aRow = aResult->Next())){
1301
1302                 TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
1303                 AliDebug(2, Form("daqSource = %s", daqSource.Data()));
1304                 list->Add(new TObjString(daqSource));
1305                 delete aRow;
1306         }
1307         delete aResult;
1308
1309         return list;
1310
1311 }
1312
1313 //______________________________________________________________________________________________
1314 const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1315 // Retrieves a file from the DCS FES.
1316
1317 return "You're in DCS";
1318
1319 }
1320
1321 //______________________________________________________________________________________________
1322 TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
1323 // Retrieves a file from the DCS FES.
1324
1325 return NULL;
1326
1327 }
1328
1329 //______________________________________________________________________________________________
1330 const char* AliShuttle::GetHLTFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1331 // Retrieves a file from the HLT FES.
1332
1333 return "You're in HLT";
1334
1335 }
1336
1337 //______________________________________________________________________________________________
1338 TList* AliShuttle::GetHLTFileSources(const char* /*detector*/, const char* /*id*/){
1339 // Retrieves a file from the HLT FES.
1340
1341 return NULL;
1342
1343 }
1344
1345 //______________________________________________________________________________________________
1346 Bool_t AliShuttle::UpdateDAQTable()
1347 {
1348 // Update DAQ table filling time_processed field in all rows corresponding to current run and detector
1349
1350         // check connection, in case connect
1351         if(!Connect(kDAQ)){
1352                 Log(fCurrentDetector, "UpdateDAQTable - Couldn't connect to DAQ Logbook");
1353                 return kFALSE;
1354         }
1355
1356         TTimeStamp now; // now
1357
1358         // Loop on FES list entries
1359         TIter iter(&fFESlist[kDAQ]);
1360         TObjString *aFESentry=0;
1361         while((aFESentry = dynamic_cast<TObjString*> (iter.Next()))){
1362                 TString aFESentrystr = aFESentry->String();
1363                 TObjArray *aFESarray = aFESentrystr.Tokenize("_!?!_");
1364                 if(!aFESarray || aFESarray->GetEntries() != 2 ) {
1365                         Log(fCurrentDetector, Form("UpdateDAQTable - error updating FES entry. Check string: <%s>",
1366                                 aFESentrystr.Data()));
1367                         if(aFESarray) delete aFESarray;
1368                         return kFALSE;
1369                 }
1370                 const char* fileId = ((TObjString*) aFESarray->At(0))->GetName();
1371                 const char* daqSource = ((TObjString*) aFESarray->At(1))->GetName();
1372                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
1373                         GetCurrentRun(), fCurrentDetector.Data(), fileId, daqSource);
1374
1375                 delete aFESarray;
1376
1377                 TString sqlQuery = Form("update logbook_fs set time_processed=%d %s", now.GetSec(), whereClause.Data());
1378
1379                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1380
1381                 // Query execution
1382                 TSQLResult* aResult;
1383                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1384                 if (!aResult) {
1385                         Log(fCurrentDetector, Form("UpdateDAQTable - Can't execute SQL query <%s>", sqlQuery.Data()));
1386                         return kFALSE;
1387                 }
1388                 delete aResult;
1389         }
1390
1391         return kTRUE;
1392 }
1393
1394
1395 //______________________________________________________________________________________________
1396 Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status)
1397 {
1398 // Update Shuttle logbook filling detector or shuttle_done column
1399 // ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
1400
1401         // check connection, in case connect
1402         if(!Connect(kDAQ)){
1403                 Log("SHUTTLE", "UpdateShuttleLogbook - Couldn't connect to DAQ Logbook.");
1404                 return kFALSE;
1405         }
1406
1407         TString detName(detector);
1408         TString setClause;
1409         if(detName == "shuttle_done") {
1410                 setClause = "set shuttle_done=1";
1411         } else {
1412                 TString statusStr(status);
1413                 if(statusStr.Contains("done", TString::kIgnoreCase) ||
1414                    statusStr.Contains("failed", TString::kIgnoreCase)){
1415                         setClause = Form("set %s=\"%s\"", detector, status);
1416                 } else {
1417                         Log("SHUTTLE",
1418                                 Form("UpdateShuttleLogbook - Invalid status <%s> for detector %s",
1419                                         status, detector));
1420                         return kFALSE;
1421                 }
1422         }
1423
1424         TString whereClause = Form("where run=%d", GetCurrentRun());
1425
1426         TString sqlQuery = Form("update logbook_shuttle %s %s",
1427                                         setClause.Data(), whereClause.Data());
1428
1429         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1430
1431         // Query execution
1432         TSQLResult* aResult;
1433         aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1434         if (!aResult) {
1435                 Log("SHUTTLE", Form("UpdateShuttleLogbook - Can't execute query <%s>", sqlQuery.Data()));
1436                 return kFALSE;
1437         }
1438         delete aResult;
1439
1440         return kTRUE;
1441 }
1442
1443 //______________________________________________________________________________________________
1444 Int_t AliShuttle::GetCurrentRun() const
1445 {
1446 // Get current run from logbook entry
1447
1448         return fLogbookEntry ? fLogbookEntry->GetRun() : -1;
1449 }
1450
1451 //______________________________________________________________________________________________
1452 UInt_t AliShuttle::GetCurrentStartTime() const
1453 {
1454 // get current start time
1455
1456         return fLogbookEntry ? fLogbookEntry->GetStartTime() : 0;
1457 }
1458
1459 //______________________________________________________________________________________________
1460 UInt_t AliShuttle::GetCurrentEndTime() const
1461 {
1462 // get current end time from logbook entry
1463
1464         return fLogbookEntry ? fLogbookEntry->GetEndTime() : 0;
1465 }
1466
1467 //______________________________________________________________________________________________
1468 void AliShuttle::Log(const char* detector, const char* message)
1469 {
1470 // Fill log string with a message
1471
1472         void* dir = gSystem->OpenDirectory(fgkShuttleLogDir);
1473         if (dir == NULL) {
1474                 if (gSystem->mkdir(fgkShuttleLogDir, kTRUE)) {
1475                         AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
1476                         return;
1477                 }
1478
1479         } else {
1480                 gSystem->FreeDirectory(dir);
1481         }
1482
1483         TString toLog = Form("%s (%d): %s - ", TTimeStamp(time(0)).AsString("s"), getpid(), detector);
1484         if(GetCurrentRun()>=0 ) toLog += Form("run %d - ", GetCurrentRun());
1485         toLog += Form("%s", message);
1486
1487         AliInfo(toLog.Data());
1488
1489         TString fileName;
1490         fileName.Form("%s/%s.log", fgkShuttleLogDir, detector);
1491         gSystem->ExpandPathName(fileName);
1492
1493         ofstream logFile;
1494         logFile.open(fileName, ofstream::out | ofstream::app);
1495
1496         if (!logFile.is_open()) {
1497                 AliError(Form("Could not open file %s", fileName.Data()));
1498                 return;
1499         }
1500
1501         logFile << toLog.Data() << "\n";
1502
1503         logFile.close();
1504 }
1505
1506 //______________________________________________________________________________________________
1507 Bool_t AliShuttle::Collect(Int_t run)
1508 {
1509 //
1510 // Collects conditions data for all UNPROCESSED run written to DAQ LogBook in case of run = -1 (default)
1511 // If a dedicated run is given this run is processed
1512 //
1513 // In operational mode, this is the Shuttle function triggered by the EOR signal.
1514 //
1515
1516         if (run == -1)
1517                 Log("SHUTTLE","Collect - Shuttle called. Collecting conditions data for unprocessed runs");
1518         else
1519                 Log("SHUTTLE", Form("Collect - Shuttle called. Collecting conditions data for run %d", run));
1520
1521         SetLastAction("Starting");
1522
1523         TString whereClause("where shuttle_done=0");
1524         if (run != -1)
1525                 whereClause += Form(" and run=%d", run);
1526
1527         TObjArray shuttleLogbookEntries;
1528         if (!QueryShuttleLogbook(whereClause, shuttleLogbookEntries)) {
1529                 Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
1530                 return kFALSE;
1531         }
1532
1533         if (!RetrieveConditionsData(shuttleLogbookEntries)) {
1534                 Log("SHUTTLE", "Collect - Process of at least one run failed");
1535                 return kFALSE;
1536         }
1537
1538         return kTRUE;
1539 }
1540
1541 //______________________________________________________________________________________________
1542 Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
1543 {
1544 // Retrieve conditions data for all runs that aren't processed yet
1545
1546         Bool_t hasError = kFALSE;
1547
1548         TIter iter(&dateEntries);
1549         AliShuttleLogbookEntry* anEntry;
1550
1551         while ((anEntry = (AliShuttleLogbookEntry*) iter.Next())){
1552                 if (!Process(anEntry)){
1553                         hasError = kTRUE;
1554                 }
1555         }
1556
1557         return hasError == kFALSE;
1558 }
1559
1560 //______________________________________________________________________________________________
1561 ULong_t AliShuttle::GetTimeOfLastAction() const
1562 {
1563         ULong_t tmp;
1564
1565         fMonitoringMutex->Lock();
1566
1567         tmp = fLastActionTime;
1568
1569         fMonitoringMutex->UnLock();
1570
1571         return tmp;
1572 }
1573
1574 //______________________________________________________________________________________________
1575 const TString AliShuttle::GetLastAction() const
1576 {
1577         // returns a string description of the last action
1578
1579         TString tmp;
1580
1581         fMonitoringMutex->Lock();
1582
1583         tmp = fLastAction;
1584
1585         fMonitoringMutex->UnLock();
1586
1587         return tmp;
1588 }
1589
1590 //______________________________________________________________________________________________
1591 void AliShuttle::SetLastAction(const char* action)
1592 {
1593         // updates the monitoring variables
1594
1595         fMonitoringMutex->Lock();
1596
1597         fLastAction = action;
1598         fLastActionTime = time(0);
1599
1600         fMonitoringMutex->UnLock();
1601 }
1602
1603 //______________________________________________________________________________________________
1604 const char* AliShuttle::GetRunParameter(const char* param)
1605 {
1606 // returns run parameter read from DAQ logbook
1607
1608         if(!fLogbookEntry) {
1609                 AliError("No logbook entry!");
1610                 return 0;
1611         }
1612
1613         return fLogbookEntry->GetRunParameter(param);
1614 }