SHUTTLE/AliShuttle.cxx

   1 /**************************************************************************
   2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
   3  *                                                                        *
   4  * Author: The ALICE Off-line Project.                                    *
   5  * Contributors are mentioned in the code where appropriate.              *
   6  *                                                                        *
   7  * Permission to use, copy, modify and distribute this software and its   *
   8  * documentation strictly for non-commercial purposes is hereby granted   *
   9  * without fee, provided that the above copyright notice appears in all   *
  10  * copies and that both the copyright notice and this permission notice   *
  11  * appear in the supporting documentation. The authors make no claims     *
  12  * about the suitability of this software for any purpose. It is          *
  13  * provided "as is" without express or implied warranty.                  *
  14  **************************************************************************/
  15
  16 /*
  17 $Log$
  18 Revision 1.15  2006/10/02 16:38:39  jgrosseo
  19 update (alberto):
  20 fixed memory leaks
  21 storing of objects that failed to be stored to the grid before
  22 interfacing of shuttle status table in daq system
  23
  24 Revision 1.14  2006/08/29 09:16:05  jgrosseo
  25 small update
  26
  27 Revision 1.13  2006/08/15 10:50:00  jgrosseo
  28 effc++ corrections (alberto)
  29
  30 Revision 1.12  2006/08/08 14:19:29  jgrosseo
  31 Update to shuttle classes (Alberto)
  32
  33 - Possibility to set the full object's path in the Preprocessor's and
  34 Shuttle's  Store functions
  35 - Possibility to extend the object's run validity in the same classes
  36 ("startValidity" and "validityInfinite" parameters)
  37 - Implementation of the StoreReferenceData function to store reference
  38 data in a dedicated CDB storage.
  39
  40 Revision 1.11  2006/07/21 07:37:20  jgrosseo
  41 last run is stored after each run
  42
  43 Revision 1.10  2006/07/20 09:54:40  jgrosseo
  44 introducing status management: The processing per subdetector is divided into several steps,
  45 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
  46 can keep track of the number of failures and skips further processing after a certain threshold is
  47 exceeded. These thresholds can be configured in LDAP.
  48
  49 Revision 1.9  2006/07/19 10:09:55  jgrosseo
  50 new configuration, accesst to DAQ FES (Alberto)
  51
  52 Revision 1.8  2006/07/11 12:44:36  jgrosseo
  53 adding parameters for extended validity range of data produced by preprocessor
  54
  55 Revision 1.7  2006/07/10 14:37:09  jgrosseo
  56 small fix + todo comment
  57
  58 Revision 1.6  2006/07/10 13:01:41  jgrosseo
  59 enhanced storing of last sucessfully processed run (alberto)
  60
  61 Revision 1.5  2006/07/04 14:59:57  jgrosseo
  62 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
  63
  64 Revision 1.4  2006/06/12 09:11:16  jgrosseo
  65 coding conventions (Alberto)
  66
  67 Revision 1.3  2006/06/06 14:26:40  jgrosseo
  68 o) removed files that were moved to STEER
  69 o) shuttle updated to follow the new interface (Alberto)
  70
  71 Revision 1.2  2006/03/07 07:52:34  hristov
  72 New version (B.Yordanov)
  73
  74 Revision 1.6  2005/11/19 17:19:14  byordano
  75 RetrieveDATEEntries and RetrieveConditionsData added
  76
  77 Revision 1.5  2005/11/19 11:09:27  byordano
  78 AliShuttle declaration added
  79
  80 Revision 1.4  2005/11/17 17:47:34  byordano
  81 TList changed to TObjArray
  82
  83 Revision 1.3  2005/11/17 14:43:23  byordano
  84 import to local CVS
  85
  86 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
  87 Initial import as subdirectory in AliRoot
  88
  89 Revision 1.2  2005/09/13 08:41:15  byordano
  90 default startTime endTime added
  91
  92 Revision 1.4  2005/08/30 09:13:02  byordano
  93 some docs added
  94
  95 Revision 1.3  2005/08/29 21:15:47  byordano
  96 some docs added
  97
  98 */
  99
 100 //
 101 // This class is the main manager for AliShuttle.
 102 // It organizes the data retrieval from DCS and call the
 103 // interface methods of AliPreprocessor.
 104 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
 105 // data for its set of aliases is retrieved. If there is registered
 106 // AliPreprocessor for this detector then it will be used
 107 // accroding to the schema (see AliPreprocessor).
 108 // If there isn't registered AliPreprocessor than the retrieved
 109 // data is stored automatically to the undelying AliCDBStorage.
 110 // For detSpec is used the alias name.
 111 //
 112
 113 #include "AliShuttle.h"
 114
 115 #include "AliCDBManager.h"
 116 #include "AliCDBStorage.h"
 117 #include "AliCDBId.h"
 118 #include "AliCDBRunRange.h"
 119 #include "AliCDBPath.h"
 120 #include "AliCDBEntry.h"
 121 #include "AliShuttleConfig.h"
 122 #include "AliDCSClient.h"
 123 #include "AliLog.h"
 124 #include "AliPreprocessor.h"
 125 #include "AliShuttleStatus.h"
 126 #include "AliShuttleLogbookEntry.h"
 127
 128 #include <TSystem.h>
 129 #include <TObject.h>
 130 #include <TString.h>
 131 #include <TTimeStamp.h>
 132 #include <TObjString.h>
 133 #include <TSQLServer.h>
 134 #include <TSQLResult.h>
 135 #include <TSQLRow.h>
 136
 137 #include <fstream>
 138
 139 ClassImp(AliShuttle)
 140
 141 TString AliShuttle::fgkMainCDB("alien://folder=ShuttleCDB");
 142 TString AliShuttle::fgkLocalCDB("local://LocalShuttleCDB");
 143 TString AliShuttle::fgkMainRefStorage("alien://folder=ShuttleReference");
 144 TString AliShuttle::fgkLocalRefStorage("local://LocalReferenceStorage");
 145
 146 Bool_t AliShuttle::fgkProcessDCS(kTRUE);
 147
 148
 149 const char* AliShuttle::fgkShuttleTempDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/temp");
 150 const char* AliShuttle::fgkShuttleLogDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/log");
 151
 152 const char* AliShuttle::fgkDetectorName[AliShuttle::kNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
 153         "PHOS", "CPV", "RICH", "EMCAL", "MUON_TRK", "MUON_TRG", "FMD", "ZDC", "PMD", "START", "VZERO"};
 154
 155 const char* AliShuttle::fgkDetectorCode[AliShuttle::kNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
 156         "PHS", "CPV", "HMP", "EMC", "MCH", "MTR", "FMD", "ZDC", "PMD", "T00", "V00"};
 157
 158 //______________________________________________________________________________________________
 159 AliShuttle::AliShuttle(const AliShuttleConfig* config,
 160                 UInt_t timeout, Int_t retries):
 161 fConfig(config),
 162 fTimeout(timeout), fRetries(retries),
 163 fPreprocessorMap(),
 164 fLogbookEntry(0),
 165 fCurrentDetector(""),
 166 fStatusEntry(0),
 167 fGridError(kFALSE)
 168 {
 169         //
 170         // config: AliShuttleConfig used
 171         // timeout: timeout used for AliDCSClient connection
 172         // retries: the number of retries in case of connection error.
 173         //
 174
 175         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
 176         for(int iSys=0;iSys<3;iSys++) {
 177                 fServer[iSys]=0;
 178                 fFESlist[iSys].SetOwner(kTRUE);
 179         }
 180         fPreprocessorMap.SetOwner(kTRUE);
 181 }
 182
 183 //______________________________________________________________________
 184 AliShuttle::AliShuttle(const AliShuttle& /*other*/):
 185 AliShuttleInterface(),
 186 fConfig(0),
 187 fTimeout(0), fRetries(0),
 188 fPreprocessorMap(),
 189 fLogbookEntry(0),
 190 fCurrentDetector(""),
 191 fStatusEntry(0),
 192 fGridError(kFALSE)
 193 {
 194 // copy constructor (not implemented)
 195
 196 }
 197
 198 //______________________________________________________________________
 199 AliShuttle &AliShuttle::operator=(const AliShuttle& /*other*/)
 200 {
 201 // assignment operator (not implemented)
 202
 203 return *this;
 204 }
 205
 206 //______________________________________________________________________________________________
 207 AliShuttle::~AliShuttle()
 208 {
 209 // destructor
 210
 211         fPreprocessorMap.DeleteAll();
 212         for(int iSys=0;iSys<3;iSys++)
 213                 if(fServer[iSys]) {
 214                         fServer[iSys]->Close();
 215                         delete fServer[iSys];
 216                 }
 217
 218         if (fStatusEntry){
 219                 delete fStatusEntry;
 220                 fStatusEntry = 0;
 221         }
 222 }
 223
 224 //______________________________________________________________________________________________
 225 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
 226 {
 227         //
 228         // Registers new AliPreprocessor.
 229         // It uses GetName() for indentificator of the pre processor.
 230         // The pre processor is registered it there isn't any other
 231         // with the same identificator (GetName()).
 232         //
 233
 234         if (fPreprocessorMap.GetValue(preprocessor->GetName())) {
 235                 AliWarning(Form("AliPreprocessor %s is already registered!",
 236                         preprocessor->GetName()));
 237                 return;
 238         }
 239
 240         fPreprocessorMap.Add(new TObjString(preprocessor->GetName()), preprocessor);
 241 }
 242 //______________________________________________________________________________________________
 243 UInt_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
 244                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
 245 {
 246   // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
 247   // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
 248   // using this function. Use StoreReferenceData instead!
 249   // It calls WriteToCDB function which perform actual storage
 250
 251         return WriteToCDB(fgkMainCDB, fgkLocalCDB, path, object,
 252                                 metaData, validityStart, validityInfinite);
 253
 254 }
 255
 256 //______________________________________________________________________________________________
 257 UInt_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData)
 258 {
 259   // Stores a CDB object in the storage for reference data. This objects will not be available during
 260   // offline reconstrunction. Use this function for reference data only!
 261   // It calls WriteToCDB function which perform actual storage
 262
 263         return WriteToCDB(fgkMainRefStorage, fgkLocalRefStorage, path, object, metaData);
 264
 265 }
 266
 267 //______________________________________________________________________________________________
 268 UInt_t AliShuttle::WriteToCDB(const char* mainUri, const char* localUri,
 269                         const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
 270                         Int_t validityStart, Bool_t validityInfinite)
 271 {
 272   // write object into the CDB. Parameters are passed by Store and StoreReferenceData functions.
 273   // The parameters are:
 274   //   1) Uri of the main storage (Grid)
 275   //   2) Uri of the backup storage (Local)
 276   //   3) the object's path.
 277   //   4) the object to be stored
 278   //   5) the metaData to be associated with the object
 279   //   6) the validity start run number w.r.t. the current run,
 280   //      if the data is valid only for this run leave the default 0
 281   //   7) specifies if the calibration data is valid for infinity (this means until updated),
 282   //      typical for calibration runs, the default is kFALSE
 283   //
 284   // returns 0 if fail
 285   //         1 if stored in main (Grid) storage
 286   //         2 if stored in backup (Local) storage
 287
 288         const char* cdbType = (mainUri == fgkMainCDB) ? "CDB" : "Reference";
 289
 290         Int_t firstRun = GetCurrentRun() - validityStart;
 291         if(firstRun < 0) {
 292                 AliError("First valid run happens to be less than 0! Setting it to 0.");
 293                 firstRun=0;
 294         }
 295
 296         Int_t lastRun = -1;
 297         if(validityInfinite) {
 298                 lastRun = AliCDBRunRange::Infinity();
 299         } else {
 300                 lastRun = GetCurrentRun();
 301         }
 302
 303         AliCDBId id(path, firstRun, lastRun, -1, -1);
 304
 305         if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
 306                 TObjString runUsed = Form("%d", GetCurrentRun());
 307                 metaData->SetProperty("RunUsed(TObjString)",&runUsed);
 308         }
 309
 310         UInt_t result = 0;
 311
 312         if (!(AliCDBManager::Instance()->GetStorage(mainUri))) {
 313                 AliError(Form("WriteToCDB - Cannot activate main %s storage", cdbType));
 314         } else {
 315                 result = (UInt_t) AliCDBManager::Instance()->GetStorage(mainUri)
 316                                         ->Put(object, id, metaData);
 317         }
 318
 319         if(!result) {
 320
 321                 Log(fCurrentDetector,
 322                         Form("WriteToCDB - Problem with main %s storage. Putting <%s> into backup storage",
 323                                 cdbType, path.GetPath().Data()));
 324
 325                 // Set Grid version to current run number, to ease retrieval later
 326                 id.SetVersion(GetCurrentRun());
 327
 328                 result = AliCDBManager::Instance()->GetStorage(localUri)
 329                                         ->Put(object, id, metaData);
 330
 331                 if(result) {
 332                         result = 2;
 333                         fGridError = kTRUE;
 334                 }else{
 335                         Log(fCurrentDetector, "WriteToCDB - Can't store data!");
 336                 }
 337         }
 338
 339         return result;
 340
 341 }
 342
 343 //______________________________________________________________________________________________
 344 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
 345 {
 346 // Reads the AliShuttleStatus from the CDB
 347
 348         if (fStatusEntry){
 349                 delete fStatusEntry;
 350                 fStatusEntry = 0;
 351         }
 352
 353         fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalCDB())
 354                 ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), GetCurrentRun());
 355
 356         if (!fStatusEntry) return 0;
 357         fStatusEntry->SetOwner(1);
 358
 359         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
 360         if (!status) {
 361                 AliError("Invalid object stored to CDB!");
 362                 return 0;
 363         }
 364
 365         return status;
 366 }
 367
 368 //______________________________________________________________________________________________
 369 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
 370 {
 371 // writes the status for one subdetector
 372
 373         if (fStatusEntry){
 374                 delete fStatusEntry;
 375                 fStatusEntry = 0;
 376         }
 377
 378         Int_t run = GetCurrentRun();
 379
 380         AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), run, run);
 381
 382         fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
 383         fStatusEntry->SetOwner(1);
 384
 385         UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
 386
 387         if (!result) {
 388                 AliError(Form("WriteShuttleStatus for %s, run %d failed", fCurrentDetector.Data(), run));
 389                 return kFALSE;
 390         }
 391
 392         return kTRUE;
 393 }
 394
 395 //______________________________________________________________________________________________
 396 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
 397 {
 398   // changes the AliShuttleStatus for the given detector and run to the given status
 399
 400         if (!fStatusEntry){
 401                 AliError("UNEXPECTED: fStatusEntry empty");
 402                 return;
 403         }
 404
 405         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
 406
 407         if (!status){
 408                 AliError("UNEXPECTED: status could not be read from current CDB entry");
 409                 return;
 410         }
 411
 412         Log("SHUTTLE", Form("UpdateShuttleStatus - %s: Changing state from %s to %s", fCurrentDetector.Data(),
 413                                 status->GetStatusName(), status->GetStatusName(newStatus)));
 414
 415         status->SetStatus(newStatus);
 416         if (increaseCount) status->IncreaseCount();
 417
 418         AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
 419 }
 420 //______________________________________________________________________________________________
 421 Bool_t AliShuttle::ContinueProcessing()
 422 {
 423 // this function reads the AliShuttleStatus information from CDB and
 424 // checks if the processing should be continued
 425 // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
 426
 427         if(!GetDetCode(fCurrentDetector)) {
 428                 Log("SHUTTLE", Form("ContinueProcessing - %s: unknown detector",
 429                                 fCurrentDetector.Data()));
 430                 return kFALSE;
 431         }
 432
 433         AliShuttleLogbookEntry::Status entryStatus =
 434                 fLogbookEntry->GetDetectorStatus(GetDetCode(fCurrentDetector));
 435
 436         if(entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
 437                 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s",
 438                                 fCurrentDetector.Data(),
 439                                 fLogbookEntry->GetDetectorStatusName(entryStatus)));
 440                 return kFALSE;
 441         }
 442
 443         // if we get here, according to Shuttle logbook subdetector is in UNPROCESSED state
 444         AliShuttleStatus* status = ReadShuttleStatus();
 445         if (!status) {
 446                 // first time
 447                 Log("SHUTTLE", Form("ContinueProcessing - %s: Processing first time",
 448                                 fCurrentDetector.Data()));
 449                 status = new AliShuttleStatus(AliShuttleStatus::kStarted);
 450                 return WriteShuttleStatus(status);
 451         }
 452
 453         // The following two cases shouldn't happen if Shuttle Logbook was correctly updated.
 454         // If it happens it may mean Logbook updating failed... let's do it now!
 455         if (status->GetStatus() == AliShuttleStatus::kDone ||
 456             status->GetStatus() == AliShuttleStatus::kFailed){
 457                 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s. Updating Shuttle Logbook",
 458                                         fCurrentDetector.Data(),
 459                                         status->GetStatusName(status->GetStatus())));
 460                 UpdateShuttleLogbook(fCurrentDetector.Data(),
 461                                         status->GetStatusName(status->GetStatus()));
 462                 return kFALSE;
 463         }
 464
 465         if (status->GetStatus() == AliShuttleStatus::kStoreFailed) {
 466                 Log("SHUTTLE",
 467                         Form("ContinueProcessing - %s: Grid storage of one or more objects failed. Trying again now",
 468                                 fCurrentDetector.Data()));
 469                 if(TryToStoreAgain()){
 470                         Log(fCurrentDetector.Data(), "ContinueProcessing - All objects successfully stored into OCDB");
 471                         UpdateShuttleStatus(AliShuttleStatus::kDone);
 472                         UpdateShuttleLogbook(fCurrentDetector.Data(), "DONE");
 473                 } else {
 474                         Log("SHUTTLE",
 475                                 Form("ContinueProcessing - %s: Grid storage failed again",
 476                                         fCurrentDetector.Data()));
 477                 }
 478                 return kFALSE;
 479         }
 480
 481         // if we get here, there is a restart
 482
 483         // abort conditions
 484         // TODO we should add two counters, one for PP and one for DCS!
 485         if ((status->GetStatus() == AliShuttleStatus::kPPStarted ||
 486              status->GetStatus() == AliShuttleStatus::kPPError) &&
 487             (status->GetCount() >= fConfig->GetMaxPPRetries() ||
 488              status->GetCount() >= fConfig->GetMaxRetries())) {
 489                 Log("SHUTTLE",
 490                         Form("ContinueProcessing - %s failed %d times in status %s - Updating Shuttle Logbook",
 491                                 fCurrentDetector.Data(),
 492                                 status->GetCount(), status->GetStatusName()));
 493                 UpdateShuttleLogbook(fCurrentDetector.Data(), "FAILED");
 494                 return kFALSE;
 495         }
 496
 497         Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. Got stuck before in %s. Retry number %d.",
 498                         fCurrentDetector.Data(),
 499                         status->GetStatusName(), status->GetCount()));
 500
 501         UpdateShuttleStatus(AliShuttleStatus::kStarted);
 502
 503         return kTRUE;
 504 }
 505
 506 //______________________________________________________________________________________________
 507 Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
 508 {
 509         //
 510         // Makes data retrieval for all detectors in the configuration.
 511         // entry: Shuttle logbook entry, contains run paramenters and status of detectors
 512         // (Unprocessed, Inactive, Failed or Done).
 513         // Returns kFALSE in case of error occured and kTRUE otherwise
 514         //
 515
 516         if(!entry) return kFALSE;
 517
 518         fLogbookEntry = entry;
 519
 520         if(fLogbookEntry->IsDone()){
 521                 Log("SHUTTLE","Process - Shuttle is already DONE. Updating logbook");
 522                 UpdateShuttleLogbook("shuttle_done");
 523                 fLogbookEntry = 0;
 524                 return kTRUE;
 525         }
 526
 527
 528         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^* \n",
 529                                         GetCurrentRun()));
 530
 531         fLogbookEntry->Print("");
 532
 533         // Initialization
 534         Bool_t hasError = kFALSE;
 535         for(Int_t iSys=0;iSys<3;iSys++) fFESCalled[iSys]=kFALSE;
 536
 537         AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
 538         if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
 539         AliCDBStorage *mainRefSto = AliCDBManager::Instance()->GetStorage(fgkMainRefStorage);
 540         if(mainRefSto) mainRefSto->QueryCDB(GetCurrentRun());
 541
 542         // Loop on detectors in the configuration
 543         TIter iter(fConfig->GetDetectors());
 544         TObjString* aDetector = 0;
 545
 546         while ((aDetector = (TObjString*) iter.Next())) {
 547                 fCurrentDetector = aDetector->String();
 548
 549                 if (!fConfig->HostProcessDetector(fCurrentDetector)) continue;
 550
 551                 AliPreprocessor* aPreprocessor =
 552                         dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
 553                 if(!aPreprocessor){
 554                         Log("SHUTTLE",Form("Process - %s: no preprocessor registered. Skipping"));
 555                         continue;
 556                 }
 557
 558                 if (ContinueProcessing() == kFALSE) continue;
 559
 560                 AliInfo(Form("\n\n \t\t\t****** run %d - %s: START  ******",
 561                                                 GetCurrentRun(), aDetector->GetName()));
 562
 563                 UInt_t result = ProcessCurrentDetector();
 564
 565                 if(!result) {
 566                         hasError = kTRUE;
 567                         AliInfo(Form("\n \t\t\t****** run %d - %s: PREPROCESSOR ERROR ****** \n\n",
 568                                                         GetCurrentRun(), aDetector->GetName()));
 569                         continue;
 570                 }
 571
 572                 if(result == 2) {
 573                         AliInfo(Form("\n \t\t\t****** run %d - %s: STORAGE ERROR ****** \n\n",
 574                                                         GetCurrentRun(), aDetector->GetName()));
 575                 } else {
 576                         AliInfo(Form("\n \t\t\t****** run %d - %s: DONE ****** \n\n",
 577                                                         GetCurrentRun(), aDetector->GetName()));
 578                 }
 579
 580                 // Process successful: Update time_processed field in FES logbooks!
 581                 if(fFESCalled[kDAQ]) {
 582                         hasError = (UpdateDAQTable() == kFALSE);
 583                         fFESlist[kDAQ].Clear();
 584                 }
 585                 //if(fFESCalled[kDCS]) {
 586                 //      hasError = UpdateDCSTable(aDetector->GetName());
 587                 //      fFESlist[kDCS].Clear();
 588                 //}
 589                 //if(fFESCalled[kHLT]) {
 590                 //      hasError = UpdateHLTTable(aDetector->GetName());
 591                 //      fFESlist[kHLT].Clear();
 592                 //}
 593
 594         }
 595
 596         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^* \n",
 597                                                         GetCurrentRun()));
 598
 599         //check if shuttle is done for this run, if so update logbook
 600         TObjArray checkEntryArray;
 601         checkEntryArray.SetOwner(1);
 602         TString whereClause = Form("where run=%d",GetCurrentRun());
 603         if(QueryShuttleLogbook(whereClause.Data(), checkEntryArray)) {
 604
 605                 AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
 606                                                         (checkEntryArray.At(0));
 607
 608                 if(checkEntry && checkEntry->IsDone()){
 609                         Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
 610                         UpdateShuttleLogbook("shuttle_done");
 611                 }
 612         }
 613
 614         fLogbookEntry = 0;
 615
 616         return hasError == kFALSE;
 617 }
 618
 619 //______________________________________________________________________________________________
 620 UInt_t AliShuttle::ProcessCurrentDetector()
 621 {
 622         //
 623         // Makes data retrieval just for a specific detector (fCurrentDetector).
 624         // Threre should be a configuration for this detector.
 625
 626         AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), GetCurrentRun()));
 627
 628         UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
 629
 630         TString host(fConfig->GetDCSHost(fCurrentDetector));
 631         Int_t port = fConfig->GetDCSPort(fCurrentDetector);
 632
 633         TIter iter(fConfig->GetDCSAliases(fCurrentDetector));
 634         TObjString* anAlias;
 635         TMap aliasMap;
 636         aliasMap.SetOwner(1);
 637
 638         Bool_t aDCSError = kFALSE;
 639         fGridError = kFALSE;
 640
 641         while ((anAlias = (TObjString*) iter.Next())) {
 642                 TObjArray *valueSet = new TObjArray();
 643                 valueSet->SetOwner(1);
 644                 // TODO Test only... I've added a flag that allows to
 645                 // exclude DCS archive DB query
 646                 if(fgkProcessDCS){
 647                         AliInfo("Querying DCS archive DB data...");
 648                         aDCSError = (GetValueSet(host, port, anAlias->String(), valueSet) == 0);
 649                 } else {
 650                         AliInfo(Form("Skipping DCS processing. Port = %d",port));
 651                         aDCSError = kFALSE;
 652                 }
 653                 if(!aDCSError) {
 654                         aliasMap.Add(anAlias->Clone(), valueSet);
 655                 }else{
 656                         Log(fCurrentDetector, Form("ProcessCurrentDetector - Error while retrieving alias %s",
 657                                         anAlias->GetName()));
 658                         UpdateShuttleStatus(AliShuttleStatus::kDCSError, kTRUE);
 659                         aliasMap.DeleteAll();
 660                         return 0;
 661                 }
 662         }
 663
 664         // DCS Archive DB processing successful. Call Preprocessor!
 665         UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
 666
 667         AliPreprocessor* aPreprocessor =
 668                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
 669
 670         aPreprocessor->Initialize(GetCurrentRun(), GetCurrentStartTime(), GetCurrentEndTime());
 671
 672         UInt_t aPPResult = aPreprocessor->Process(&aliasMap);
 673
 674         UInt_t returnValue = 0;
 675         if (aPPResult == 0) { // Preprocessor error
 676                 UpdateShuttleStatus(AliShuttleStatus::kPPError, kTRUE);
 677                 returnValue = 0;
 678         } else if (fGridError == kFALSE) { // process and Grid storage ok!
 679                 UpdateShuttleStatus(AliShuttleStatus::kDone);
 680                 UpdateShuttleLogbook(fCurrentDetector, "DONE");
 681                 Log(fCurrentDetector.Data(),
 682                         "ProcessCurrentDetector - Preprocessor and Grid storage ended successfully");
 683                 returnValue = 1;
 684         } else { // Grid storage error (process ok, but object put in local storage)
 685                 UpdateShuttleStatus(AliShuttleStatus::kStoreFailed);
 686                 returnValue = 2;
 687         }
 688
 689         aliasMap.DeleteAll();
 690
 691         return returnValue;
 692 }
 693
 694 //______________________________________________________________________________________________
 695 Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
 696                 TObjArray& entries)
 697 {
 698 // Query DAQ's Shuttle logbook and fills detector status object.
 699 // Call QueryRunParameters to query DAQ logbook for run parameters.
 700
 701         // check connection, in case connect
 702         if(!Connect(kDAQ)) return kFALSE;
 703
 704         TString sqlQuery;
 705         sqlQuery = Form("select * from logbook_shuttle %s order by run", whereClause);
 706
 707         TSQLResult* aResult = fServer[kDAQ]->Query(sqlQuery);
 708         if (!aResult) {
 709                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
 710                 return kFALSE;
 711         }
 712
 713         if(aResult->GetRowCount() == 0) {
 714                 if(sqlQuery.Contains("where shuttle_done=0")){
 715                         Log("SHUTTLE", "QueryShuttleLogbook - All runs in Shuttle Logbook are already DONE");
 716                         delete aResult;
 717                         return kTRUE;
 718                 } else {
 719                         AliError("No entries in Shuttle Logbook match request");
 720                         delete aResult;
 721                         return kFALSE;
 722                 }
 723         }
 724
 725         // TODO Check field count!
 726         const UInt_t nCols = 24;
 727         if (aResult->GetFieldCount() != (Int_t) nCols) {
 728                 AliError("Invalid SQL result field number!");
 729                 delete aResult;
 730                 return kFALSE;
 731         }
 732
 733         entries.SetOwner(1);
 734
 735         TSQLRow* aRow;
 736         while ((aRow = aResult->Next())) {
 737                 TString runString(aRow->GetField(0), aRow->GetFieldLength(0));
 738                 Int_t run = runString.Atoi();
 739
 740                 UInt_t startTime, endTime;
 741                 if(!QueryRunParameters(run, startTime, endTime)) continue;
 742
 743                 const UInt_t nDet = AliShuttle::kNDetectors;
 744                 AliShuttleLogbookEntry::Status detStatus[nDet];
 745
 746                 // loop on detectors
 747                 for(UInt_t ii = 0; ii < nCols; ii++){
 748                         TString detCode(aResult->GetFieldName(ii));
 749                         Int_t detPos = AliShuttle::GetDetPos(detCode.Data());
 750                         if(detPos < 0) continue;
 751                         TString statusString(aRow->GetField(ii), aRow->GetFieldLength(ii));
 752                         if(statusString == "UNPROCESSED"){
 753                                 detStatus[detPos] = AliShuttleLogbookEntry::kUnprocessed;
 754                         } else if (statusString == "INACTIVE") {
 755                                 detStatus[detPos] = AliShuttleLogbookEntry::kInactive;
 756                         } else if (statusString == "FAILED") {
 757                                 detStatus[detPos] = AliShuttleLogbookEntry::kFailed;
 758                         } else if (statusString == "DONE") {
 759                                 detStatus[detPos] = AliShuttleLogbookEntry::kDone;
 760                         }
 761                 }
 762
 763                 entries.AddLast(new AliShuttleLogbookEntry(run, startTime, endTime, detStatus));
 764                 delete aRow;
 765         }
 766
 767         if(sqlQuery.Contains("where shuttle_done=0"))
 768                 Log("SHUTTLE", Form("QueryShuttleLogbook - Found %d unprocessed runs in Shuttle Logbook",
 769                                                         entries.GetEntriesFast()));
 770         delete aResult;
 771         return kTRUE;
 772 }
 773
 774 //______________________________________________________________________________________________
 775 Bool_t AliShuttle::QueryRunParameters(Int_t& run, UInt_t& startTime, UInt_t& endTime)
 776 {
 777 // Retrieve start time and end time for run in the DAQ logbook
 778
 779         // check connection, in case connect
 780         if(!Connect(kDAQ)) return kFALSE;
 781
 782         TString sqlQuery;
 783         sqlQuery = Form("select time_start, time_end from logbook where run=%d", run);
 784
 785         TSQLResult* aResult = fServer[kDAQ]->Query(sqlQuery);
 786         if (!aResult) {
 787                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
 788                 return kFALSE;
 789         }
 790
 791         if(aResult->GetRowCount() == 0) {
 792                 Log("SHUTTLE", Form("QueryRunParameters - No entry in DAQ Logbook for run %d. Skipping", run));
 793                 delete aResult;
 794                 return kFALSE;
 795         }
 796
 797         if(aResult->GetRowCount() > 1) {
 798                 AliError(Form("More than one entry in DAQ Logbook for run %d. Skipping", run));
 799                 delete aResult;
 800                 return kFALSE;
 801         }
 802
 803         TSQLRow* aRow;
 804         while ((aRow = aResult->Next())) {
 805
 806                 TString startTimeString(aRow->GetField(0),
 807                                 aRow->GetFieldLength(0));
 808                 startTime = startTimeString.Atoi();
 809                 TString endTimeString(aRow->GetField(1),
 810                                 aRow->GetFieldLength(1));
 811                 endTime = endTimeString.Atoi();
 812
 813                 if (!startTime || !endTime || startTime > endTime) {
 814                         Log("SHUTTLE",
 815                                 Form("QueryRunParameters - Invalid parameters for Run %d: startTime = %d, endTime = %d",
 816                                         run, startTime, endTime));
 817                         delete aRow;
 818                         delete aResult;
 819                         return kFALSE;
 820                 }
 821
 822                 delete aRow;
 823         }
 824
 825         delete aResult;
 826         return kTRUE;
 827 }
 828
 829 //______________________________________________________________________________________________
 830 Bool_t AliShuttle::TryToStoreAgain()
 831 {
 832   // Called in case the detector failed to store the object in Grid OCDB
 833   // It tries to store the object again, if it does not find more recent and overlapping objects
 834   // Calls underlying TryToStoreAgain(const char*) function twice, for OCDB and Reference storage.
 835
 836         AliInfo("Trying to store OCDB data again...");
 837         Bool_t resultCDB = TryToStoreAgain(fgkMainCDB);
 838
 839         AliInfo("Trying to store reference data again...");
 840         Bool_t resultRef = TryToStoreAgain(fgkMainRefStorage);
 841
 842         return resultCDB && resultRef;
 843 }
 844
 845 //______________________________________________________________________________________________
 846 Bool_t AliShuttle::TryToStoreAgain(TString& gridURI)
 847 {
 848   // Called by TryToStoreAgain(), performs actual storage retry
 849
 850         TList* gridIds=0;
 851
 852         Bool_t result = kTRUE;
 853
 854         const char* type = 0;
 855         TString backupURI;
 856         if(gridURI == fgkMainCDB) {
 857                 type = "OCDB";
 858                 backupURI = fgkLocalCDB;
 859         } else if(gridURI == fgkMainRefStorage) {
 860                 type = "reference";
 861                 backupURI = fgkLocalRefStorage;
 862         } else {
 863                 AliError(Form("Invalid storage URI: %s", gridURI.Data()));
 864                 return kFALSE;
 865         }
 866
 867         AliCDBManager* man = AliCDBManager::Instance();
 868
 869         AliCDBStorage *gridSto = man->GetStorage(gridURI);
 870         if(!gridSto) {
 871                 Log(fCurrentDetector.Data(),
 872                         Form("TryToStoreAgain - cannot activate main %s storage", type));
 873                 return kFALSE;
 874         }
 875
 876         gridIds = gridSto->GetQueryCDBList();
 877
 878         // get objects previously stored in local CDB
 879         AliCDBStorage *backupSto = man->GetStorage(backupURI);
 880         AliCDBPath aPath(fCurrentDetector,"*","*");
 881         // Local objects were stored with current run as Grid version!
 882         TList* localEntries = backupSto->GetAll(aPath.GetPath(), GetCurrentRun(), GetCurrentRun());
 883         localEntries->SetOwner(1);
 884
 885         // loop on local stored objects
 886         TIter localIter(localEntries);
 887         AliCDBEntry *aLocEntry = 0;
 888         while((aLocEntry = dynamic_cast<AliCDBEntry*> (localIter.Next()))){
 889                 aLocEntry->SetOwner(1);
 890                 AliCDBId aLocId = aLocEntry->GetId();
 891                 aLocEntry->SetVersion(-1);
 892                 aLocEntry->SetSubVersion(-1);
 893
 894                 // loop on Grid valid Id's
 895                 Bool_t store = kTRUE;
 896                 TIter gridIter(gridIds);
 897                 AliCDBId* aGridId = 0;
 898                 while((aGridId = dynamic_cast<AliCDBId*> (gridIter.Next()))){
 899                         // If local object is valid up to infinity we store it anyway
 900                         // TODO This does not work! It may hide more recent objects...
 901                         if(aLocId.GetLastRun() == AliCDBRunRange::Infinity()) {
 902                                 // TODO Check that it won't hide more recent files! how????
 903                                 break;
 904                         }
 905                         if(aGridId->GetPath() != aLocId.GetPath()) continue;
 906                         // skip all objects valid up to infinity
 907                         if(aGridId->GetLastRun() == AliCDBRunRange::Infinity()) continue;
 908                         // if we get here, it means there's already some more recent object stored on Grid!
 909                         store = kFALSE;
 910                         break;
 911                 }
 912
 913                 if(!store){
 914                         Log(fCurrentDetector.Data(),
 915                                 Form("TryToStoreAgain - A more recent object already exists in %s storage: <%s>",
 916                                         type, aGridId->ToString().Data()));
 917                         // removing local filename...
 918                         // TODO maybe it's better not to remove it, it was not copied to the Grid!
 919                         TString filename;
 920                         backupSto->IdToFilename(aLocId, filename);
 921                         AliInfo(Form("Removing local file %s", filename.Data()));
 922                         gSystem->Exec(Form("rm %s",filename.Data()));
 923                         continue;
 924                 }
 925
 926                 // If we get here, the file can be stored!
 927                 Bool_t storeOk = gridSto->Put(aLocEntry);
 928                 if(storeOk){
 929                         Log(fCurrentDetector.Data(),
 930                                 Form("TryToStoreAgain - Object <%s> successfully put into %s storage",
 931                                         aLocId.ToString().Data(), type));
 932
 933                         // removing local filename...
 934                         TString filename;
 935                         backupSto->IdToFilename(aLocId, filename);
 936                         AliInfo(Form("Removing local file %s", filename.Data()));
 937                         gSystem->Exec(Form("rm %s", filename.Data()));
 938                         continue;
 939                 } else  {
 940                         Log(fCurrentDetector.Data(),
 941                                 Form("TryToStoreAgain - Grid %s storage of object <%s> failed again",
 942                                         type, aLocId.ToString().Data()));
 943                         result = kFALSE;
 944                 }
 945         }
 946         localEntries->Clear();
 947
 948         return result;
 949 }
 950
 951 //______________________________________________________________________________________________
 952 Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* alias,
 953                                 TObjArray* valueSet)
 954 {
 955 // Retrieve all "alias" data points from the DCS server
 956 // host, port: TSocket connection parameters
 957 // alias: name of the alias
 958 // valueSet: array of retrieved AliDCSValue's
 959
 960         AliDCSClient client(host, port, fTimeout, fRetries);
 961         if (!client.IsConnected()) {
 962                 return kFALSE;
 963         }
 964
 965         Int_t result = client.GetAliasValues(alias,
 966                 GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
 967
 968         if (result < 0) {
 969                 Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get '%s'! Reason: %s",
 970                         alias, AliDCSClient::GetErrorString(result)));
 971
 972                 if (result == AliDCSClient::fgkServerError) {
 973                         Log(fCurrentDetector.Data(), Form("GetValueSet - Server error: %s",
 974                                 client.GetServerError().Data()));
 975                 }
 976
 977                 return kFALSE;
 978         }
 979
 980         return kTRUE;
 981 }
 982
 983 //______________________________________________________________________________________________
 984 const char* AliShuttle::GetFile(Int_t system, const char* detector,
 985                 const char* id, const char* source)
 986 {
 987 // Get calibration file from file exchange servers
 988 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
 989
 990         switch(system){
 991                 case kDAQ:
 992                         return GetDAQFileName(detector, id, source);
 993                         break;
 994                 case kDCS:
 995                         return GetDCSFileName(detector, id, source);
 996                         break;
 997                 case kHLT:
 998                         return GetHLTFileName(detector, id, source);
 999                         break;
1000                 default:
1001                         AliError(Form("No valid system index: %d",system));
1002         }
1003
1004         return 0;
1005 }
1006
1007 //______________________________________________________________________________________________
1008 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
1009 {
1010 // Get sources producing the condition file Id from file exchange servers
1011 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
1012
1013         switch(system){
1014                 case kDAQ:
1015                         return GetDAQFileSources(detector, id);
1016                         break;
1017                 case kDCS:
1018                         return GetDCSFileSources(detector, id);
1019                         break;
1020                 case kHLT:
1021                         return GetHLTFileSources(detector, id);
1022                         break;
1023                 default:
1024                         AliError(Form("No valid system index: %d",system));
1025         }
1026
1027         return NULL;
1028 }
1029
1030 //______________________________________________________________________________________________
1031 Bool_t AliShuttle::Connect(Int_t system)
1032 {
1033 // Connect to MySQL Server of the system's FES logbook
1034 // DAQ Logbook, Shuttle Logbook and DAQ FES Logbook are on the same host
1035
1036         // check connection: if already connected return
1037         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
1038
1039         TString aFESlbHost= Form("mysql://%s", fConfig->GetFESlbHost(system));
1040
1041         fServer[system] = TSQLServer::Connect(aFESlbHost,
1042                         fConfig->GetFESlbUser(system),
1043                         fConfig->GetFESlbPass(system));
1044         if (!fServer[system] || !fServer[system]->IsConnected()) {
1045                 AliError(Form("Can't establish connection to FES logbook for %s",fkSystemNames[system]));
1046                 if(fServer[system]) delete fServer[system];
1047                 return kFALSE;
1048         }
1049
1050         // Get tables
1051         // TODO in the configuration should the table name be there too?
1052         TSQLResult* aResult=0;
1053         switch(system){
1054                 case kDAQ:
1055                         aResult = fServer[kDAQ]->GetTables("REFSYSLOG");
1056                         break;
1057                 case kDCS:
1058                         //aResult = fServer[kDCS]->GetTables("REFSYSLOG");
1059                         break;
1060                 case kHLT:
1061                         //aResult = fServer[kHLT]->GetTables("REFSYSLOG");
1062                         break;
1063                 default:
1064                         break;
1065         }
1066
1067         delete aResult;
1068         return kTRUE;
1069 }
1070
1071 //______________________________________________________________________________________________
1072 const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source)
1073 {
1074 // Retrieves a file from the DAQ FES.
1075 // First queris the DAQ logbook_fs for the DAQ file name, using the run, detector, id and source info
1076 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
1077 // run: current run being processed (given by Logbook entry fLogbookEntry)
1078 // detector: comes from the Preprocessor name (must be converted into detector code with GetDetCode)
1079 // id: provided as a parameter by the Preprocessor
1080 // source: provided by the Preprocessor through GetFileSources function
1081
1082         // check connection, in case connect
1083         if(!Connect(kDAQ)){
1084                 Log(detector, "GetDAQFileName - Couldn't connect to DAQ Logbook");
1085                 return 0;
1086         }
1087
1088         // Query preparation
1089         TString sqlQueryStart = "select filePath from logbook_fs where";
1090         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
1091                                 GetCurrentRun(), GetDetCode(detector), id, source);
1092         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1093
1094         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1095
1096         // Query execution
1097         TSQLResult* aResult = 0;
1098         aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1099         if (!aResult) {
1100                 Log(detector, Form("GetDAQFileName - Can't execute SQL query for: id = %s, source = %s",
1101                                 id, source));
1102                 return 0;
1103         }
1104
1105         if (aResult->GetRowCount() == 0) {
1106                 Log(detector,
1107                         Form("GetDAQFileName - No entry in FES table for: id = %s, source = %s",
1108                                 id, source));
1109                 delete aResult;
1110                 return 0;
1111         }
1112
1113         if (aResult->GetRowCount() >1) {
1114                 Log(detector,
1115                         Form("GetDAQFileName - More than one entry in FES table for: id = %s, source = %s",
1116                                 id, source));
1117                 delete aResult;
1118                 return 0;
1119         }
1120
1121         TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
1122
1123         if(!aRow){
1124                 Log(detector, Form("GetDAQFileName - Empty set result from query: id = %s, source = %s",
1125                                 id, source));
1126                 delete aResult;
1127                 return 0;
1128         }
1129
1130         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
1131
1132         delete aResult;
1133         delete aRow;
1134
1135         AliDebug(2, Form("filePath = %s",filePath.Data()));
1136
1137         // retrieved file is renamed to make it unique
1138         TString localFileName = Form("%s_%d_%s_%s.shuttle",
1139                                         detector, GetCurrentRun(), id, source);
1140
1141         // file retrieval from DAQ FES
1142         Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
1143         if(!result) {
1144                 Log(detector, Form("GetDAQFileName - Copy of file %s from DAQ FES failed", filePath.Data()));
1145                 return 0;
1146         } else {
1147                 AliInfo(Form("File %s copied from DAQ FES into %s/%s",
1148                         filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
1149         }
1150
1151
1152         fFESCalled[kDAQ]=kTRUE;
1153         TObjString *fileParams = new TObjString(Form("%s_!?!_%s", id, source));
1154         fFESlist[kDAQ].Add(fileParams);
1155
1156         return localFileName.Data();
1157
1158 }
1159
1160 //______________________________________________________________________________________________
1161 Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName)
1162 {
1163
1164         // check temp directory: trying to cd to temp; if it does not exist, create it
1165         AliDebug(2, Form("Copy file %s from DAQ FES into folder %s and rename it as %s",
1166                         daqFileName,fgkShuttleTempDir, localFileName));
1167
1168         void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
1169         if (dir == NULL) {
1170                 if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
1171                         AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
1172                         return kFALSE;
1173                 }
1174
1175         } else {
1176                 gSystem->FreeDirectory(dir);
1177         }
1178
1179         TString baseDAQFESFolder = "DAQ";
1180         TString command = Form("scp %s@%s:%s/%s %s/%s",
1181                 fConfig->GetFESUser(kDAQ),
1182                 fConfig->GetFESHost(kDAQ),
1183                 baseDAQFESFolder.Data(),
1184                 daqFileName,
1185                 fgkShuttleTempDir,
1186                 localFileName);
1187
1188         AliDebug(2, Form("%s",command.Data()));
1189
1190         UInt_t nRetries = 0;
1191         UInt_t maxRetries = 3;
1192
1193         // copy!! if successful TSystem::Exec returns 0
1194         while(nRetries++ < maxRetries) {
1195                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
1196                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
1197         }
1198
1199         return kFALSE;
1200
1201 }
1202
1203 //______________________________________________________________________________________________
1204 TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id)
1205 {
1206 // Retrieves a file from the DCS FES.
1207
1208         // check connection, in case connect
1209         if(!Connect(kDAQ)){
1210                 Log(detector, "GetDAQFileSources - Couldn't connect to DAQ Logbook");
1211                 return 0;
1212         }
1213
1214         // Query preparation
1215         TString sqlQueryStart = "select DAQsource from logbook_fs where";
1216         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
1217                                 GetCurrentRun(), GetDetCode(detector), id);
1218         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1219
1220         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1221
1222         // Query execution
1223         TSQLResult* aResult;
1224         aResult = fServer[kDAQ]->Query(sqlQuery);
1225         if (!aResult) {
1226                 Log(detector, Form("GetDAQFileSources - Can't execute SQL query for id: %s", id));
1227                 return 0;
1228         }
1229
1230         if (aResult->GetRowCount() == 0) {
1231                 Log(detector,
1232                         Form("GetDAQFileSources - No entry in FES table for id: %s", id));
1233                 delete aResult;
1234                 return 0;
1235         }
1236
1237         TSQLRow* aRow;
1238         TList *list = new TList();
1239         list->SetOwner(1);
1240
1241         while((aRow = aResult->Next())){
1242
1243                 TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
1244                 AliDebug(2, Form("daqSource = %s", daqSource.Data()));
1245                 list->Add(new TObjString(daqSource));
1246                 delete aRow;
1247         }
1248         delete aResult;
1249
1250         return list;
1251
1252 }
1253
1254 //______________________________________________________________________________________________
1255 const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1256 // Retrieves a file from the DCS FES.
1257
1258 return "You're in DCS";
1259
1260 }
1261
1262 //______________________________________________________________________________________________
1263 TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
1264 // Retrieves a file from the DCS FES.
1265
1266 return NULL;
1267
1268 }
1269
1270 //______________________________________________________________________________________________
1271 const char* AliShuttle::GetHLTFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1272 // Retrieves a file from the HLT FES.
1273
1274 return "You're in HLT";
1275
1276 }
1277
1278 //______________________________________________________________________________________________
1279 TList* AliShuttle::GetHLTFileSources(const char* /*detector*/, const char* /*id*/){
1280 // Retrieves a file from the HLT FES.
1281
1282 return NULL;
1283
1284 }
1285
1286 //______________________________________________________________________________________________
1287 Bool_t AliShuttle::UpdateDAQTable()
1288 {
1289 // Update DAQ table filling time_processed field in all rows corresponding to current run and detector
1290
1291         // check connection, in case connect
1292         if(!Connect(kDAQ)){
1293                 Log(fCurrentDetector, "UpdateDAQTable - Couldn't connect to DAQ Logbook");
1294                 return kFALSE;
1295         }
1296
1297         TTimeStamp now; // now
1298
1299         // Loop on FES list entries
1300         TIter iter(&fFESlist[kDAQ]);
1301         TObjString *aFESentry=0;
1302         while((aFESentry = dynamic_cast<TObjString*> (iter.Next()))){
1303                 TString aFESentrystr = aFESentry->String();
1304                 TObjArray *aFESarray = aFESentrystr.Tokenize("_!?!_");
1305                 if(!aFESarray || aFESarray->GetEntries() != 2 ) {
1306                         Log(fCurrentDetector, Form("UpdateDAQTable - error updating FES entry. Check string: <%s>",
1307                                 aFESentrystr.Data()));
1308                         if(aFESarray) delete aFESarray;
1309                         return kFALSE;
1310                 }
1311                 const char* fileId = ((TObjString*) aFESarray->At(0))->GetName();
1312                 const char* daqSource = ((TObjString*) aFESarray->At(1))->GetName();
1313                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
1314                         GetCurrentRun(), GetDetCode(fCurrentDetector), fileId, daqSource);
1315
1316                 delete aFESarray;
1317
1318                 TString sqlQuery = Form("update logbook_fs set time_processed=%d %s", now.GetSec(), whereClause.Data());
1319
1320                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1321
1322                 // Query execution
1323                 TSQLResult* aResult;
1324                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1325                 if (!aResult) {
1326                         Log(fCurrentDetector, Form("UpdateDAQTable - Can't execute SQL query <%s>", sqlQuery.Data()));
1327                         return kFALSE;
1328                 }
1329                 delete aResult;
1330         }
1331
1332         return kTRUE;
1333 }
1334
1335
1336 //______________________________________________________________________________________________
1337 Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status)
1338 {
1339 // Update Shuttle logbook filling detector or shuttle_done column
1340 // ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
1341
1342         // check connection, in case connect
1343         if(!Connect(kDAQ)){
1344                 Log("SHUTTLE", "UpdateShuttleLogbook - Couldn't connect to DAQ Logbook.");
1345                 return kFALSE;
1346         }
1347
1348         TString detName(detector);
1349         TString setClause;
1350         if(detName == "shuttle_done") {
1351                 setClause = "set shuttle_done=1";
1352         } else {
1353                 TString detCode = GetDetCode(detector);
1354                 if(detCode.IsNull()) {
1355                         Log("SHUTTLE", Form("UpdateShuttleLogbook - Unknown detector %s", detector));
1356                         return kFALSE;
1357                 }
1358                 TString statusStr(status);
1359                 if(statusStr.Contains("done", TString::kIgnoreCase) ||
1360                    statusStr.Contains("failed", TString::kIgnoreCase)){
1361                         setClause = Form("set %s=\"%s\"", detCode.Data(), status);
1362                 } else {
1363                         Log("SHUTTLE",
1364                                 Form("UpdateShuttleLogbook - Invalid status <%s> for detector %s",
1365                                         status, detector));
1366                         return kFALSE;
1367                 }
1368         }
1369
1370         TString whereClause = Form("where run=%d", GetCurrentRun());
1371
1372         TString sqlQuery = Form("update logbook_shuttle %s %s",
1373                                         setClause.Data(), whereClause.Data());
1374
1375         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1376
1377         // Query execution
1378         TSQLResult* aResult;
1379         aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1380         if (!aResult) {
1381                 Log("SHUTTLE", Form("UpdateShuttleLogbook - Can't execute query <%s>", sqlQuery.Data()));
1382                 return kFALSE;
1383         }
1384         delete aResult;
1385
1386         return kTRUE;
1387 }
1388
1389 //______________________________________________________________________________________________
1390 Int_t AliShuttle::GetCurrentRun() const
1391 {
1392 // Get current run from logbook entry
1393
1394         return fLogbookEntry ? fLogbookEntry->GetRun() : -1;
1395 }
1396
1397 //______________________________________________________________________________________________
1398 UInt_t AliShuttle::GetCurrentStartTime() const
1399 {
1400 // get current start time
1401
1402         return fLogbookEntry ? fLogbookEntry->GetStartTime() : 0;
1403 }
1404
1405 //______________________________________________________________________________________________
1406 UInt_t AliShuttle::GetCurrentEndTime() const
1407 {
1408 // get current end time from logbook entry
1409
1410         return fLogbookEntry ? fLogbookEntry->GetEndTime() : 0;
1411 }
1412
1413 //______________________________________________________________________________________________
1414 const char* AliShuttle::GetDetCode(const char* detector){
1415 // Return detector code
1416
1417         for(UInt_t iDet=0; iDet < kNDetectors; iDet++){
1418                 if(!strcmp(fgkDetectorName[iDet], detector)) return fgkDetectorCode[iDet];
1419         }
1420
1421         AliErrorClass(Form("Unknown detector: %s",detector));
1422         return 0;
1423 }
1424
1425 //______________________________________________________________________________________________
1426 const char* AliShuttle::GetDetCode(UInt_t detPos){
1427 // Return detector code
1428
1429         if( detPos >= kNDetectors) {
1430                 AliErrorClass(Form("Invalid parameter: %d", detPos));
1431                 return 0;
1432         }
1433         return fgkDetectorCode[detPos];
1434 }
1435
1436 //______________________________________________________________________________________________
1437 const Int_t AliShuttle::GetDetPos(const char* detCode){
1438 // Return detector position in the detector code array
1439
1440         for(UInt_t iDet=0; iDet < kNDetectors; iDet++){
1441                 if(!strcmp(fgkDetectorCode[iDet], detCode)) return iDet;
1442         }
1443         return -1;
1444 }
1445
1446 //______________________________________________________________________________________________
1447 void AliShuttle::Log(const char* detector, const char* message)
1448 {
1449 // Fill log string with a message
1450
1451         void* dir = gSystem->OpenDirectory(fgkShuttleLogDir);
1452         if (dir == NULL) {
1453                 if (gSystem->mkdir(fgkShuttleLogDir, kTRUE)) {
1454                         AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
1455                         return;
1456                 }
1457
1458         } else {
1459                 gSystem->FreeDirectory(dir);
1460         }
1461
1462         TString toLog = Form("%s: %s - ", TTimeStamp(time(0)).AsString("s"), detector);
1463         if(GetCurrentRun()>=0 ) toLog += Form("run %d - ", GetCurrentRun());
1464         toLog += Form("%s", message);
1465
1466         AliInfo(toLog.Data());
1467
1468         TString fileName;
1469         fileName.Form("%s/%s.log", fgkShuttleLogDir, detector);
1470         gSystem->ExpandPathName(fileName);
1471
1472         ofstream logFile;
1473         logFile.open(fileName, ofstream::out | ofstream::app);
1474
1475         if (!logFile.is_open()) {
1476                 AliError(Form("Could not open file %s", fileName.Data()));
1477                 return;
1478         }
1479
1480         logFile << toLog.Data() << "\n";
1481
1482         logFile.close();
1483 }
1484
1485
1486 //______________________________________________________________________________________________
1487 Bool_t AliShuttle::Collect(Int_t run)
1488 {
1489         //
1490         // Collects conditions data for the given run.
1491         //
1492
1493         AliInfo(Form("Collecting conditions data for run %d", run));
1494
1495         TString whereClause("where run=");
1496         whereClause += run;
1497
1498         TObjArray dateEntries;
1499         if (!QueryShuttleLogbook(whereClause, dateEntries)) {
1500                 AliError("Can't retrieve entries from Shuttle logbook!");
1501                 return kFALSE;
1502         }
1503
1504         if (!dateEntries.GetEntriesFast()) {
1505                 AliError(Form("Retrieval of parameters for run %d failed!", run));
1506                 return kFALSE;
1507         }
1508
1509         if (dateEntries.GetEntriesFast() > 1) {
1510                 AliError(Form("There is more than one entry for run <%d> in Shuttle logbook!", run));
1511                 return kFALSE;
1512         }
1513
1514         if (!RetrieveConditionsData(dateEntries)) {
1515                 AliError("An error occured during conditions data retrieval!");
1516                 return kFALSE;
1517         }
1518
1519         return kTRUE;
1520 }
1521
1522 //______________________________________________________________________________________________
1523 Bool_t AliShuttle::CollectNew()
1524 {
1525         //
1526         // Collects conditions data for all UNPROCESSED run written to DAQ LogBook.
1527         // In operational mode, this is the Shuttle function triggered by the EOR signal.
1528         //
1529
1530         Log("SHUTTLE","CollectNew - Shuttle called. Collecting conditions data for unprocessed runs");
1531
1532         TString whereClause("where shuttle_done=0");
1533
1534         TObjArray shuttleLogbookEntries;
1535         if (!QueryShuttleLogbook(whereClause, shuttleLogbookEntries)) {
1536                 Log("SHUTTLE", "CollectNew - Can't retrieve entries from Shuttle logbook");
1537                 return kFALSE;
1538         }
1539
1540         if (!RetrieveConditionsData(shuttleLogbookEntries)) {
1541                 Log("SHUTTLE", "CollectNew - Process of at least one run failed");
1542                 return kFALSE;
1543         }
1544
1545         return kTRUE;
1546 }
1547
1548 //______________________________________________________________________________________________
1549 Bool_t AliShuttle::CollectAll()
1550 {
1551         //
1552         // Collects conditions data for all runs (even if they're already done!) written in Shuttle LogBook.
1553         //
1554
1555         AliInfo("Collecting conditions data for all runs ...");
1556
1557         TObjArray dateEntries;
1558         if (!QueryShuttleLogbook("", dateEntries)) {
1559                 AliError("Can't retrieve entries from Shuttle logbook");
1560                 return kFALSE;
1561         }
1562
1563         if (!RetrieveConditionsData(dateEntries)) {
1564                 AliError("An error occured during conditions data retrieval!");
1565                 return kFALSE;
1566         }
1567
1568         return kTRUE;
1569 }
1570
1571
1572 //______________________________________________________________________________________________
1573 Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
1574 {
1575 // Retrieve conditions data for all runs that aren't processed yet
1576
1577         Bool_t hasError = kFALSE;
1578
1579         TIter iter(&dateEntries);
1580         AliShuttleLogbookEntry* anEntry;
1581
1582         while ((anEntry = (AliShuttleLogbookEntry*) iter.Next())){
1583                 if (!Process(anEntry)){
1584                         hasError = kTRUE;
1585                 }
1586         }
1587
1588         return hasError == kFALSE;
1589 }