SHUTTLE/AliShuttle.cxx

   1 /**************************************************************************
   2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
   3  *                                                                        *
   4  * Author: The ALICE Off-line Project.                                    *
   5  * Contributors are mentioned in the code where appropriate.              *
   6  *                                                                        *
   7  * Permission to use, copy, modify and distribute this software and its   *
   8  * documentation strictly for non-commercial purposes is hereby granted   *
   9  * without fee, provided that the above copyright notice appears in all   *
  10  * copies and that both the copyright notice and this permission notice   *
  11  * appear in the supporting documentation. The authors make no claims     *
  12  * about the suitability of this software for any purpose. It is          *
  13  * provided "as is" without express or implied warranty.                  *
  14  **************************************************************************/
  15
  16 /*
  17 $Log$
  18 Revision 1.14  2006/08/29 09:16:05  jgrosseo
  19 small update
  20
  21 Revision 1.13  2006/08/15 10:50:00  jgrosseo
  22 effc++ corrections (alberto)
  23
  24 Revision 1.12  2006/08/08 14:19:29  jgrosseo
  25 Update to shuttle classes (Alberto)
  26
  27 - Possibility to set the full object's path in the Preprocessor's and
  28 Shuttle's  Store functions
  29 - Possibility to extend the object's run validity in the same classes
  30 ("startValidity" and "validityInfinite" parameters)
  31 - Implementation of the StoreReferenceData function to store reference
  32 data in a dedicated CDB storage.
  33
  34 Revision 1.11  2006/07/21 07:37:20  jgrosseo
  35 last run is stored after each run
  36
  37 Revision 1.10  2006/07/20 09:54:40  jgrosseo
  38 introducing status management: The processing per subdetector is divided into several steps,
  39 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
  40 can keep track of the number of failures and skips further processing after a certain threshold is
  41 exceeded. These thresholds can be configured in LDAP.
  42
  43 Revision 1.9  2006/07/19 10:09:55  jgrosseo
  44 new configuration, accesst to DAQ FES (Alberto)
  45
  46 Revision 1.8  2006/07/11 12:44:36  jgrosseo
  47 adding parameters for extended validity range of data produced by preprocessor
  48
  49 Revision 1.7  2006/07/10 14:37:09  jgrosseo
  50 small fix + todo comment
  51
  52 Revision 1.6  2006/07/10 13:01:41  jgrosseo
  53 enhanced storing of last sucessfully processed run (alberto)
  54
  55 Revision 1.5  2006/07/04 14:59:57  jgrosseo
  56 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
  57
  58 Revision 1.4  2006/06/12 09:11:16  jgrosseo
  59 coding conventions (Alberto)
  60
  61 Revision 1.3  2006/06/06 14:26:40  jgrosseo
  62 o) removed files that were moved to STEER
  63 o) shuttle updated to follow the new interface (Alberto)
  64
  65 Revision 1.2  2006/03/07 07:52:34  hristov
  66 New version (B.Yordanov)
  67
  68 Revision 1.6  2005/11/19 17:19:14  byordano
  69 RetrieveDATEEntries and RetrieveConditionsData added
  70
  71 Revision 1.5  2005/11/19 11:09:27  byordano
  72 AliShuttle declaration added
  73
  74 Revision 1.4  2005/11/17 17:47:34  byordano
  75 TList changed to TObjArray
  76
  77 Revision 1.3  2005/11/17 14:43:23  byordano
  78 import to local CVS
  79
  80 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
  81 Initial import as subdirectory in AliRoot
  82
  83 Revision 1.2  2005/09/13 08:41:15  byordano
  84 default startTime endTime added
  85
  86 Revision 1.4  2005/08/30 09:13:02  byordano
  87 some docs added
  88
  89 Revision 1.3  2005/08/29 21:15:47  byordano
  90 some docs added
  91
  92 */
  93
  94 //
  95 // This class is the main manager for AliShuttle.
  96 // It organizes the data retrieval from DCS and call the
  97 // interface methods of AliPreprocessor.
  98 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
  99 // data for its set of aliases is retrieved. If there is registered
 100 // AliPreprocessor for this detector then it will be used
 101 // accroding to the schema (see AliPreprocessor).
 102 // If there isn't registered AliPreprocessor than the retrieved
 103 // data is stored automatically to the undelying AliCDBStorage.
 104 // For detSpec is used the alias name.
 105 //
 106
 107 #include "AliShuttle.h"
 108
 109 #include "AliCDBManager.h"
 110 #include "AliCDBStorage.h"
 111 #include "AliCDBId.h"
 112 #include "AliCDBRunRange.h"
 113 #include "AliCDBPath.h"
 114 #include "AliCDBEntry.h"
 115 #include "AliShuttleConfig.h"
 116 #include "AliDCSClient.h"
 117 #include "AliLog.h"
 118 #include "AliPreprocessor.h"
 119 #include "AliShuttleStatus.h"
 120 #include "AliShuttleLogbookEntry.h"
 121
 122 #include <TSystem.h>
 123 #include <TObject.h>
 124 #include <TString.h>
 125 #include <TTimeStamp.h>
 126 #include <TObjString.h>
 127 #include <TSQLServer.h>
 128 #include <TSQLResult.h>
 129 #include <TSQLRow.h>
 130
 131 #include <fstream>
 132
 133 ClassImp(AliShuttle)
 134
 135 TString AliShuttle::fgkMainCDB("alien://folder=ShuttleCDB");
 136 TString AliShuttle::fgkLocalCDB("local://LocalShuttleCDB");
 137 TString AliShuttle::fgkMainRefStorage("alien://folder=ShuttleReference");
 138 TString AliShuttle::fgkLocalRefStorage("local://LocalReferenceStorage");
 139
 140 Bool_t AliShuttle::fgkProcessDCS(kTRUE);
 141
 142
 143 const char* AliShuttle::fgkShuttleTempDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/temp");
 144 const char* AliShuttle::fgkShuttleLogDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/log");
 145
 146 const char* AliShuttle::fgkDetectorName[AliShuttle::kNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
 147         "PHOS", "CPV", "RICH", "EMCAL", "MUON_TRK", "MUON_TRG", "FMD", "ZDC", "PMD", "START", "VZERO"};
 148
 149 const char* AliShuttle::fgkDetectorCode[AliShuttle::kNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
 150         "PHS", "CPV", "HMP", "EMC", "MCH", "MTR", "FMD", "ZDC", "PMD", "T00", "V00"};
 151
 152 //______________________________________________________________________________________________
 153 AliShuttle::AliShuttle(const AliShuttleConfig* config,
 154                 UInt_t timeout, Int_t retries):
 155 fConfig(config),
 156 fTimeout(timeout), fRetries(retries),
 157 fPreprocessorMap(),
 158 fLogbookEntry(0),
 159 fCurrentDetector(""),
 160 fStatusEntry(0),
 161 fGridError(kFALSE)
 162 {
 163         //
 164         // config: AliShuttleConfig used
 165         // timeout: timeout used for AliDCSClient connection
 166         // retries: the number of retries in case of connection error.
 167         //
 168
 169         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
 170         for(int iSys=0;iSys<3;iSys++) {
 171                 fServer[iSys]=0;
 172                 fFESlist[iSys].SetOwner(kTRUE);
 173         }
 174         fPreprocessorMap.SetOwner(kTRUE);
 175 }
 176
 177 //______________________________________________________________________
 178 AliShuttle::AliShuttle(const AliShuttle& /*other*/):
 179 AliShuttleInterface(),
 180 fConfig(0),
 181 fTimeout(0), fRetries(0),
 182 fPreprocessorMap(),
 183 fLogbookEntry(0),
 184 fCurrentDetector(""),
 185 fStatusEntry(0),
 186 fGridError(kFALSE)
 187 {
 188 // copy constructor (not implemented)
 189
 190 }
 191
 192 //______________________________________________________________________
 193 AliShuttle &AliShuttle::operator=(const AliShuttle& /*other*/)
 194 {
 195 // assignment operator (not implemented)
 196
 197 return *this;
 198 }
 199
 200 //______________________________________________________________________________________________
 201 AliShuttle::~AliShuttle()
 202 {
 203 // destructor
 204
 205         fPreprocessorMap.DeleteAll();
 206         for(int iSys=0;iSys<3;iSys++)
 207                 if(fServer[iSys]) {
 208                         fServer[iSys]->Close();
 209                         delete fServer[iSys];
 210                 }
 211
 212         if (fStatusEntry){
 213                 delete fStatusEntry;
 214                 fStatusEntry = 0;
 215         }
 216 }
 217
 218 //______________________________________________________________________________________________
 219 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
 220 {
 221         //
 222         // Registers new AliPreprocessor.
 223         // It uses GetName() for indentificator of the pre processor.
 224         // The pre processor is registered it there isn't any other
 225         // with the same identificator (GetName()).
 226         //
 227
 228         if (fPreprocessorMap.GetValue(preprocessor->GetName())) {
 229                 AliWarning(Form("AliPreprocessor %s is already registered!",
 230                         preprocessor->GetName()));
 231                 return;
 232         }
 233
 234         fPreprocessorMap.Add(new TObjString(preprocessor->GetName()), preprocessor);
 235 }
 236 //______________________________________________________________________________________________
 237 UInt_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
 238                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
 239 {
 240   // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
 241   // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
 242   // using this function. Use StoreReferenceData instead!
 243   // It calls WriteToCDB function which perform actual storage
 244
 245         return WriteToCDB(fgkMainCDB, fgkLocalCDB, path, object,
 246                                 metaData, validityStart, validityInfinite);
 247
 248 }
 249
 250 //______________________________________________________________________________________________
 251 UInt_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object,
 252                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
 253 {
 254   // Stores a CDB object in the storage for reference data. This objects will not be available during
 255   // offline reconstrunction. Use this function for reference data only!
 256   // It calls WriteToCDB function which perform actual storage
 257
 258         return WriteToCDB(fgkMainRefStorage, fgkLocalRefStorage, path, object,
 259                                 metaData, validityStart, validityInfinite);
 260
 261 }
 262
 263 //______________________________________________________________________________________________
 264 UInt_t AliShuttle::WriteToCDB(const char* mainUri, const char* localUri,
 265                         const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
 266                         Int_t validityStart, Bool_t validityInfinite)
 267 {
 268   // write object into the CDB. Parameters are passed by Store and StoreReferenceData functions.
 269   // The parameters are:
 270   //   1) Uri of the main storage (Grid)
 271   //   2) Uri of the backup storage (Local)
 272   //   3) the object's path.
 273   //   4) the object to be stored
 274   //   5) the metaData to be associated with the object
 275   //   6) the validity start run number w.r.t. the current run,
 276   //      if the data is valid only for this run leave the default 0
 277   //   7) specifies if the calibration data is valid for infinity (this means until updated),
 278   //      typical for calibration runs, the default is kFALSE
 279   //
 280   // returns 0 if fail
 281   //         1 if stored in main (Grid) storage
 282   //         2 if stored in backup (Local) storage
 283
 284         const char* cdbType = (mainUri == fgkMainCDB) ? "CDB" : "Reference";
 285
 286         Int_t firstRun = GetCurrentRun() - validityStart;
 287         if(firstRun < 0) {
 288                 AliError("First valid run happens to be less than 0! Setting it to 0.");
 289                 firstRun=0;
 290         }
 291
 292         Int_t lastRun = -1;
 293         if(validityInfinite) {
 294                 lastRun = AliCDBRunRange::Infinity();
 295         } else {
 296                 lastRun = GetCurrentRun();
 297         }
 298
 299         AliCDBId id(path, firstRun, lastRun, -1, -1);
 300
 301         if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
 302                 TObjString runUsed = Form("%d", GetCurrentRun());
 303                 metaData->SetProperty("RunUsed(TObjString)",&runUsed);
 304         }
 305
 306         UInt_t result = 0;
 307
 308         if (!(AliCDBManager::Instance()->GetStorage(mainUri))) {
 309                 AliError(Form("WriteToCDB - Cannot activate main %s storage", cdbType));
 310         } else {
 311                 result = (UInt_t) AliCDBManager::Instance()->GetStorage(mainUri)
 312                                         ->Put(object, id, metaData);
 313         }
 314
 315         if(!result) {
 316
 317                 Log(fCurrentDetector,
 318                         Form("WriteToCDB - Problem with main %s storage. Putting <%s> into backup storage",
 319                                 cdbType, path.GetPath().Data()));
 320
 321                 // Set Grid version to current run number, to ease retrieval later
 322                 id.SetVersion(GetCurrentRun());
 323
 324                 result = AliCDBManager::Instance()->GetStorage(localUri)
 325                                         ->Put(object, id, metaData);
 326
 327                 if(result) {
 328                         result = 2;
 329                         fGridError = kTRUE;
 330                 }else{
 331                         Log(fCurrentDetector, "WriteToCDB - Can't store data!");
 332                 }
 333         }
 334
 335         return result;
 336
 337 }
 338
 339 //______________________________________________________________________________________________
 340 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
 341 {
 342 // Reads the AliShuttleStatus from the CDB
 343
 344         if (fStatusEntry){
 345                 delete fStatusEntry;
 346                 fStatusEntry = 0;
 347         }
 348
 349         fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalCDB())
 350                 ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), GetCurrentRun());
 351
 352         if (!fStatusEntry) return 0;
 353         fStatusEntry->SetOwner(1);
 354
 355         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
 356         if (!status) {
 357                 AliError("Invalid object stored to CDB!");
 358                 return 0;
 359         }
 360
 361         return status;
 362 }
 363
 364 //______________________________________________________________________________________________
 365 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
 366 {
 367 // writes the status for one subdetector
 368
 369         if (fStatusEntry){
 370                 delete fStatusEntry;
 371                 fStatusEntry = 0;
 372         }
 373
 374         Int_t run = GetCurrentRun();
 375
 376         AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), run, run);
 377
 378         fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
 379         fStatusEntry->SetOwner(1);
 380
 381         UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
 382
 383         if (!result) {
 384                 AliError(Form("WriteShuttleStatus for %s, run %d failed", fCurrentDetector.Data(), run));
 385                 return kFALSE;
 386         }
 387
 388         return kTRUE;
 389 }
 390
 391 //______________________________________________________________________________________________
 392 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
 393 {
 394   // changes the AliShuttleStatus for the given detector and run to the given status
 395
 396         if (!fStatusEntry){
 397                 AliError("UNEXPECTED: fStatusEntry empty");
 398                 return;
 399         }
 400
 401         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
 402
 403         if (!status){
 404                 AliError("UNEXPECTED: status could not be read from current CDB entry");
 405                 return;
 406         }
 407
 408         Log("SHUTTLE", Form("UpdateShuttleStatus - %s: Changing state from %s to %s", fCurrentDetector.Data(),
 409                                 status->GetStatusName(), status->GetStatusName(newStatus)));
 410
 411         status->SetStatus(newStatus);
 412         if (increaseCount) status->IncreaseCount();
 413
 414         AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
 415 }
 416 //______________________________________________________________________________________________
 417 Bool_t AliShuttle::ContinueProcessing()
 418 {
 419 // this function reads the AliShuttleStatus information from CDB and
 420 // checks if the processing should be continued
 421 // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
 422
 423         if(!GetDetCode(fCurrentDetector)) {
 424                 Log("SHUTTLE", Form("ContinueProcessing - %s: unknown detector",
 425                                 fCurrentDetector.Data()));
 426                 return kFALSE;
 427         }
 428
 429         AliShuttleLogbookEntry::Status entryStatus =
 430                 fLogbookEntry->GetDetectorStatus(GetDetCode(fCurrentDetector));
 431
 432         if(entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
 433                 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s",
 434                                 fCurrentDetector.Data(),
 435                                 fLogbookEntry->GetDetectorStatusName(entryStatus)));
 436                 return kFALSE;
 437         }
 438
 439         // if we get here, according to Shuttle logbook subdetector is in UNPROCESSED state
 440         AliShuttleStatus* status = ReadShuttleStatus();
 441         if (!status) {
 442                 // first time
 443                 Log("SHUTTLE", Form("ContinueProcessing - %s: Processing first time",
 444                                 fCurrentDetector.Data()));
 445                 status = new AliShuttleStatus(AliShuttleStatus::kStarted);
 446                 return WriteShuttleStatus(status);
 447         }
 448
 449         // The following two cases shouldn't happen if Shuttle Logbook was correctly updated.
 450         // If it happens it may mean Logbook updating failed... let's do it now!
 451         if (status->GetStatus() == AliShuttleStatus::kDone ||
 452             status->GetStatus() == AliShuttleStatus::kFailed){
 453                 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s. Updating Shuttle Logbook",
 454                                         fCurrentDetector.Data(),
 455                                         status->GetStatusName(status->GetStatus())));
 456                 UpdateShuttleLogbook(fCurrentDetector.Data(),
 457                                         status->GetStatusName(status->GetStatus()));
 458                 return kFALSE;
 459         }
 460
 461         if (status->GetStatus() == AliShuttleStatus::kStoreFailed) {
 462                 Log("SHUTTLE",
 463                         Form("ContinueProcessing - %s: Grid storage of one or more objects failed. Trying again now",
 464                                 fCurrentDetector.Data()));
 465                 if(TryToStoreAgain()){
 466                         Log(fCurrentDetector.Data(), "ContinueProcessing - All objects successfully stored into OCDB");
 467                         UpdateShuttleStatus(AliShuttleStatus::kDone);
 468                         UpdateShuttleLogbook(fCurrentDetector.Data(), "DONE");
 469                 } else {
 470                         Log("SHUTTLE",
 471                                 Form("ContinueProcessing - %s: Grid storage failed again",
 472                                         fCurrentDetector.Data()));
 473                 }
 474                 return kFALSE;
 475         }
 476
 477         // if we get here, there is a restart
 478
 479         // abort conditions
 480         // TODO we should add two counters, one for PP and one for DCS!
 481         if ((status->GetStatus() == AliShuttleStatus::kPPStarted ||
 482              status->GetStatus() == AliShuttleStatus::kPPError) &&
 483             (status->GetCount() >= fConfig->GetMaxPPRetries() ||
 484              status->GetCount() >= fConfig->GetMaxRetries())) {
 485                 Log("SHUTTLE",
 486                         Form("ContinueProcessing - %s failed %d times in status %s - Updating Shuttle Logbook",
 487                                 fCurrentDetector.Data(),
 488                                 status->GetCount(), status->GetStatusName()));
 489                 UpdateShuttleLogbook(fCurrentDetector.Data(), "FAILED");
 490                 return kFALSE;
 491         }
 492
 493         Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. Got stuck before in %s. Retry number %d.",
 494                         fCurrentDetector.Data(),
 495                         status->GetStatusName(), status->GetCount()));
 496
 497         UpdateShuttleStatus(AliShuttleStatus::kStarted);
 498
 499         return kTRUE;
 500 }
 501
 502 //______________________________________________________________________________________________
 503 Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
 504 {
 505         //
 506         // Makes data retrieval for all detectors in the configuration.
 507         // entry: Shuttle logbook entry, contains run paramenters and status of detectors
 508         // (Unprocessed, Inactive, Failed or Done).
 509         // Returns kFALSE in case of error occured and kTRUE otherwise
 510         //
 511
 512         if(!entry) return kFALSE;
 513
 514         fLogbookEntry = entry;
 515
 516         if(fLogbookEntry->IsDone()){
 517                 Log("SHUTTLE","Process - Shuttle is already DONE. Updating logbook");
 518                 UpdateShuttleLogbook("shuttle_done");
 519                 fLogbookEntry = 0;
 520                 return kTRUE;
 521         }
 522
 523
 524         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^* \n",
 525                                         GetCurrentRun()));
 526
 527         fLogbookEntry->Print("");
 528
 529         // Initialization
 530         Bool_t hasError = kFALSE;
 531         for(Int_t iSys=0;iSys<3;iSys++) fFESCalled[iSys]=kFALSE;
 532
 533         AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
 534         if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
 535         AliCDBStorage *mainRefSto = AliCDBManager::Instance()->GetStorage(fgkMainRefStorage);
 536         if(mainRefSto) mainRefSto->QueryCDB(GetCurrentRun());
 537
 538         // Loop on detectors in the configuration
 539         TIter iter(fConfig->GetDetectors());
 540         TObjString* aDetector = 0;
 541
 542         while ((aDetector = (TObjString*) iter.Next())) {
 543                 fCurrentDetector = aDetector->String();
 544
 545                 if (!fConfig->HostProcessDetector(fCurrentDetector)) continue;
 546
 547                 AliPreprocessor* aPreprocessor =
 548                         dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
 549                 if(!aPreprocessor){
 550                         Log("SHUTTLE",Form("Process - %s: no preprocessor registered. Skipping"));
 551                         continue;
 552                 }
 553
 554                 if (ContinueProcessing() == kFALSE) continue;
 555
 556                 AliInfo(Form("\n\n \t\t\t****** run %d - %s: START  ******",
 557                                                 GetCurrentRun(), aDetector->GetName()));
 558
 559                 UInt_t result = ProcessCurrentDetector();
 560
 561                 if(!result) {
 562                         hasError = kTRUE;
 563                         AliInfo(Form("\n \t\t\t****** run %d - %s: PREPROCESSOR ERROR ****** \n\n",
 564                                                         GetCurrentRun(), aDetector->GetName()));
 565                         continue;
 566                 }
 567
 568                 if(result == 2) {
 569                         AliInfo(Form("\n \t\t\t****** run %d - %s: STORAGE ERROR ****** \n\n",
 570                                                         GetCurrentRun(), aDetector->GetName()));
 571                 } else {
 572                         AliInfo(Form("\n \t\t\t****** run %d - %s: DONE ****** \n\n",
 573                                                         GetCurrentRun(), aDetector->GetName()));
 574                 }
 575
 576                 // Process successful: Update time_processed field in FES logbooks!
 577                 if(fFESCalled[kDAQ]) {
 578                         hasError = (UpdateDAQTable() == kFALSE);
 579                         fFESlist[kDAQ].Clear();
 580                 }
 581                 //if(fFESCalled[kDCS]) {
 582                 //      hasError = UpdateDCSTable(aDetector->GetName());
 583                 //      fFESlist[kDCS].Clear();
 584                 //}
 585                 //if(fFESCalled[kHLT]) {
 586                 //      hasError = UpdateHLTTable(aDetector->GetName());
 587                 //      fFESlist[kHLT].Clear();
 588                 //}
 589
 590         }
 591
 592         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^* \n",
 593                                                         GetCurrentRun()));
 594
 595         //check if shuttle is done for this run, if so update logbook
 596         TObjArray checkEntryArray;
 597         checkEntryArray.SetOwner(1);
 598         TString whereClause = Form("where run=%d",GetCurrentRun());
 599         if(QueryShuttleLogbook(whereClause.Data(), checkEntryArray)) {
 600
 601                 AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
 602                                                         (checkEntryArray.At(0));
 603
 604                 if(checkEntry && checkEntry->IsDone()){
 605                         Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
 606                         UpdateShuttleLogbook("shuttle_done");
 607                 }
 608         }
 609
 610         fLogbookEntry = 0;
 611
 612         return hasError == kFALSE;
 613 }
 614
 615 //______________________________________________________________________________________________
 616 UInt_t AliShuttle::ProcessCurrentDetector()
 617 {
 618         //
 619         // Makes data retrieval just for a specific detector (fCurrentDetector).
 620         // Threre should be a configuration for this detector.
 621
 622         AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), GetCurrentRun()));
 623
 624         UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
 625
 626         TString host(fConfig->GetDCSHost(fCurrentDetector));
 627         Int_t port = fConfig->GetDCSPort(fCurrentDetector);
 628
 629         TIter iter(fConfig->GetDCSAliases(fCurrentDetector));
 630         TObjString* anAlias;
 631         TMap aliasMap;
 632         aliasMap.SetOwner(1);
 633
 634         Bool_t aDCSError = kFALSE;
 635         fGridError = kFALSE;
 636
 637         while ((anAlias = (TObjString*) iter.Next())) {
 638                 TObjArray *valueSet = new TObjArray();
 639                 valueSet->SetOwner(1);
 640                 // TODO Test only... I've added a flag that allows to
 641                 // exclude DCS archive DB query
 642                 if(fgkProcessDCS){
 643                         AliInfo("Querying DCS archive DB data...");
 644                         aDCSError = (GetValueSet(host, port, anAlias->String(), valueSet) == 0);
 645                 } else {
 646                         AliInfo(Form("Skipping DCS processing. Port = %d",port));
 647                         aDCSError = kFALSE;
 648                 }
 649                 if(!aDCSError) {
 650                         aliasMap.Add(anAlias->Clone(), valueSet);
 651                 }else{
 652                         Log(fCurrentDetector, Form("ProcessCurrentDetector - Error while retrieving alias %s",
 653                                         anAlias->GetName()));
 654                         UpdateShuttleStatus(AliShuttleStatus::kDCSError, kTRUE);
 655                         aliasMap.DeleteAll();
 656                         return 0;
 657                 }
 658         }
 659
 660         // DCS Archive DB processing successful. Call Preprocessor!
 661         UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
 662
 663         AliPreprocessor* aPreprocessor =
 664                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
 665
 666         aPreprocessor->Initialize(GetCurrentRun(), GetCurrentStartTime(), GetCurrentEndTime());
 667
 668         UInt_t aPPResult = aPreprocessor->Process(&aliasMap);
 669
 670         UInt_t returnValue = 0;
 671         if (aPPResult == 0) { // Preprocessor error
 672                 UpdateShuttleStatus(AliShuttleStatus::kPPError, kTRUE);
 673                 returnValue = 0;
 674         } else if (fGridError == kFALSE) { // process and Grid storage ok!
 675                 UpdateShuttleStatus(AliShuttleStatus::kDone);
 676                 UpdateShuttleLogbook(fCurrentDetector, "DONE");
 677                 Log(fCurrentDetector.Data(),
 678                         "ProcessCurrentDetector - Preprocessor and Grid storage ended successfully");
 679                 returnValue = 1;
 680         } else { // Grid storage error (process ok, but object put in local storage)
 681                 UpdateShuttleStatus(AliShuttleStatus::kStoreFailed);
 682                 returnValue = 2;
 683         }
 684
 685         aliasMap.DeleteAll();
 686
 687         return returnValue;
 688 }
 689
 690 //______________________________________________________________________________________________
 691 Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
 692                 TObjArray& entries)
 693 {
 694 // Query DAQ's Shuttle logbook and fills detector status object.
 695 // Call QueryRunParameters to query DAQ logbook for run parameters.
 696
 697         // check connection, in case connect
 698         if(!Connect(kDAQ)) return kFALSE;
 699
 700         TString sqlQuery;
 701         sqlQuery = Form("select * from logbook_shuttle %s order by run", whereClause);
 702
 703         TSQLResult* aResult = fServer[kDAQ]->Query(sqlQuery);
 704         if (!aResult) {
 705                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
 706                 return kFALSE;
 707         }
 708
 709         if(aResult->GetRowCount() == 0) {
 710                 if(sqlQuery.Contains("where shuttle_done=0")){
 711                         Log("SHUTTLE", "QueryShuttleLogbook - All runs in Shuttle Logbook are already DONE");
 712                         delete aResult;
 713                         return kTRUE;
 714                 } else {
 715                         AliError("No entries in Shuttle Logbook match request");
 716                         delete aResult;
 717                         return kFALSE;
 718                 }
 719         }
 720
 721         // TODO Check field count!
 722         const UInt_t nCols = 24;
 723         if (aResult->GetFieldCount() != (Int_t) nCols) {
 724                 AliError("Invalid SQL result field number!");
 725                 delete aResult;
 726                 return kFALSE;
 727         }
 728
 729         entries.SetOwner(1);
 730
 731         TSQLRow* aRow;
 732         while ((aRow = aResult->Next())) {
 733                 TString runString(aRow->GetField(0), aRow->GetFieldLength(0));
 734                 Int_t run = runString.Atoi();
 735
 736                 UInt_t startTime, endTime;
 737                 if(!QueryRunParameters(run, startTime, endTime)) continue;
 738
 739                 const UInt_t nDet = AliShuttle::kNDetectors;
 740                 AliShuttleLogbookEntry::Status detStatus[nDet];
 741
 742                 // loop on detectors
 743                 for(UInt_t ii = 0; ii < nCols; ii++){
 744                         TString detCode(aResult->GetFieldName(ii));
 745                         Int_t detPos = AliShuttle::GetDetPos(detCode.Data());
 746                         if(detPos < 0) continue;
 747                         TString statusString(aRow->GetField(ii), aRow->GetFieldLength(ii));
 748                         if(statusString == "UNPROCESSED"){
 749                                 detStatus[detPos] = AliShuttleLogbookEntry::kUnprocessed;
 750                         } else if (statusString == "INACTIVE") {
 751                                 detStatus[detPos] = AliShuttleLogbookEntry::kInactive;
 752                         } else if (statusString == "FAILED") {
 753                                 detStatus[detPos] = AliShuttleLogbookEntry::kFailed;
 754                         } else if (statusString == "DONE") {
 755                                 detStatus[detPos] = AliShuttleLogbookEntry::kDone;
 756                         }
 757                 }
 758
 759                 entries.AddLast(new AliShuttleLogbookEntry(run, startTime, endTime, detStatus));
 760                 delete aRow;
 761         }
 762
 763         if(sqlQuery.Contains("where shuttle_done=0"))
 764                 Log("SHUTTLE", Form("QueryShuttleLogbook - Found %d unprocessed runs in Shuttle Logbook",
 765                                                         entries.GetEntriesFast()));
 766         delete aResult;
 767         return kTRUE;
 768 }
 769
 770 //______________________________________________________________________________________________
 771 Bool_t AliShuttle::QueryRunParameters(Int_t& run, UInt_t& startTime, UInt_t& endTime)
 772 {
 773 // Retrieve start time and end time for run in the DAQ logbook
 774
 775         // check connection, in case connect
 776         if(!Connect(kDAQ)) return kFALSE;
 777
 778         TString sqlQuery;
 779         sqlQuery = Form("select time_start, time_end from logbook where run=%d", run);
 780
 781         TSQLResult* aResult = fServer[kDAQ]->Query(sqlQuery);
 782         if (!aResult) {
 783                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
 784                 return kFALSE;
 785         }
 786
 787         if(aResult->GetRowCount() == 0) {
 788                 Log("SHUTTLE", Form("QueryRunParameters - No entry in DAQ Logbook for run %d. Skipping", run));
 789                 delete aResult;
 790                 return kFALSE;
 791         }
 792
 793         if(aResult->GetRowCount() > 1) {
 794                 AliError(Form("More than one entry in DAQ Logbook for run %d. Skipping", run));
 795                 delete aResult;
 796                 return kFALSE;
 797         }
 798
 799         TSQLRow* aRow;
 800         while ((aRow = aResult->Next())) {
 801
 802                 TString startTimeString(aRow->GetField(0),
 803                                 aRow->GetFieldLength(0));
 804                 startTime = startTimeString.Atoi();
 805                 TString endTimeString(aRow->GetField(1),
 806                                 aRow->GetFieldLength(1));
 807                 endTime = endTimeString.Atoi();
 808
 809                 if (!startTime || !endTime || startTime > endTime) {
 810                         Log("SHUTTLE",
 811                                 Form("QueryRunParameters - Invalid parameters for Run %d: startTime = %d, endTime = %d",
 812                                         run, startTime, endTime));
 813                         delete aRow;
 814                         delete aResult;
 815                         return kFALSE;
 816                 }
 817
 818                 delete aRow;
 819         }
 820
 821         delete aResult;
 822         return kTRUE;
 823 }
 824
 825 //______________________________________________________________________________________________
 826 Bool_t AliShuttle::TryToStoreAgain()
 827 {
 828   // Called in case the detector failed to store the object in Grid OCDB
 829   // It tries to store the object again, if it does not find more recent and overlapping objects
 830   // Calls underlying TryToStoreAgain(const char*) function twice, for OCDB and Reference storage.
 831
 832         AliInfo("Trying to store OCDB data again...");
 833         Bool_t resultCDB = TryToStoreAgain(fgkMainCDB);
 834
 835         AliInfo("Trying to store reference data again...");
 836         Bool_t resultRef = TryToStoreAgain(fgkMainRefStorage);
 837
 838         return resultCDB && resultRef;
 839 }
 840
 841 //______________________________________________________________________________________________
 842 Bool_t AliShuttle::TryToStoreAgain(TString& gridURI)
 843 {
 844   // Called by TryToStoreAgain(), performs actual storage retry
 845
 846         TList* gridIds=0;
 847
 848         Bool_t result = kTRUE;
 849
 850         const char* type = 0;
 851         TString backupURI;
 852         if(gridURI == fgkMainCDB) {
 853                 type = "OCDB";
 854                 backupURI = fgkLocalCDB;
 855         } else if(gridURI == fgkMainRefStorage) {
 856                 type = "reference";
 857                 backupURI = fgkLocalRefStorage;
 858         } else {
 859                 AliError(Form("Invalid storage URI: %s", gridURI.Data()));
 860                 return kFALSE;
 861         }
 862
 863         AliCDBManager* man = AliCDBManager::Instance();
 864
 865         AliCDBStorage *gridSto = man->GetStorage(gridURI);
 866         if(!gridSto) {
 867                 Log(fCurrentDetector.Data(),
 868                         Form("TryToStoreAgain - cannot activate main %s storage", type));
 869                 return kFALSE;
 870         }
 871
 872         gridIds = gridSto->GetQueryCDBList();
 873
 874         // get objects previously stored in local CDB
 875         AliCDBStorage *backupSto = man->GetStorage(backupURI);
 876         AliCDBPath aPath(fCurrentDetector,"*","*");
 877         // Local objects were stored with current run as Grid version!
 878         TList* localEntries = backupSto->GetAll(aPath.GetPath(), GetCurrentRun(), GetCurrentRun());
 879         localEntries->SetOwner(1);
 880
 881         // loop on local stored objects
 882         TIter localIter(localEntries);
 883         AliCDBEntry *aLocEntry = 0;
 884         while((aLocEntry = dynamic_cast<AliCDBEntry*> (localIter.Next()))){
 885                 aLocEntry->SetOwner(1);
 886                 AliCDBId aLocId = aLocEntry->GetId();
 887                 aLocEntry->SetVersion(-1);
 888                 aLocEntry->SetSubVersion(-1);
 889
 890                 // loop on Grid valid Id's
 891                 Bool_t store = kTRUE;
 892                 TIter gridIter(gridIds);
 893                 AliCDBId* aGridId = 0;
 894                 while((aGridId = dynamic_cast<AliCDBId*> (gridIter.Next()))){
 895                         // If local object is valid up to infinity we store it anyway
 896                         // TODO This does not work! It may hide more recent objects...
 897                         if(aLocId.GetLastRun() == AliCDBRunRange::Infinity()) {
 898                                 // TODO Check that it won't hide more recent files! how????
 899                                 break;
 900                         }
 901                         if(aGridId->GetPath() != aLocId.GetPath()) continue;
 902                         // skip all objects valid up to infinity
 903                         if(aGridId->GetLastRun() == AliCDBRunRange::Infinity()) continue;
 904                         // if we get here, it means there's already some more recent object stored on Grid!
 905                         store = kFALSE;
 906                         break;
 907                 }
 908
 909                 if(!store){
 910                         Log(fCurrentDetector.Data(),
 911                                 Form("TryToStoreAgain - A more recent object already exists in %s storage: <%s>",
 912                                         type, aGridId->ToString().Data()));
 913                         // removing local filename...
 914                         // TODO maybe it's better not to remove it, it was not copied to the Grid!
 915                         TString filename;
 916                         backupSto->IdToFilename(aLocId, filename);
 917                         AliInfo(Form("Removing local file %s", filename.Data()));
 918                         gSystem->Exec(Form("rm %s",filename.Data()));
 919                         continue;
 920                 }
 921
 922                 // If we get here, the file can be stored!
 923                 Bool_t storeOk = gridSto->Put(aLocEntry);
 924                 if(storeOk){
 925                         Log(fCurrentDetector.Data(),
 926                                 Form("TryToStoreAgain - Object <%s> successfully put into %s storage",
 927                                         aLocId.ToString().Data(), type));
 928
 929                         // removing local filename...
 930                         TString filename;
 931                         backupSto->IdToFilename(aLocId, filename);
 932                         AliInfo(Form("Removing local file %s", filename.Data()));
 933                         gSystem->Exec(Form("rm %s", filename.Data()));
 934                         continue;
 935                 } else  {
 936                         Log(fCurrentDetector.Data(),
 937                                 Form("TryToStoreAgain - Grid %s storage of object <%s> failed again",
 938                                         type, aLocId.ToString().Data()));
 939                         result = kFALSE;
 940                 }
 941         }
 942         localEntries->Clear();
 943
 944         return result;
 945 }
 946
 947 //______________________________________________________________________________________________
 948 Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* alias,
 949                                 TObjArray* valueSet)
 950 {
 951 // Retrieve all "alias" data points from the DCS server
 952 // host, port: TSocket connection parameters
 953 // alias: name of the alias
 954 // valueSet: array of retrieved AliDCSValue's
 955
 956         AliDCSClient client(host, port, fTimeout, fRetries);
 957         if (!client.IsConnected()) {
 958                 return kFALSE;
 959         }
 960
 961         Int_t result = client.GetAliasValues(alias,
 962                 GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
 963
 964         if (result < 0) {
 965                 Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get '%s'! Reason: %s",
 966                         alias, AliDCSClient::GetErrorString(result)));
 967
 968                 if (result == AliDCSClient::fgkServerError) {
 969                         Log(fCurrentDetector.Data(), Form("GetValueSet - Server error: %s",
 970                                 client.GetServerError().Data()));
 971                 }
 972
 973                 return kFALSE;
 974         }
 975
 976         return kTRUE;
 977 }
 978
 979 //______________________________________________________________________________________________
 980 const char* AliShuttle::GetFile(Int_t system, const char* detector,
 981                 const char* id, const char* source)
 982 {
 983 // Get calibration file from file exchange servers
 984 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
 985
 986         switch(system){
 987                 case kDAQ:
 988                         return GetDAQFileName(detector, id, source);
 989                         break;
 990                 case kDCS:
 991                         return GetDCSFileName(detector, id, source);
 992                         break;
 993                 case kHLT:
 994                         return GetHLTFileName(detector, id, source);
 995                         break;
 996                 default:
 997                         AliError(Form("No valid system index: %d",system));
 998         }
 999
1000         return 0;
1001 }
1002
1003 //______________________________________________________________________________________________
1004 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
1005 {
1006 // Get sources producing the condition file Id from file exchange servers
1007 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
1008
1009         switch(system){
1010                 case kDAQ:
1011                         return GetDAQFileSources(detector, id);
1012                         break;
1013                 case kDCS:
1014                         return GetDCSFileSources(detector, id);
1015                         break;
1016                 case kHLT:
1017                         return GetHLTFileSources(detector, id);
1018                         break;
1019                 default:
1020                         AliError(Form("No valid system index: %d",system));
1021         }
1022
1023         return NULL;
1024 }
1025
1026 //______________________________________________________________________________________________
1027 Bool_t AliShuttle::Connect(Int_t system)
1028 {
1029 // Connect to MySQL Server of the system's FES logbook
1030 // DAQ Logbook, Shuttle Logbook and DAQ FES Logbook are on the same host
1031
1032         // check connection: if already connected return
1033         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
1034
1035         TString aFESlbHost= Form("mysql://%s", fConfig->GetFESlbHost(system));
1036
1037         fServer[system] = TSQLServer::Connect(aFESlbHost,
1038                         fConfig->GetFESlbUser(system),
1039                         fConfig->GetFESlbPass(system));
1040         if (!fServer[system] || !fServer[system]->IsConnected()) {
1041                 AliError(Form("Can't establish connection to FES logbook for %s",fkSystemNames[system]));
1042                 if(fServer[system]) delete fServer[system];
1043                 return kFALSE;
1044         }
1045
1046         // Get tables
1047         // TODO in the configuration should the table name be there too?
1048         TSQLResult* aResult=0;
1049         switch(system){
1050                 case kDAQ:
1051                         aResult = fServer[kDAQ]->GetTables("REFSYSLOG");
1052                         break;
1053                 case kDCS:
1054                         //aResult = fServer[kDCS]->GetTables("REFSYSLOG");
1055                         break;
1056                 case kHLT:
1057                         //aResult = fServer[kHLT]->GetTables("REFSYSLOG");
1058                         break;
1059                 default:
1060                         break;
1061         }
1062
1063         delete aResult;
1064         return kTRUE;
1065 }
1066
1067 //______________________________________________________________________________________________
1068 const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source)
1069 {
1070 // Retrieves a file from the DAQ FES.
1071 // First queris the DAQ logbook_fs for the DAQ file name, using the run, detector, id and source info
1072 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
1073 // run: current run being processed (given by Logbook entry fLogbookEntry)
1074 // detector: comes from the Preprocessor name (must be converted into detector code with GetDetCode)
1075 // id: provided as a parameter by the Preprocessor
1076 // source: provided by the Preprocessor through GetFileSources function
1077
1078         // check connection, in case connect
1079         if(!Connect(kDAQ)){
1080                 Log(detector, "GetDAQFileName - Couldn't connect to DAQ Logbook");
1081                 return 0;
1082         }
1083
1084         // Query preparation
1085         TString sqlQueryStart = "select filePath from logbook_fs where";
1086         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
1087                                 GetCurrentRun(), GetDetCode(detector), id, source);
1088         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1089
1090         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1091
1092         // Query execution
1093         TSQLResult* aResult = 0;
1094         aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1095         if (!aResult) {
1096                 Log(detector, Form("GetDAQFileName - Can't execute SQL query for: id = %s, source = %s",
1097                                 id, source));
1098                 return 0;
1099         }
1100
1101         if (aResult->GetRowCount() == 0) {
1102                 Log(detector,
1103                         Form("GetDAQFileName - No entry in FES table for: id = %s, source = %s",
1104                                 id, source));
1105                 delete aResult;
1106                 return 0;
1107         }
1108
1109         if (aResult->GetRowCount() >1) {
1110                 Log(detector,
1111                         Form("GetDAQFileName - More than one entry in FES table for: id = %s, source = %s",
1112                                 id, source));
1113                 delete aResult;
1114                 return 0;
1115         }
1116
1117         TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
1118
1119         if(!aRow){
1120                 Log(detector, Form("GetDAQFileName - Empty set result from query: id = %s, source = %s",
1121                                 id, source));
1122                 delete aResult;
1123                 return 0;
1124         }
1125
1126         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
1127
1128         delete aResult;
1129         delete aRow;
1130
1131         AliDebug(2, Form("filePath = %s",filePath.Data()));
1132
1133         // retrieved file is renamed to make it unique
1134         TString localFileName = Form("%s_%d_%s_%s.shuttle",
1135                                         detector, GetCurrentRun(), id, source);
1136
1137         // file retrieval from DAQ FES
1138         Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
1139         if(!result) {
1140                 Log(detector, Form("GetDAQFileName - Copy of file %s from DAQ FES failed", filePath.Data()));
1141                 return 0;
1142         } else {
1143                 AliInfo(Form("File %s copied from DAQ FES into %s/%s",
1144                         filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
1145         }
1146
1147
1148         fFESCalled[kDAQ]=kTRUE;
1149         TObjString *fileParams = new TObjString(Form("%s_!?!_%s", id, source));
1150         fFESlist[kDAQ].Add(fileParams);
1151
1152         return localFileName.Data();
1153
1154 }
1155
1156 //______________________________________________________________________________________________
1157 Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName)
1158 {
1159
1160         // check temp directory: trying to cd to temp; if it does not exist, create it
1161         AliDebug(2, Form("Copy file %s from DAQ FES into folder %s and rename it as %s",
1162                         daqFileName,fgkShuttleTempDir, localFileName));
1163
1164         void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
1165         if (dir == NULL) {
1166                 if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
1167                         AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
1168                         return kFALSE;
1169                 }
1170
1171         } else {
1172                 gSystem->FreeDirectory(dir);
1173         }
1174
1175         TString baseDAQFESFolder = "DAQ";
1176         TString command = Form("scp %s@%s:%s/%s %s/%s",
1177                 fConfig->GetFESUser(kDAQ),
1178                 fConfig->GetFESHost(kDAQ),
1179                 baseDAQFESFolder.Data(),
1180                 daqFileName,
1181                 fgkShuttleTempDir,
1182                 localFileName);
1183
1184         AliDebug(2, Form("%s",command.Data()));
1185
1186         UInt_t nRetries = 0;
1187         UInt_t maxRetries = 3;
1188
1189         // copy!! if successful TSystem::Exec returns 0
1190         while(nRetries++ < maxRetries) {
1191                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
1192                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
1193         }
1194
1195         return kFALSE;
1196
1197 }
1198
1199 //______________________________________________________________________________________________
1200 TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id)
1201 {
1202 // Retrieves a file from the DCS FES.
1203
1204         // check connection, in case connect
1205         if(!Connect(kDAQ)){
1206                 Log(detector, "GetDAQFileSources - Couldn't connect to DAQ Logbook");
1207                 return 0;
1208         }
1209
1210         // Query preparation
1211         TString sqlQueryStart = "select DAQsource from logbook_fs where";
1212         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
1213                                 GetCurrentRun(), GetDetCode(detector), id);
1214         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1215
1216         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1217
1218         // Query execution
1219         TSQLResult* aResult;
1220         aResult = fServer[kDAQ]->Query(sqlQuery);
1221         if (!aResult) {
1222                 Log(detector, Form("GetDAQFileSources - Can't execute SQL query for id: %s", id));
1223                 return 0;
1224         }
1225
1226         if (aResult->GetRowCount() == 0) {
1227                 Log(detector,
1228                         Form("GetDAQFileSources - No entry in FES table for id: %s", id));
1229                 delete aResult;
1230                 return 0;
1231         }
1232
1233         TSQLRow* aRow;
1234         TList *list = new TList();
1235         list->SetOwner(1);
1236
1237         while((aRow = aResult->Next())){
1238
1239                 TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
1240                 AliDebug(2, Form("daqSource = %s", daqSource.Data()));
1241                 list->Add(new TObjString(daqSource));
1242                 delete aRow;
1243         }
1244         delete aResult;
1245
1246         return list;
1247
1248 }
1249
1250 //______________________________________________________________________________________________
1251 const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1252 // Retrieves a file from the DCS FES.
1253
1254 return "You're in DCS";
1255
1256 }
1257
1258 //______________________________________________________________________________________________
1259 TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
1260 // Retrieves a file from the DCS FES.
1261
1262 return NULL;
1263
1264 }
1265
1266 //______________________________________________________________________________________________
1267 const char* AliShuttle::GetHLTFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1268 // Retrieves a file from the HLT FES.
1269
1270 return "You're in HLT";
1271
1272 }
1273
1274 //______________________________________________________________________________________________
1275 TList* AliShuttle::GetHLTFileSources(const char* /*detector*/, const char* /*id*/){
1276 // Retrieves a file from the HLT FES.
1277
1278 return NULL;
1279
1280 }
1281
1282 //______________________________________________________________________________________________
1283 Bool_t AliShuttle::UpdateDAQTable()
1284 {
1285 // Update DAQ table filling time_processed field in all rows corresponding to current run and detector
1286
1287         // check connection, in case connect
1288         if(!Connect(kDAQ)){
1289                 Log(fCurrentDetector, "UpdateDAQTable - Couldn't connect to DAQ Logbook");
1290                 return kFALSE;
1291         }
1292
1293         TTimeStamp now; // now
1294
1295         // Loop on FES list entries
1296         TIter iter(&fFESlist[kDAQ]);
1297         TObjString *aFESentry=0;
1298         while((aFESentry = dynamic_cast<TObjString*> (iter.Next()))){
1299                 TString aFESentrystr = aFESentry->String();
1300                 TObjArray *aFESarray = aFESentrystr.Tokenize("_!?!_");
1301                 if(!aFESarray || aFESarray->GetEntries() != 2 ) {
1302                         Log(fCurrentDetector, Form("UpdateDAQTable - error updating FES entry. Check string: <%s>",
1303                                 aFESentrystr.Data()));
1304                         if(aFESarray) delete aFESarray;
1305                         return kFALSE;
1306                 }
1307                 const char* fileId = ((TObjString*) aFESarray->At(0))->GetName();
1308                 const char* daqSource = ((TObjString*) aFESarray->At(1))->GetName();
1309                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
1310                         GetCurrentRun(), GetDetCode(fCurrentDetector), fileId, daqSource);
1311
1312                 delete aFESarray;
1313
1314                 TString sqlQuery = Form("update logbook_fs set time_processed=%d %s", now.GetSec(), whereClause.Data());
1315
1316                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1317
1318                 // Query execution
1319                 TSQLResult* aResult;
1320                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1321                 if (!aResult) {
1322                         Log(fCurrentDetector, Form("UpdateDAQTable - Can't execute SQL query <%s>", sqlQuery.Data()));
1323                         return kFALSE;
1324                 }
1325                 delete aResult;
1326         }
1327
1328         return kTRUE;
1329 }
1330
1331
1332 //______________________________________________________________________________________________
1333 Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status)
1334 {
1335 // Update Shuttle logbook filling detector or shuttle_done column
1336 // ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
1337
1338         // check connection, in case connect
1339         if(!Connect(kDAQ)){
1340                 Log("SHUTTLE", "UpdateShuttleLogbook - Couldn't connect to DAQ Logbook.");
1341                 return kFALSE;
1342         }
1343
1344         TString detName(detector);
1345         TString setClause;
1346         if(detName == "shuttle_done") {
1347                 setClause = "set shuttle_done=1";
1348         } else {
1349                 TString detCode = GetDetCode(detector);
1350                 if(detCode.IsNull()) {
1351                         Log("SHUTTLE", Form("UpdateShuttleLogbook - Unknown detector %s", detector));
1352                         return kFALSE;
1353                 }
1354                 TString statusStr(status);
1355                 if(statusStr.Contains("done", TString::kIgnoreCase) ||
1356                    statusStr.Contains("failed", TString::kIgnoreCase)){
1357                         setClause = Form("set %s=\"%s\"", detCode.Data(), status);
1358                 } else {
1359                         Log("SHUTTLE",
1360                                 Form("UpdateShuttleLogbook - Invalid status <%s> for detector %s",
1361                                         status, detector));
1362                         return kFALSE;
1363                 }
1364         }
1365
1366         TString whereClause = Form("where run=%d", GetCurrentRun());
1367
1368         TString sqlQuery = Form("update logbook_shuttle %s %s",
1369                                         setClause.Data(), whereClause.Data());
1370
1371         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1372
1373         // Query execution
1374         TSQLResult* aResult;
1375         aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1376         if (!aResult) {
1377                 Log("SHUTTLE", Form("UpdateShuttleLogbook - Can't execute query <%s>", sqlQuery.Data()));
1378                 return kFALSE;
1379         }
1380         delete aResult;
1381
1382         return kTRUE;
1383 }
1384
1385 //______________________________________________________________________________________________
1386 Int_t AliShuttle::GetCurrentRun() const
1387 {
1388 // Get current run from logbook entry
1389
1390         return fLogbookEntry ? fLogbookEntry->GetRun() : -1;
1391 }
1392
1393 //______________________________________________________________________________________________
1394 UInt_t AliShuttle::GetCurrentStartTime() const
1395 {
1396 // get current start time
1397
1398         return fLogbookEntry ? fLogbookEntry->GetStartTime() : 0;
1399 }
1400
1401 //______________________________________________________________________________________________
1402 UInt_t AliShuttle::GetCurrentEndTime() const
1403 {
1404 // get current end time from logbook entry
1405
1406         return fLogbookEntry ? fLogbookEntry->GetEndTime() : 0;
1407 }
1408
1409 //______________________________________________________________________________________________
1410 const char* AliShuttle::GetDetCode(const char* detector){
1411 // Return detector code
1412
1413         for(UInt_t iDet=0; iDet < kNDetectors; iDet++){
1414                 if(!strcmp(fgkDetectorName[iDet], detector)) return fgkDetectorCode[iDet];
1415         }
1416
1417         AliErrorClass(Form("Unknown detector: %s",detector));
1418         return 0;
1419 }
1420
1421 //______________________________________________________________________________________________
1422 const char* AliShuttle::GetDetCode(UInt_t detPos){
1423 // Return detector code
1424
1425         if( detPos >= kNDetectors) {
1426                 AliErrorClass(Form("Invalid parameter: %d", detPos));
1427                 return 0;
1428         }
1429         return fgkDetectorCode[detPos];
1430 }
1431
1432 //______________________________________________________________________________________________
1433 const Int_t AliShuttle::GetDetPos(const char* detCode){
1434 // Return detector position in the detector code array
1435
1436         for(UInt_t iDet=0; iDet < kNDetectors; iDet++){
1437                 if(!strcmp(fgkDetectorCode[iDet], detCode)) return iDet;
1438         }
1439         return -1;
1440 }
1441
1442 //______________________________________________________________________________________________
1443 void AliShuttle::Log(const char* detector, const char* message)
1444 {
1445 // Fill log string with a message
1446
1447         void* dir = gSystem->OpenDirectory(fgkShuttleLogDir);
1448         if (dir == NULL) {
1449                 if (gSystem->mkdir(fgkShuttleLogDir, kTRUE)) {
1450                         AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
1451                         return;
1452                 }
1453
1454         } else {
1455                 gSystem->FreeDirectory(dir);
1456         }
1457
1458         TString toLog = Form("%s: %s - ", TTimeStamp(time(0)).AsString("s"), detector);
1459         if(GetCurrentRun()>=0 ) toLog += Form("run %d - ", GetCurrentRun());
1460         toLog += Form("%s", message);
1461
1462         AliInfo(toLog.Data());
1463
1464         TString fileName;
1465         fileName.Form("%s/%s.log", fgkShuttleLogDir, detector);
1466         gSystem->ExpandPathName(fileName);
1467
1468         ofstream logFile;
1469         logFile.open(fileName, ofstream::out | ofstream::app);
1470
1471         if (!logFile.is_open()) {
1472                 AliError(Form("Could not open file %s", fileName.Data()));
1473                 return;
1474         }
1475
1476         logFile << toLog.Data() << "\n";
1477
1478         logFile.close();
1479 }
1480
1481
1482 //______________________________________________________________________________________________
1483 Bool_t AliShuttle::Collect(Int_t run)
1484 {
1485         //
1486         // Collects conditions data for the given run.
1487         //
1488
1489         AliInfo(Form("Collecting conditions data for run %d", run));
1490
1491         TString whereClause("where run=");
1492         whereClause += run;
1493
1494         TObjArray dateEntries;
1495         if (!QueryShuttleLogbook(whereClause, dateEntries)) {
1496                 AliError("Can't retrieve entries from Shuttle logbook!");
1497                 return kFALSE;
1498         }
1499
1500         if (!dateEntries.GetEntriesFast()) {
1501                 AliError(Form("Retrieval of parameters for run %d failed!", run));
1502                 return kFALSE;
1503         }
1504
1505         if (dateEntries.GetEntriesFast() > 1) {
1506                 AliError(Form("There is more than one entry for run <%d> in Shuttle logbook!", run));
1507                 return kFALSE;
1508         }
1509
1510         if (!RetrieveConditionsData(dateEntries)) {
1511                 AliError("An error occured during conditions data retrieval!");
1512                 return kFALSE;
1513         }
1514
1515         return kTRUE;
1516 }
1517
1518 //______________________________________________________________________________________________
1519 Bool_t AliShuttle::CollectNew()
1520 {
1521         //
1522         // Collects conditions data for all UNPROCESSED run written to DAQ LogBook.
1523         // In operational mode, this is the Shuttle function triggered by the EOR signal.
1524         //
1525
1526         Log("SHUTTLE","CollectNew - Shuttle called. Collecting conditions data for unprocessed runs");
1527
1528         TString whereClause("where shuttle_done=0");
1529
1530         TObjArray shuttleLogbookEntries;
1531         if (!QueryShuttleLogbook(whereClause, shuttleLogbookEntries)) {
1532                 Log("SHUTTLE", "CollectNew - Can't retrieve entries from Shuttle logbook");
1533                 return kFALSE;
1534         }
1535
1536         if (!RetrieveConditionsData(shuttleLogbookEntries)) {
1537                 Log("SHUTTLE", "CollectNew - Process of at least one run failed");
1538                 return kFALSE;
1539         }
1540
1541         return kTRUE;
1542 }
1543
1544 //______________________________________________________________________________________________
1545 Bool_t AliShuttle::CollectAll()
1546 {
1547         //
1548         // Collects conditions data for all runs (even if they're already done!) written in Shuttle LogBook.
1549         //
1550
1551         AliInfo("Collecting conditions data for all runs ...");
1552
1553         TObjArray dateEntries;
1554         if (!QueryShuttleLogbook("", dateEntries)) {
1555                 AliError("Can't retrieve entries from Shuttle logbook");
1556                 return kFALSE;
1557         }
1558
1559         if (!RetrieveConditionsData(dateEntries)) {
1560                 AliError("An error occured during conditions data retrieval!");
1561                 return kFALSE;
1562         }
1563
1564         return kTRUE;
1565 }
1566
1567
1568 //______________________________________________________________________________________________
1569 Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
1570 {
1571 // Retrieve conditions data for all runs that aren't processed yet
1572
1573         Bool_t hasError = kFALSE;
1574
1575         TIter iter(&dateEntries);
1576         AliShuttleLogbookEntry* anEntry;
1577
1578         while ((anEntry = (AliShuttleLogbookEntry*) iter.Next())){
1579                 if (!Process(anEntry)){
1580                         hasError = kTRUE;
1581                 }
1582         }
1583
1584         return hasError == kFALSE;
1585 }