SHUTTLE/AliShuttle.cxx

   1 /**************************************************************************
   2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
   3  *                                                                        *
   4  * Author: The ALICE Off-line Project.                                    *
   5  * Contributors are mentioned in the code where appropriate.              *
   6  *                                                                        *
   7  * Permission to use, copy, modify and distribute this software and its   *
   8  * documentation strictly for non-commercial purposes is hereby granted   *
   9  * without fee, provided that the above copyright notice appears in all   *
  10  * copies and that both the copyright notice and this permission notice   *
  11  * appear in the supporting documentation. The authors make no claims     *
  12  * about the suitability of this software for any purpose. It is          *
  13  * provided "as is" without express or implied warranty.                  *
  14  **************************************************************************/
  15
  16 /*
  17 $Log$
  18 Revision 1.12  2006/08/08 14:19:29  jgrosseo
  19 Update to shuttle classes (Alberto)
  20
  21 - Possibility to set the full object's path in the Preprocessor's and
  22 Shuttle's  Store functions
  23 - Possibility to extend the object's run validity in the same classes
  24 ("startValidity" and "validityInfinite" parameters)
  25 - Implementation of the StoreReferenceData function to store reference
  26 data in a dedicated CDB storage.
  27
  28 Revision 1.11  2006/07/21 07:37:20  jgrosseo
  29 last run is stored after each run
  30
  31 Revision 1.10  2006/07/20 09:54:40  jgrosseo
  32 introducing status management: The processing per subdetector is divided into several steps,
  33 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
  34 can keep track of the number of failures and skips further processing after a certain threshold is
  35 exceeded. These thresholds can be configured in LDAP.
  36
  37 Revision 1.9  2006/07/19 10:09:55  jgrosseo
  38 new configuration, accesst to DAQ FES (Alberto)
  39
  40 Revision 1.8  2006/07/11 12:44:36  jgrosseo
  41 adding parameters for extended validity range of data produced by preprocessor
  42
  43 Revision 1.7  2006/07/10 14:37:09  jgrosseo
  44 small fix + todo comment
  45
  46 Revision 1.6  2006/07/10 13:01:41  jgrosseo
  47 enhanced storing of last sucessfully processed run (alberto)
  48
  49 Revision 1.5  2006/07/04 14:59:57  jgrosseo
  50 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
  51
  52 Revision 1.4  2006/06/12 09:11:16  jgrosseo
  53 coding conventions (Alberto)
  54
  55 Revision 1.3  2006/06/06 14:26:40  jgrosseo
  56 o) removed files that were moved to STEER
  57 o) shuttle updated to follow the new interface (Alberto)
  58
  59 Revision 1.2  2006/03/07 07:52:34  hristov
  60 New version (B.Yordanov)
  61
  62 Revision 1.6  2005/11/19 17:19:14  byordano
  63 RetrieveDATEEntries and RetrieveConditionsData added
  64
  65 Revision 1.5  2005/11/19 11:09:27  byordano
  66 AliShuttle declaration added
  67
  68 Revision 1.4  2005/11/17 17:47:34  byordano
  69 TList changed to TObjArray
  70
  71 Revision 1.3  2005/11/17 14:43:23  byordano
  72 import to local CVS
  73
  74 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
  75 Initial import as subdirectory in AliRoot
  76
  77 Revision 1.2  2005/09/13 08:41:15  byordano
  78 default startTime endTime added
  79
  80 Revision 1.4  2005/08/30 09:13:02  byordano
  81 some docs added
  82
  83 Revision 1.3  2005/08/29 21:15:47  byordano
  84 some docs added
  85
  86 */
  87
  88 //
  89 // This class is the main manager for AliShuttle.
  90 // It organizes the data retrieval from DCS and call the
  91 // interface methods of AliPreprocessor.
  92 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
  93 // data for its set of aliases is retrieved. If there is registered
  94 // AliPreprocessor for this detector then it will be used
  95 // accroding to the schema (see AliPreprocessor).
  96 // If there isn't registered AliPreprocessor than the retrieved
  97 // data is stored automatically to the undelying AliCDBStorage.
  98 // For detSpec is used the alias name.
  99 //
 100
 101 #include "AliShuttle.h"
 102
 103 #include "AliCDBManager.h"
 104 #include "AliCDBStorage.h"
 105 #include "AliCDBId.h"
 106 #include "AliCDBRunRange.h"
 107 #include "AliCDBPath.h"
 108 #include "AliCDBEntry.h"
 109 #include "AliShuttleConfig.h"
 110 #include "AliDCSClient.h"
 111 #include "AliLog.h"
 112 #include "AliPreprocessor.h"
 113 #include "AliShuttleStatus.h"
 114
 115 #include <TSystem.h>
 116 #include <TObject.h>
 117 #include <TString.h>
 118 #include <TTimeStamp.h>
 119 #include <TObjString.h>
 120 #include <TSQLServer.h>
 121 #include <TSQLResult.h>
 122 #include <TSQLRow.h>
 123
 124 #include <fstream>
 125
 126 ClassImp(AliShuttle)
 127
 128 TString AliShuttle::fgkMainCDB("alien://DBFolder=ShuttleCDB");
 129 TString AliShuttle::fgkLocalCDB("local://LocalShuttleCDB");
 130 TString AliShuttle::fgkMainRefStorage("alien://DBFolder=ShuttleReference");
 131 TString AliShuttle::fgkLocalRefStorage("local://LocalReferenceStorage");
 132
 133 Bool_t AliShuttle::fgkProcessDCS(kTRUE);
 134
 135
 136 const char* AliShuttle::fgkShuttleTempDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/temp");
 137 const char* AliShuttle::fgkShuttleLogDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/log");
 138
 139 const char* AliShuttle::fgkDetectorName[AliShuttle::fgkNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
 140         "PHOS", "CPV", "RICH", "EMCAL", "MUON_TRK", "MUON_TRG", "FMD", "ZDC", "PMD", "START", "VZERO"};
 141
 142 const char* AliShuttle::fgkDetectorCode[AliShuttle::fgkNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
 143         "PHS", "CPV", "HMP", "EMC", "MCH", "MTR", "FMD", "ZDC", "PMD", "T00", "V00"};
 144
 145 //______________________________________________________________________________________________
 146 AliShuttle::AliShuttle(const AliShuttleConfig* config,
 147                 UInt_t timeout, Int_t retries):
 148 fConfig(config),
 149 fTimeout(timeout), fRetries(retries),
 150 fPreprocessorMap(),
 151 fCurrentRun(-1),
 152 fCurrentStartTime(0), fCurrentEndTime(0),
 153 fCurrentDetector(""),
 154 fStatusEntry(0)
 155 {
 156         //
 157         // config: AliShuttleConfig used
 158         // timeout: timeout used for AliDCSClient connection
 159         // retries: the number of retries in case of connection error.
 160         //
 161
 162         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
 163         for(int iSys=0;iSys<3;iSys++) {
 164                 fServer[iSys]=0;
 165                 fFESlist[iSys].SetOwner(kTRUE);
 166         }
 167 }
 168
 169 //______________________________________________________________________
 170 AliShuttle::AliShuttle(const AliShuttle& /*other*/):
 171 AliShuttleInterface(),
 172 fConfig(0),
 173 fTimeout(0), fRetries(0),
 174 fPreprocessorMap(),
 175 fCurrentRun(-1),
 176 fCurrentStartTime(0), fCurrentEndTime(0),
 177 fCurrentDetector(""),
 178 fStatusEntry(0)
 179
 180 {
 181 // copy constructor (not implemented)
 182
 183 }
 184
 185 //______________________________________________________________________
 186 AliShuttle &AliShuttle::operator=(const AliShuttle& /*other*/)
 187 {
 188 // assignment operator (not implemented)
 189
 190 return *this;
 191 }
 192
 193 //______________________________________________________________________________________________
 194 AliShuttle::~AliShuttle()
 195 {
 196 // destructor
 197
 198         fPreprocessorMap.DeleteAll();
 199         for(int iSys=0;iSys<3;iSys++)
 200                 if(fServer[iSys]) {
 201                         fServer[iSys]->Close();
 202                         delete fServer[iSys];
 203                 }
 204 }
 205
 206 //______________________________________________________________________________________________
 207 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
 208 {
 209         //
 210         // Registers new AliPreprocessor.
 211         // It uses GetName() for indentificator of the pre processor.
 212         // The pre processor is registered it there isn't any other
 213         // with the same identificator (GetName()).
 214         //
 215
 216         if (fPreprocessorMap.GetValue(preprocessor->GetName())) {
 217                 AliWarning(Form("AliPreprocessor %s is already registered!",
 218                         preprocessor->GetName()));
 219                 return;
 220         }
 221
 222         fPreprocessorMap.Add(new TObjString(preprocessor->GetName()), preprocessor);
 223 }
 224
 225 //______________________________________________________________________________________________
 226 UInt_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
 227                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
 228 {
 229   // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
 230   // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
 231   // using this function. Use StoreReferenceData instead!
 232   //
 233
 234   // The parameters are
 235   //   1) the object's path.
 236   //   2) the object to be stored
 237   //   3) the metaData to be associated with the object
 238   //   4) the validity start run number w.r.t. the current run,
 239   //      if the data is valid only for this run leave the default 0
 240   //   5) specifies if the calibration data is valid for infinity (this means until updated),
 241   //      typical for calibration runs, the default is kFALSE
 242   //
 243   //
 244   // returns 0 if fail
 245   //         1 if stored in main (Grid) CDB storage
 246   //         2 if stored in backup (Local) CDB storage
 247
 248
 249         Int_t firstRun = GetCurrentRun() - validityStart;
 250         if(firstRun < 0) {
 251                 AliError("First valid run happens to be less than 0! Setting it to 0...");
 252                 firstRun=0;
 253         }
 254
 255         Int_t lastRun = -1;
 256         if(validityInfinite) {
 257                 lastRun = AliCDBRunRange::Infinity();
 258         } else {
 259                 lastRun = GetCurrentRun();
 260         }
 261
 262         AliCDBId id(path, firstRun, lastRun);
 263
 264         UInt_t result = 0;
 265
 266         if (!(AliCDBManager::Instance()->GetStorage(fgkMainCDB))) {
 267                 Log(fCurrentDetector, "Cannot activate main CDB storage!");
 268         } else {
 269                 result = (UInt_t) AliCDBManager::Instance()->GetStorage(fgkMainCDB)
 270                                         ->Put(object, id, metaData);
 271         }
 272
 273         if(!result) {
 274
 275                 Log(fCurrentDetector,
 276                         "Error while storing object in main storage: it will go to local storage!");
 277
 278                 result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)
 279                                         ->Put(object, id, metaData);
 280
 281                 if(result) {
 282                         result = 2;
 283                 }else{
 284                         Log(fCurrentDetector, "Can't store data!");
 285                 }
 286         }
 287         return result;
 288
 289 }
 290
 291 //______________________________________________________________________________________________
 292 UInt_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object,
 293                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
 294 {
 295   // Stores a CDB object in the storage for reference data. This objects will not be available during
 296   // offline reconstrunction. Use this function for reference data only!
 297   //
 298
 299   // The parameters are
 300   //   1) the object's path.
 301   //   2) the object to be stored
 302   //   3) the metaData to be associated with the object
 303   //   4) the validity start run number w.r.t. the current run,
 304   //      if the data is valid only for this run leave the default 0
 305   //   5) specifies if the calibration data is valid for infinity (this means until updated),
 306   //      typical for calibration runs, the default is kFALSE
 307   //
 308   //
 309   // returns 0 if fail
 310   //         1 if stored in main (Grid) reference storage
 311   //         2 if stored in backup (Local) reference storage
 312
 313         Int_t firstRun = GetCurrentRun() - validityStart;
 314         if(firstRun < 0) {
 315                 AliError("First valid run happens to be less than 0! Setting it to 0...");
 316                 firstRun=0;
 317         }
 318
 319         Int_t lastRun = -1;
 320         if(validityInfinite) {
 321                 lastRun = AliCDBRunRange::Infinity();
 322         } else {
 323                 lastRun = GetCurrentRun();
 324         }
 325
 326         AliCDBId id(path, firstRun, lastRun);
 327
 328         UInt_t result = 0;
 329
 330         if (!(AliCDBManager::Instance()->GetStorage(fgkMainRefStorage))) {
 331                 Log(fCurrentDetector, "Cannot activate main reference storage!");
 332         } else {
 333                 result = (UInt_t) AliCDBManager::Instance()->GetStorage(fgkMainRefStorage)
 334                                         ->Put(object, id, metaData);
 335         }
 336
 337         if(!result) {
 338
 339                 Log(fCurrentDetector,
 340                         "Error while storing object in main reference storage: it will go to local ref storage!");
 341
 342                 result = AliCDBManager::Instance()->GetStorage(fgkLocalRefStorage)
 343                                         ->Put(object, id, metaData);
 344
 345                 if(result) {
 346                         result = 2;
 347                 }else{
 348                         Log(fCurrentDetector, "Can't store reference data!");
 349                 }
 350         }
 351         return result;
 352
 353 }
 354
 355 //______________________________________________________________________________________________
 356 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
 357 {
 358   // Reads the AliShuttleStatus from the CDB
 359
 360   if (fStatusEntry)
 361   {
 362     delete fStatusEntry;
 363     fStatusEntry = 0;
 364   }
 365
 366   fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalCDB())
 367       ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), fCurrentRun);
 368
 369   if (!fStatusEntry)
 370     return 0;
 371
 372   TObject* anObject = fStatusEntry->GetObject();
 373   if (anObject == NULL || anObject->IsA() != AliShuttleStatus::Class())
 374   {
 375     AliError("Invalid object stored to CDB!");
 376     return 0;
 377   }
 378
 379   AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (anObject);
 380   return status;
 381 }
 382
 383 //______________________________________________________________________________________________
 384 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
 385 {
 386   // writes the status for one subdetector
 387
 388   if (fStatusEntry)
 389   {
 390     delete fStatusEntry;
 391     fStatusEntry = 0;
 392   }
 393
 394   AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), fCurrentRun, fCurrentRun);
 395
 396   fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
 397
 398   UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
 399
 400   if (!result)
 401   {
 402     AliError(Form("WriteShuttleStatus for %s, run %d failed", fCurrentDetector.Data(), fCurrentRun));
 403     return kFALSE;
 404   }
 405
 406   return kTRUE;
 407 }
 408
 409 //______________________________________________________________________________________________
 410 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
 411 {
 412   // changes the AliShuttleStatus for the given detector and run to the given status
 413
 414   if (!fStatusEntry)
 415   {
 416     AliError("UNEXPECTED: fStatusEntry empty");
 417     return;
 418   }
 419
 420   TObject* anObject = fStatusEntry->GetObject();
 421   AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (anObject);
 422
 423   if (!status)
 424   {
 425     AliError("UNEXPECTED: status could not be read from current CDB entry");
 426     return;
 427   }
 428
 429   Log("SHUTTLE", Form("%s: Changing state from %s to %s", fCurrentDetector.Data(),
 430                                 status->GetStatusName(), status->GetStatusName(newStatus)));
 431
 432   status->SetStatus(newStatus);
 433   if (increaseCount)
 434     status->IncreaseCount();
 435
 436   AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
 437 }
 438
 439 //______________________________________________________________________________________________
 440 Bool_t AliShuttle::ContinueProcessing()
 441 {
 442   // this function reads the AliShuttleStatus information from CDB and
 443   // checks if the processing should be continued
 444   // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
 445
 446   AliShuttleStatus* status = ReadShuttleStatus();
 447   if (!status)
 448   {
 449     // first time
 450
 451     Log("SHUTTLE", Form("%s: Processing first time.", fCurrentDetector.Data()));
 452     status = new AliShuttleStatus(AliShuttleStatus::kStarted);
 453     return WriteShuttleStatus(status);
 454   }
 455
 456   if (status->GetStatus() == AliShuttleStatus::kDone)
 457   {
 458     Log("SHUTTLE", Form("%s already done for run %d", fCurrentDetector.Data(), fCurrentRun));
 459     return kFALSE;
 460   }
 461
 462   if (status->GetStatus() == AliShuttleStatus::kFailed)
 463   {
 464     Log("SHUTTLE", Form("%s already in failed state for run %d", fCurrentDetector.Data(), fCurrentRun));
 465     return kFALSE;
 466   }
 467
 468   // if we get here, there is a restart
 469
 470   // abort conditions
 471   if (status->GetStatus() == AliShuttleStatus::kPPStarted && status->GetCount() >= fConfig->GetMaxPPRetries() ||
 472       status->GetCount() >= fConfig->GetMaxRetries())
 473   {
 474     Log("SHUTTLE", Form("%s, run %d failed too often, %d times, status %s. Skipping processing.",
 475                 fCurrentDetector.Data(), fCurrentRun, status->GetCount(), status->GetStatusName()));
 476
 477     return kFALSE;
 478   }
 479
 480   Log("SHUTTLE", Form("Restart of %s, run %d. Got stuck before in %s, count %d",
 481                 fCurrentDetector.Data(), fCurrentRun, status->GetStatusName(), status->GetCount()));
 482
 483   UpdateShuttleStatus(AliShuttleStatus::kStarted, kTRUE);
 484
 485   return kTRUE;
 486 }
 487
 488 //______________________________________________________________________________________________
 489 Bool_t AliShuttle::Process(Int_t run, UInt_t startTime, UInt_t endTime)
 490 {
 491         //
 492         // Makes data retrieval for all detectors in the configuration.
 493         // run: is the run number used
 494         // startTime: is the run start time
 495         // endTime: is the run end time
 496         // Returns kFALSE in case of error occured and kTRUE otherwise
 497         //
 498
 499         AliInfo(Form("\n\n ^*^*^*^*^*^* Processing run %d ^*^*^*^*^*^*", run));
 500
 501         // Initialization
 502         Bool_t hasError = kFALSE;
 503         for(Int_t iSys=0;iSys<3;iSys++) fFESCalled[iSys]=kFALSE;
 504
 505         fCurrentRun = run;
 506         fCurrentStartTime = startTime;
 507         fCurrentEndTime = endTime;
 508
 509         // Loop on detectors in the configuration
 510         TIter iter(fConfig->GetDetectors());
 511         TObjString* aDetector;
 512
 513         while ((aDetector = (TObjString*) iter.Next())) {
 514                 fCurrentDetector = aDetector->String();
 515
 516                 Bool_t detectorError=kFALSE;
 517                 if (!fConfig->HostProcessDetector(fCurrentDetector)) continue;
 518
 519                 if (ContinueProcessing() == kFALSE) continue;
 520
 521                 if(!Process()) {
 522                         hasError = kTRUE;
 523                         detectorError=kTRUE;
 524                         continue;
 525                 }
 526                 AliInfo(Form("Process ended successfully for detector %s!",aDetector->GetName()));
 527
 528                 // Process successful: Update time_processed field in FES logbooks!
 529                 if(fFESCalled[kDAQ]) {
 530                         hasError = (UpdateDAQTable() == kFALSE);
 531                         fFESlist[kDAQ].Clear();
 532                 }
 533                 //if(fFESCalled[kDCS]) {
 534                 //      hasError = UpdateDCSTable(aDetector->GetName());
 535                 //      fFESlist[kDCS].Clear();
 536                 //}
 537                 //if(fFESCalled[kHLT]) {
 538                 //      hasError = UpdateHLTTable(aDetector->GetName());
 539                 //      fFESlist[kHLT].Clear();
 540                 //}
 541
 542                 UpdateShuttleStatus(AliShuttleStatus::kDone);
 543         }
 544
 545         fCurrentRun = -1;
 546         fCurrentStartTime = 0;
 547         fCurrentEndTime = 0;
 548
 549         return hasError == kFALSE;
 550 }
 551
 552 //______________________________________________________________________________________________
 553 Bool_t AliShuttle::Process()
 554 {
 555         //
 556         // Makes data retrieval just for one specific detector.
 557         // Threre should be a configuration for this detector.
 558         // run: is the run number used
 559         // startTime: is the run start time
 560         // endTime: is the run end time
 561         // detector: detector for which the retrieval will be made
 562         // Returns kFALSE in case of error occured and kTRUE otherwise
 563         //
 564
 565         AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), fCurrentRun));
 566
 567         if (!fConfig->HasDetector(fCurrentDetector)) {
 568                 Log(fCurrentDetector, "There isn't any configuration for %s !");
 569                 UpdateShuttleStatus(AliShuttleStatus::kFailed);
 570                 return kFALSE;
 571         }
 572
 573         UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
 574
 575         TString host(fConfig->GetDCSHost(fCurrentDetector));
 576         Int_t port = fConfig->GetDCSPort(fCurrentDetector);
 577
 578         TIter iter(fConfig->GetDCSAliases(fCurrentDetector));
 579         TObjString* anAlias;
 580         TMap aliasMap;
 581
 582         Bool_t hasError = kFALSE;
 583         Bool_t result=kFALSE;
 584
 585         while ((anAlias = (TObjString*) iter.Next())) {
 586                 TObjArray valueSet;
 587                 // TODO Test only... I've added a flag that allows to
 588                 // exclude DCS archive DB query
 589                 if(fgkProcessDCS){
 590                         AliInfo("Querying DCS archive DB data...");
 591                         result = GetValueSet(host, port, anAlias->String(), valueSet);
 592                 } else {
 593                         AliInfo(Form("Skipping DCS processing. Port = %d",port));
 594                         result = kTRUE;
 595                 }
 596                 if(result) {
 597                         aliasMap.Add(anAlias->Clone(), valueSet.Clone());
 598                 }else{
 599                         TString message = Form("Error while retrieving alias %s !",
 600                                         anAlias->GetName());
 601                         Log(fCurrentDetector, message.Data());
 602                         hasError = kTRUE;
 603                         break;
 604                 }
 605         }
 606
 607         if (hasError)
 608         {
 609                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
 610                 return kFALSE;
 611         }
 612
 613   UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
 614
 615   AliPreprocessor* aPreprocessor =
 616                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
 617         if(aPreprocessor)
 618         {
 619                 aPreprocessor->Initialize(fCurrentRun, fCurrentStartTime, fCurrentEndTime);
 620
 621                 // TODO Think about what to do in case of "Grid storage error"
 622                 // (-> object put in local backup storage, return 2)
 623                 hasError = (aPreprocessor->Process(&aliasMap) == 0);
 624         }else{
 625     // TODO default behaviour?
 626                 AliInfo(Form("No Preprocessor for %s: storing TMap of DP arrays into CDB!", fCurrentDetector.Data()));
 627                 AliCDBMetaData metaData;
 628                 AliDCSValue dcsValue(fCurrentStartTime, fCurrentEndTime);
 629                 metaData.SetResponsible(Form("Duck, Donald"));
 630                 metaData.SetProperty("StartEndTime", &dcsValue);
 631                 metaData.SetComment("Automatically stored by Shuttle!");
 632                 AliCDBPath path(fCurrentDetector,"DCS","Data");
 633                 hasError = (Store(path, &aliasMap, &metaData) == 0);
 634         }
 635
 636   if (hasError)
 637     UpdateShuttleStatus(AliShuttleStatus::kPPError);
 638   else
 639     UpdateShuttleStatus(AliShuttleStatus::kPPDone);
 640
 641         aliasMap.Delete();
 642
 643         return hasError == kFALSE;
 644 }
 645
 646 //______________________________________________________________________________________________
 647 Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* alias,
 648                                 TObjArray& valueSet)
 649 {
 650 // Retrieve all "alias" data points from the DCS server
 651 // host, port: TSocket connection parameters
 652 // alias: name of the alias
 653 // valueSet: array of retrieved AliDCSValue's
 654
 655         AliDCSClient client(host, port, fTimeout, fRetries);
 656         if (!client.IsConnected()) {
 657                 return kFALSE;
 658         }
 659
 660         Int_t result = client.GetAliasValues(alias,
 661                 GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
 662
 663         if (result < 0) {
 664                 AliError(Form("Can't get '%s'! Reason: %s",
 665                         alias, AliDCSClient::GetErrorString(result)));
 666
 667                 if (result == AliDCSClient::fgkServerError) {
 668                         AliError(Form("Server error: %s",
 669                                 client.GetServerError().Data()));
 670                 }
 671
 672                 return kFALSE;
 673         }
 674
 675         return kTRUE;
 676 }
 677
 678 //______________________________________________________________________________________________
 679 const char* AliShuttle::GetFile(Int_t system, const char* detector,
 680                 const char* id, const char* source)
 681 {
 682 // Get calibration file from file exchange servers
 683 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
 684
 685         switch(system){
 686                 case kDAQ:
 687                         return GetDAQFileName(detector, id, source);
 688                         break;
 689                 case kDCS:
 690                         return GetDCSFileName(detector, id, source);
 691                         break;
 692                 case kHLT:
 693                         return GetHLTFileName(detector, id, source);
 694                         break;
 695                 default:
 696                         AliError(Form("No valid system index: %d",system));
 697         }
 698
 699         return 0;
 700 }
 701
 702 //______________________________________________________________________________________________
 703 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
 704 {
 705 // Get sources producing the condition file Id from file exchange servers
 706 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
 707
 708         switch(system){
 709                 case kDAQ:
 710                         return GetDAQFileSources(detector, id);
 711                         break;
 712                 case kDCS:
 713                         return GetDCSFileSources(detector, id);
 714                         break;
 715                 case kHLT:
 716                         return GetHLTFileSources(detector, id);
 717                         break;
 718                 default:
 719                         AliError(Form("No valid system index: %d",system));
 720         }
 721
 722         return NULL;
 723 }
 724
 725 //______________________________________________________________________________________________
 726 Bool_t AliShuttle::Connect(Int_t system){
 727 // Connect to MySQL Server of the system's FES logbook
 728
 729         // check connection: if already connected return
 730         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
 731
 732         TString aFESlbHost= Form("mysql://%s", fConfig->GetFESlbHost(system));
 733
 734         fServer[system] = TSQLServer::Connect(aFESlbHost,
 735                         fConfig->GetFESlbUser(system),
 736                         fConfig->GetFESlbPass(system));
 737         if (!fServer[system] || !fServer[system]->IsConnected()) {
 738                 AliError(Form("Can't establish connection to FES logbook for %s !",fkSystemNames[system]));
 739                 return kFALSE;
 740         }
 741
 742         // Get tables
 743         // TODO in the configuration should the table name be there too?
 744         switch(system){
 745                 case kDAQ:
 746                         fServer[kDAQ]->GetTables("REFSYSLOG");
 747                         break;
 748                 case kDCS:
 749                         //fServer[kDCS]->GetTables("REFSYSLOG");
 750                         break;
 751                 case kHLT:
 752                         //fServer[kHLT]->GetTables("REFSYSLOG");
 753                         break;
 754                 default:
 755                         break;
 756         }
 757
 758         return kTRUE;
 759 }
 760
 761 //______________________________________________________________________________________________
 762 const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source){
 763 // Retrieves a file from the DAQ FES.
 764 // First queris the DAQ logbook_fs for the DAQ file name, using the run, detector, id and source info
 765 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
 766 // run: current run being processed (fCurrentRun)
 767 // detector: comes from the Preprocessor name (must be converted into detector code with GetDetCode)
 768 // id: provided as a parameter by the Preprocessor
 769 // source: provided by the Preprocessor through GetFileSources function
 770
 771         // check connection, in case connect
 772         if(!Connect(kDAQ)){
 773                 Log(detector, "GetDAQFileName: Couldn't connect to DAQ Logbook !");
 774                 return 0;
 775         }
 776
 777         // Query preparation
 778         TString sqlQueryStart = "select filePath from logbook_fs where";
 779         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
 780                                 fCurrentRun, GetDetCode(detector), id, source);
 781         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
 782
 783         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
 784
 785         // Query execution
 786         TSQLResult* aResult;
 787         aResult = fServer[kDAQ]->Query(sqlQuery);
 788         if (!aResult) {
 789                 Log(detector, Form("Can't execute query <%s>!", sqlQuery.Data()));
 790                 return 0;
 791         }
 792
 793         if (aResult->GetRowCount() == 0) {
 794                 Log(detector,
 795                         Form("GetDAQFileName: No result from SQL query <%s>!", sqlQuery.Data()));
 796                 delete aResult;
 797                 return 0;
 798         }
 799
 800         if (aResult->GetRowCount() >1) {
 801                 Log(detector,
 802                         Form("GetDAQFileName: More than one row resulting from SQL query <%s>!", sqlQuery.Data()));
 803                 delete aResult;
 804                 return 0;
 805         }
 806
 807         TSQLRow* aRow = aResult->Next();
 808
 809         if(!aRow){
 810                 Log(detector, Form("GetDAQFileName: Empty set result from query <%s>!", sqlQuery.Data()));
 811                 delete aResult;
 812                 return 0;
 813         }
 814
 815         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
 816
 817         delete aResult;
 818
 819         AliDebug(2, Form("filePath = %s",filePath.Data()));
 820
 821         // retrieved file is renamed to make it unique
 822         TString localFileName = Form("%s_%d_%s_%s.shuttle",
 823                                         detector, fCurrentRun, id, source);
 824
 825         // file retrieval from DAQ FES
 826         Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
 827         if(!result) {
 828                 Log(detector, Form("copying file %s from DAQ FES failed!", filePath.Data()));
 829                 return 0;
 830         } else {
 831                 AliInfo(Form("File %s copied from DAQ FES into %s/%s !",
 832                         filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
 833         }
 834
 835
 836         fFESCalled[kDAQ]=kTRUE;
 837         TObjString *fileParams = new TObjString(Form("%s_!?!_%s", id, source));
 838         fFESlist[kDAQ].Add(fileParams);
 839
 840         return localFileName.Data();
 841
 842 }
 843
 844 //______________________________________________________________________________________________
 845 Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName){
 846
 847         // check temp directory: trying to cd to temp; if it does not exist, create it
 848         AliDebug(2, Form("Copy file %s from DAQ FES into folder %s and rename it as %s",
 849                         daqFileName,fgkShuttleTempDir, localFileName));
 850
 851         void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
 852         if (dir == NULL) {
 853                 if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
 854                         AliError(Form("Can't open directory <%s>!", fgkShuttleTempDir));
 855                         return kFALSE;
 856                 }
 857
 858         } else {
 859                 gSystem->FreeDirectory(dir);
 860         }
 861
 862         TString baseDAQFESFolder = "DAQ";
 863         TString command = Form("scp %s@%s:%s/%s %s/%s",
 864                 fConfig->GetFESUser(kDAQ),
 865                 fConfig->GetFESHost(kDAQ),
 866                 baseDAQFESFolder.Data(),
 867                 daqFileName,
 868                 fgkShuttleTempDir,
 869                 localFileName);
 870
 871         AliDebug(2, Form("%s",command.Data()));
 872
 873         UInt_t nRetries = 0;
 874         UInt_t maxRetries = 3;
 875
 876         // copy!! if successful TSystem::Exec returns 0
 877         while(nRetries++ < maxRetries) {
 878                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
 879                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
 880         }
 881
 882         return kFALSE;
 883
 884 }
 885
 886 //______________________________________________________________________________________________
 887 TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id){
 888 // Retrieves a file from the DCS FES.
 889
 890         // check connection, in case connect
 891         if(!Connect(kDAQ)){
 892                 Log(detector, "GetDAQFileName: Couldn't connect to DAQ Logbook !");
 893                 return 0;
 894         }
 895
 896         // Query preparation
 897         TString sqlQueryStart = "select DAQsource from logbook_fs where";
 898         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
 899                                 fCurrentRun, GetDetCode(detector), id);
 900         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
 901
 902         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
 903
 904         // Query execution
 905         TSQLResult* aResult;
 906         aResult = fServer[kDAQ]->Query(sqlQuery);
 907         if (!aResult) {
 908                 Log(detector, Form("GetDAQFileSources: Can't execute query <%s>!", sqlQuery.Data()));
 909                 return 0;
 910         }
 911
 912         if (aResult->GetRowCount() == 0) {
 913                 Log(detector,
 914                         Form("GetDAQFileSources: No result from SQL query <%s>!", sqlQuery.Data()));
 915                 delete aResult;
 916                 return 0;
 917         }
 918
 919         TSQLRow* aRow;
 920         TList *list = new TList();
 921         list->SetOwner(1);
 922
 923         while((aRow = aResult->Next())){
 924
 925                 TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
 926                 AliDebug(2, Form("daqSource = %s", daqSource.Data()));
 927                 list->Add(new TObjString(daqSource));
 928         }
 929         delete aResult;
 930
 931         return list;
 932
 933 }
 934
 935 //______________________________________________________________________________________________
 936 Bool_t AliShuttle::UpdateDAQTable(){
 937 // Update DAQ table filling time_processed field in all rows corresponding to current run and detector
 938
 939         // check connection, in case connect
 940         if(!Connect(kDAQ)){
 941                 Log(fCurrentDetector, "UpdateDAQTable: Couldn't connect to DAQ Logbook !");
 942                 return kFALSE;
 943         }
 944
 945         TTimeStamp now; // now
 946
 947         // Loop on FES list entries
 948         TIter iter(&fFESlist[kDAQ]);
 949         TObjString *aFESentry=0;
 950         while((aFESentry = dynamic_cast<TObjString*> (iter.Next()))){
 951                 TString aFESentrystr = aFESentry->String();
 952                 TObjArray *aFESarray = aFESentrystr.Tokenize("_!?!_");
 953                 if(!aFESarray || aFESarray->GetEntries() != 2 ) {
 954                         Log(fCurrentDetector,Form("UpdateDAQTable: error updating FES entry! string = %s",
 955                                 aFESentrystr.Data()));
 956                         if(aFESarray) delete aFESarray;
 957                         return kFALSE;
 958                 }
 959                 const char* fileId = ((TObjString*) aFESarray->At(0))->GetName();
 960                 const char* daqSource = ((TObjString*) aFESarray->At(1))->GetName();
 961                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
 962                         fCurrentRun,GetDetCode(fCurrentDetector), fileId, daqSource);
 963
 964                 delete aFESarray;
 965
 966                 TString sqlQuery = Form("update logbook_fs set time_processed=%d %s", now.GetSec(), whereClause.Data());
 967
 968                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
 969
 970                 // Query execution
 971                 TSQLResult* aResult;
 972                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
 973                 if (!aResult) {
 974                         Log(fCurrentDetector, Form("UpdateDAQTable: Can't execute query <%s>!", sqlQuery.Data()));
 975                         return kFALSE;
 976                 }
 977                 delete aResult;
 978
 979                 // check result - TODO Is it necessary?
 980                 sqlQuery = Form("select time_processed from logbook_fs %s", whereClause.Data());
 981                 AliDebug(2, Form(" CHECK - SQL query: \n%s",sqlQuery.Data()));
 982
 983                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
 984                 if (!aResult) {
 985                         AliWarning("Can't check result!");
 986                         continue;
 987                 }
 988
 989         if (aResult->GetRowCount() == 0) {
 990                 Log(fCurrentDetector,
 991                         Form("GetDAQFileName: No result from SQL query <%s>!", sqlQuery.Data()));
 992                 delete aResult;
 993                 //return 0;
 994         }
 995
 996         if (aResult->GetRowCount() >1) {
 997                 Log(fCurrentDetector,
 998                         Form("GetDAQFileName: More than one row resulting from SQL query <%s>!", sqlQuery.Data()));
 999                 delete aResult;
1000                 //return 0;
1001         }
1002
1003                 TSQLRow *row = dynamic_cast<TSQLRow*> (aResult->Next());
1004                 TString processedTimeString(row->GetField(0), row->GetFieldLength(0));
1005                 Int_t processedTime = processedTimeString.Atoi();
1006                 if(processedTime != now.GetSec()){
1007                         Log(fCurrentDetector, Form("UpdateDAQTable: Update table error: processed_time=%d, now=%d !",
1008                                 processedTime, now.GetSec()));
1009                         delete aResult;
1010                         return kFALSE;
1011                 }
1012
1013                 delete aResult;
1014
1015         }
1016
1017         return kTRUE;
1018 }
1019
1020 //______________________________________________________________________________________________
1021 const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1022 // Retrieves a file from the DCS FES.
1023
1024 return "You're in DCS";
1025
1026 }
1027
1028 //______________________________________________________________________________________________
1029 TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
1030 // Retrieves a file from the DCS FES.
1031
1032 return NULL;
1033
1034 }
1035
1036 //______________________________________________________________________________________________
1037 const char* AliShuttle::GetHLTFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1038 // Retrieves a file from the HLT FES.
1039
1040 return "You're in HLT";
1041
1042 }
1043
1044 //______________________________________________________________________________________________
1045 TList* AliShuttle::GetHLTFileSources(const char* /*detector*/, const char* /*id*/){
1046 // Retrieves a file from the HLT FES.
1047
1048 return NULL;
1049
1050 }
1051
1052 //______________________________________________________________________________________________
1053 const char* AliShuttle::GetDetCode(const char* detector){
1054 // Return detector code
1055
1056         for(int iDet=0; iDet < fgkNDetectors; iDet++){
1057                 if(!strcmp(fgkDetectorName[iDet], detector)) return fgkDetectorCode[iDet];
1058         }
1059
1060         return 0;
1061 }
1062
1063 //______________________________________________________________________________________________
1064 void AliShuttle::Log(const char* detector, const char* message)
1065 {
1066 // Fill log string with a message
1067
1068         void* dir = gSystem->OpenDirectory(fgkShuttleLogDir);
1069         if (dir == NULL) {
1070                 if (gSystem->mkdir(fgkShuttleLogDir, kTRUE)) {
1071                         AliError(Form("Can't open directory <%s>!", fgkShuttleTempDir));
1072                         return;
1073                 }
1074
1075         } else {
1076                 gSystem->FreeDirectory(dir);
1077         }
1078
1079         TString toLog = Form("%s: %s, run %d - %s", TTimeStamp(time(0)).AsString(),
1080         detector, GetCurrentRun(), message);
1081         AliInfo(toLog.Data());
1082
1083         TString fileName;
1084         fileName.Form("%s/%s.log", fgkShuttleLogDir, detector);
1085         gSystem->ExpandPathName(fileName);
1086
1087         ofstream logFile;
1088         logFile.open(fileName, ofstream::out | ofstream::app);
1089
1090         if (!logFile.is_open()) {
1091                 AliError(Form("Could not open file %s", fileName.Data()));
1092                 return;
1093         }
1094
1095         logFile << toLog.Data() << "\n";
1096
1097         logFile.close();
1098 }