SHUTTLE/AliShuttle.cxx

   1 /**************************************************************************
   2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
   3  *                                                                        *
   4  * Author: The ALICE Off-line Project.                                    *
   5  * Contributors are mentioned in the code where appropriate.              *
   6  *                                                                        *
   7  * Permission to use, copy, modify and distribute this software and its   *
   8  * documentation strictly for non-commercial purposes is hereby granted   *
   9  * without fee, provided that the above copyright notice appears in all   *
  10  * copies and that both the copyright notice and this permission notice   *
  11  * appear in the supporting documentation. The authors make no claims     *
  12  * about the suitability of this software for any purpose. It is          *
  13  * provided "as is" without express or implied warranty.                  *
  14  **************************************************************************/
  15
  16 /*
  17 $Log$
  18 Revision 1.21  2006/12/07 08:51:26  jgrosseo
  19 update (alberto):
  20 table, db names in ldap configuration
  21 added GRP preprocessor
  22 DCS data can also be retrieved by data point
  23
  24 Revision 1.20  2006/11/16 16:16:48  jgrosseo
  25 introducing strict run ordering flag
  26 removed giving preprocessor name to preprocessor, they have to know their name themselves ;-)
  27
  28 Revision 1.19  2006/11/06 14:23:04  jgrosseo
  29 major update (Alberto)
  30 o) reading of run parameters from the logbook
  31 o) online offline naming conversion
  32 o) standalone DCSclient package
  33
  34 Revision 1.18  2006/10/20 15:22:59  jgrosseo
  35 o) Adding time out to the execution of the preprocessors: The Shuttle forks and the parent process monitors the child
  36 o) Merging Collect, CollectAll, CollectNew function
  37 o) Removing implementation of empty copy constructors (declaration still there!)
  38
  39 Revision 1.17  2006/10/05 16:20:55  jgrosseo
  40 adapting to new CDB classes
  41
  42 Revision 1.16  2006/10/05 15:46:26  jgrosseo
  43 applying to the new interface
  44
  45 Revision 1.15  2006/10/02 16:38:39  jgrosseo
  46 update (alberto):
  47 fixed memory leaks
  48 storing of objects that failed to be stored to the grid before
  49 interfacing of shuttle status table in daq system
  50
  51 Revision 1.14  2006/08/29 09:16:05  jgrosseo
  52 small update
  53
  54 Revision 1.13  2006/08/15 10:50:00  jgrosseo
  55 effc++ corrections (alberto)
  56
  57 Revision 1.12  2006/08/08 14:19:29  jgrosseo
  58 Update to shuttle classes (Alberto)
  59
  60 - Possibility to set the full object's path in the Preprocessor's and
  61 Shuttle's  Store functions
  62 - Possibility to extend the object's run validity in the same classes
  63 ("startValidity" and "validityInfinite" parameters)
  64 - Implementation of the StoreReferenceData function to store reference
  65 data in a dedicated CDB storage.
  66
  67 Revision 1.11  2006/07/21 07:37:20  jgrosseo
  68 last run is stored after each run
  69
  70 Revision 1.10  2006/07/20 09:54:40  jgrosseo
  71 introducing status management: The processing per subdetector is divided into several steps,
  72 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
  73 can keep track of the number of failures and skips further processing after a certain threshold is
  74 exceeded. These thresholds can be configured in LDAP.
  75
  76 Revision 1.9  2006/07/19 10:09:55  jgrosseo
  77 new configuration, accesst to DAQ FES (Alberto)
  78
  79 Revision 1.8  2006/07/11 12:44:36  jgrosseo
  80 adding parameters for extended validity range of data produced by preprocessor
  81
  82 Revision 1.7  2006/07/10 14:37:09  jgrosseo
  83 small fix + todo comment
  84
  85 Revision 1.6  2006/07/10 13:01:41  jgrosseo
  86 enhanced storing of last sucessfully processed run (alberto)
  87
  88 Revision 1.5  2006/07/04 14:59:57  jgrosseo
  89 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
  90
  91 Revision 1.4  2006/06/12 09:11:16  jgrosseo
  92 coding conventions (Alberto)
  93
  94 Revision 1.3  2006/06/06 14:26:40  jgrosseo
  95 o) removed files that were moved to STEER
  96 o) shuttle updated to follow the new interface (Alberto)
  97
  98 Revision 1.2  2006/03/07 07:52:34  hristov
  99 New version (B.Yordanov)
 100
 101 Revision 1.6  2005/11/19 17:19:14  byordano
 102 RetrieveDATEEntries and RetrieveConditionsData added
 103
 104 Revision 1.5  2005/11/19 11:09:27  byordano
 105 AliShuttle declaration added
 106
 107 Revision 1.4  2005/11/17 17:47:34  byordano
 108 TList changed to TObjArray
 109
 110 Revision 1.3  2005/11/17 14:43:23  byordano
 111 import to local CVS
 112
 113 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
 114 Initial import as subdirectory in AliRoot
 115
 116 Revision 1.2  2005/09/13 08:41:15  byordano
 117 default startTime endTime added
 118
 119 Revision 1.4  2005/08/30 09:13:02  byordano
 120 some docs added
 121
 122 Revision 1.3  2005/08/29 21:15:47  byordano
 123 some docs added
 124
 125 */
 126
 127 //
 128 // This class is the main manager for AliShuttle.
 129 // It organizes the data retrieval from DCS and call the
 130 // interface methods of AliPreprocessor.
 131 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
 132 // data for its set of aliases is retrieved. If there is registered
 133 // AliPreprocessor for this detector then it will be used
 134 // accroding to the schema (see AliPreprocessor).
 135 // If there isn't registered AliPreprocessor than the retrieved
 136 // data is stored automatically to the undelying AliCDBStorage.
 137 // For detSpec is used the alias name.
 138 //
 139
 140 #include "AliShuttle.h"
 141
 142 #include "AliCDBManager.h"
 143 #include "AliCDBStorage.h"
 144 #include "AliCDBId.h"
 145 #include "AliCDBRunRange.h"
 146 #include "AliCDBPath.h"
 147 #include "AliCDBEntry.h"
 148 #include "AliShuttleConfig.h"
 149 #include "DCSClient/AliDCSClient.h"
 150 #include "AliLog.h"
 151 #include "AliPreprocessor.h"
 152 #include "AliShuttleStatus.h"
 153 #include "AliShuttleLogbookEntry.h"
 154
 155 #include <TSystem.h>
 156 #include <TObject.h>
 157 #include <TString.h>
 158 #include <TTimeStamp.h>
 159 #include <TObjString.h>
 160 #include <TSQLServer.h>
 161 #include <TSQLResult.h>
 162 #include <TSQLRow.h>
 163 #include <TMutex.h>
 164
 165 #include <fstream>
 166
 167 #include <sys/types.h>
 168 #include <sys/wait.h>
 169
 170 ClassImp(AliShuttle)
 171
 172 TString AliShuttle::fgkMainCDB("alien://folder=ShuttleCDB");
 173 TString AliShuttle::fgkLocalCDB("local://LocalShuttleCDB");
 174 TString AliShuttle::fgkMainRefStorage("alien://folder=ShuttleReference");
 175 TString AliShuttle::fgkLocalRefStorage("local://LocalReferenceStorage");
 176
 177 Bool_t AliShuttle::fgkProcessDCS(kTRUE);
 178
 179 const char* AliShuttle::fgkShuttleTempDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/temp");
 180 const char* AliShuttle::fgkShuttleLogDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/log");
 181
 182 //______________________________________________________________________________________________
 183 AliShuttle::AliShuttle(const AliShuttleConfig* config,
 184                 UInt_t timeout, Int_t retries):
 185 fConfig(config),
 186 fTimeout(timeout), fRetries(retries),
 187 fPreprocessorMap(),
 188 fLogbookEntry(0),
 189 fCurrentDetector(),
 190 fStatusEntry(0),
 191 fGridError(kFALSE),
 192 fMonitoringMutex(0),
 193 fLastActionTime(0),
 194 fLastAction()
 195 {
 196         //
 197         // config: AliShuttleConfig used
 198         // timeout: timeout used for AliDCSClient connection
 199         // retries: the number of retries in case of connection error.
 200         //
 201
 202         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
 203         for(int iSys=0;iSys<4;iSys++) {
 204                 fServer[iSys]=0;
 205                 if (iSys < 3)
 206                         fFXSlist[iSys].SetOwner(kTRUE);
 207         }
 208         fPreprocessorMap.SetOwner(kTRUE);
 209
 210         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
 211                 fFirstUnprocessed[iDet] = kFALSE;
 212
 213         fMonitoringMutex = new TMutex();
 214 }
 215
 216 //______________________________________________________________________________________________
 217 AliShuttle::~AliShuttle()
 218 {
 219 // destructor
 220
 221         fPreprocessorMap.DeleteAll();
 222         for(int iSys=0;iSys<4;iSys++)
 223                 if(fServer[iSys]) {
 224                         fServer[iSys]->Close();
 225                         delete fServer[iSys];
 226                         fServer[iSys] = 0;
 227                 }
 228
 229         if (fStatusEntry){
 230                 delete fStatusEntry;
 231                 fStatusEntry = 0;
 232         }
 233
 234         if (fMonitoringMutex)
 235         {
 236                 delete fMonitoringMutex;
 237                 fMonitoringMutex = 0;
 238         }
 239 }
 240
 241 //______________________________________________________________________________________________
 242 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
 243 {
 244         //
 245         // Registers new AliPreprocessor.
 246         // It uses GetName() for indentificator of the pre processor.
 247         // The pre processor is registered it there isn't any other
 248         // with the same identificator (GetName()).
 249         //
 250
 251         const char* detName = preprocessor->GetName();
 252         if(GetDetPos(detName) < 0)
 253                 AliFatal(Form("********** !!!!! Invalid detector name: %s !!!!! **********", detName));
 254
 255         if (fPreprocessorMap.GetValue(detName)) {
 256                 AliWarning(Form("AliPreprocessor %s is already registered!", detName));
 257                 return;
 258         }
 259
 260         fPreprocessorMap.Add(new TObjString(detName), preprocessor);
 261 }
 262 //______________________________________________________________________________________________
 263 UInt_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
 264                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
 265 {
 266   // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
 267   // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
 268   // using this function. Use StoreReferenceData instead!
 269   // It calls WriteToCDB function which perform actual storage
 270
 271         return WriteToCDB(fgkMainCDB, fgkLocalCDB, path, object,
 272                                 metaData, validityStart, validityInfinite);
 273
 274 }
 275
 276 //______________________________________________________________________________________________
 277 UInt_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData)
 278 {
 279   // Stores a CDB object in the storage for reference data. This objects will not be available during
 280   // offline reconstrunction. Use this function for reference data only!
 281   // It calls WriteToCDB function which perform actual storage
 282
 283         return WriteToCDB(fgkMainRefStorage, fgkLocalRefStorage, path, object, metaData);
 284
 285 }
 286
 287 //______________________________________________________________________________________________
 288 UInt_t AliShuttle::WriteToCDB(const char* mainUri, const char* localUri,
 289                         const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
 290                         Int_t validityStart, Bool_t validityInfinite)
 291 {
 292   // write object into the CDB. Parameters are passed by Store and StoreReferenceData functions.
 293   // The parameters are:
 294   //   1) Uri of the main storage (Grid)
 295   //   2) Uri of the backup storage (Local)
 296   //   3) the object's path.
 297   //   4) the object to be stored
 298   //   5) the metaData to be associated with the object
 299   //   6) the validity start run number w.r.t. the current run,
 300   //      if the data is valid only for this run leave the default 0
 301   //   7) specifies if the calibration data is valid for infinity (this means until updated),
 302   //      typical for calibration runs, the default is kFALSE
 303   //
 304   // returns 0 if fail
 305   //         1 if stored in main (Grid) storage
 306   //         2 if stored in backup (Local) storage
 307
 308         const char* cdbType = (mainUri == fgkMainCDB) ? "CDB" : "Reference";
 309
 310         Int_t firstRun = GetCurrentRun() - validityStart;
 311         if(firstRun < 0) {
 312                 AliError("First valid run happens to be less than 0! Setting it to 0.");
 313                 firstRun=0;
 314         }
 315
 316         Int_t lastRun = -1;
 317         if(validityInfinite) {
 318                 lastRun = AliCDBRunRange::Infinity();
 319         } else {
 320                 lastRun = GetCurrentRun();
 321         }
 322
 323         AliCDBId id(path, firstRun, lastRun, -1, -1);
 324
 325         if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
 326                 TObjString runUsed = Form("%d", GetCurrentRun());
 327                 metaData->SetProperty("RunUsed(TObjString)", runUsed.Clone());
 328         }
 329
 330         UInt_t result = 0;
 331
 332         if (!(AliCDBManager::Instance()->GetStorage(mainUri))) {
 333                 AliError(Form("WriteToCDB - Cannot activate main %s storage", cdbType));
 334         } else {
 335                 result = (UInt_t) AliCDBManager::Instance()->GetStorage(mainUri)
 336                                         ->Put(object, id, metaData);
 337         }
 338
 339         if(!result) {
 340
 341                 Log(fCurrentDetector,
 342                         Form("WriteToCDB - Problem with main %s storage. Putting <%s> into backup storage",
 343                                 cdbType, path.GetPath().Data()));
 344
 345                 // Set Grid version to current run number, to ease retrieval later
 346                 id.SetVersion(GetCurrentRun());
 347
 348                 result = AliCDBManager::Instance()->GetStorage(localUri)
 349                                         ->Put(object, id, metaData);
 350
 351                 if(result) {
 352                         result = 2;
 353                         fGridError = kTRUE;
 354                 }else{
 355                         Log(fCurrentDetector, "WriteToCDB - Can't store data!");
 356                 }
 357         }
 358
 359         return result;
 360
 361 }
 362
 363 //______________________________________________________________________________________________
 364 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
 365 {
 366 // Reads the AliShuttleStatus from the CDB
 367
 368         if (fStatusEntry){
 369                 delete fStatusEntry;
 370                 fStatusEntry = 0;
 371         }
 372
 373         fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalCDB())
 374                 ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), GetCurrentRun());
 375
 376         if (!fStatusEntry) return 0;
 377         fStatusEntry->SetOwner(1);
 378
 379         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
 380         if (!status) {
 381                 AliError("Invalid object stored to CDB!");
 382                 return 0;
 383         }
 384
 385         return status;
 386 }
 387
 388 //______________________________________________________________________________________________
 389 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
 390 {
 391 // writes the status for one subdetector
 392
 393         if (fStatusEntry){
 394                 delete fStatusEntry;
 395                 fStatusEntry = 0;
 396         }
 397
 398         Int_t run = GetCurrentRun();
 399
 400         AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), run, run);
 401
 402         fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
 403         fStatusEntry->SetOwner(1);
 404
 405         UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
 406
 407         if (!result) {
 408                 AliError(Form("WriteShuttleStatus for %s, run %d failed", fCurrentDetector.Data(), run));
 409                 return kFALSE;
 410         }
 411
 412         return kTRUE;
 413 }
 414
 415 //______________________________________________________________________________________________
 416 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
 417 {
 418   // changes the AliShuttleStatus for the given detector and run to the given status
 419
 420         if (!fStatusEntry){
 421                 AliError("UNEXPECTED: fStatusEntry empty");
 422                 return;
 423         }
 424
 425         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
 426
 427         if (!status){
 428                 AliError("UNEXPECTED: status could not be read from current CDB entry");
 429                 return;
 430         }
 431
 432         TString actionStr = Form("UpdateShuttleStatus - %s: Changing state from %s to %s",
 433                                 fCurrentDetector.Data(),
 434                                 status->GetStatusName(),
 435                                 status->GetStatusName(newStatus));
 436         Log("SHUTTLE", actionStr);
 437         SetLastAction(actionStr);
 438
 439         status->SetStatus(newStatus);
 440         if (increaseCount) status->IncreaseCount();
 441
 442         AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
 443 }
 444 //______________________________________________________________________________________________
 445 Bool_t AliShuttle::ContinueProcessing()
 446 {
 447 // this function reads the AliShuttleStatus information from CDB and
 448 // checks if the processing should be continued
 449 // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
 450
 451         if (!fConfig->HostProcessDetector(fCurrentDetector)) return kFALSE;
 452
 453         AliPreprocessor* aPreprocessor =
 454                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
 455         if (!aPreprocessor)
 456         {
 457                 AliInfo(Form("%s: no preprocessor registered", fCurrentDetector.Data()));
 458                 return kFALSE;
 459         }
 460
 461         AliShuttleLogbookEntry::Status entryStatus =
 462                 fLogbookEntry->GetDetectorStatus(fCurrentDetector);
 463
 464         if(entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
 465                 AliInfo(Form("ContinueProcessing - %s is %s",
 466                                 fCurrentDetector.Data(),
 467                                 fLogbookEntry->GetDetectorStatusName(entryStatus)));
 468                 return kFALSE;
 469         }
 470
 471         // if we get here, according to Shuttle logbook subdetector is in UNPROCESSED state
 472
 473         // check if current run is first unprocessed run for current detector
 474         if (fConfig->StrictRunOrder(fCurrentDetector) &&
 475                 !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
 476         {
 477                 Log("SHUTTLE", Form("ContinueProcessing - %s requires strict run ordering but this is not the first unprocessed run!"));
 478                 return kFALSE;
 479         }
 480
 481         AliShuttleStatus* status = ReadShuttleStatus();
 482         if (!status) {
 483                 // first time
 484                 Log("SHUTTLE", Form("ContinueProcessing - %s: Processing first time",
 485                                 fCurrentDetector.Data()));
 486                 status = new AliShuttleStatus(AliShuttleStatus::kStarted);
 487                 return WriteShuttleStatus(status);
 488         }
 489
 490         // The following two cases shouldn't happen if Shuttle Logbook was correctly updated.
 491         // If it happens it may mean Logbook updating failed... let's do it now!
 492         if (status->GetStatus() == AliShuttleStatus::kDone ||
 493             status->GetStatus() == AliShuttleStatus::kFailed){
 494                 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s. Updating Shuttle Logbook",
 495                                         fCurrentDetector.Data(),
 496                                         status->GetStatusName(status->GetStatus())));
 497                 UpdateShuttleLogbook(fCurrentDetector.Data(),
 498                                         status->GetStatusName(status->GetStatus()));
 499                 return kFALSE;
 500         }
 501
 502         if (status->GetStatus() == AliShuttleStatus::kStoreFailed) {
 503                 Log("SHUTTLE",
 504                         Form("ContinueProcessing - %s: Grid storage of one or more objects failed. Trying again now",
 505                                 fCurrentDetector.Data()));
 506                 if(TryToStoreAgain()){
 507                         Log(fCurrentDetector.Data(), "ContinueProcessing - All objects successfully stored into OCDB");
 508                         UpdateShuttleStatus(AliShuttleStatus::kDone);
 509                         UpdateShuttleLogbook(fCurrentDetector.Data(), "DONE");
 510                 } else {
 511                         Log("SHUTTLE",
 512                                 Form("ContinueProcessing - %s: Grid storage failed again",
 513                                         fCurrentDetector.Data()));
 514                 }
 515                 return kFALSE;
 516         }
 517
 518         // if we get here, there is a restart
 519         Bool_t cont = kFALSE;
 520
 521         // abort conditions
 522         if (status->GetCount() >= fConfig->GetMaxRetries()) {
 523                 Log("SHUTTLE", Form("ContinueProcessing - %s failed %d times in status %s - "
 524                                 "Updating Shuttle Logbook", fCurrentDetector.Data(),
 525                                 status->GetCount(), status->GetStatusName()));
 526                 UpdateShuttleLogbook(fCurrentDetector.Data(), "FAILED");
 527         } else {
 528                 Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. "
 529                                 "Aborted before with %s. Retry number %d.", fCurrentDetector.Data(),
 530                                 status->GetStatusName(), status->GetCount()));
 531                 UpdateShuttleStatus(AliShuttleStatus::kStarted, kTRUE);
 532                 cont = kTRUE;
 533         }
 534
 535         // Send mail to detector expert!
 536         AliInfo(Form("Sending mail to %s expert...", fCurrentDetector.Data()));
 537         if (!SendMail())
 538                 Log("SHUTTLE", Form("ContinueProcessing - Could not send mail to %s expert",
 539                                 fCurrentDetector.Data()));
 540
 541         return cont;
 542 }
 543
 544 //______________________________________________________________________________________________
 545 Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
 546 {
 547         //
 548         // Makes data retrieval for all detectors in the configuration.
 549         // entry: Shuttle logbook entry, contains run paramenters and status of detectors
 550         // (Unprocessed, Inactive, Failed or Done).
 551         // Returns kFALSE in case of error occured and kTRUE otherwise
 552         //
 553
 554         if(!entry) return kFALSE;
 555
 556         fLogbookEntry = entry;
 557
 558         if(fLogbookEntry->IsDone()){
 559                 Log("SHUTTLE","Process - Shuttle is already DONE. Updating logbook");
 560                 UpdateShuttleLogbook("shuttle_done");
 561                 fLogbookEntry = 0;
 562                 return kTRUE;
 563         }
 564
 565
 566         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^* \n",
 567                                         GetCurrentRun()));
 568
 569         fLogbookEntry->Print("all");
 570
 571         // Initialization
 572         Bool_t hasError = kFALSE;
 573         for(Int_t iSys=0;iSys<3;iSys++) fFXSCalled[iSys]=kFALSE;
 574
 575         AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
 576         if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
 577         AliCDBStorage *mainRefSto = AliCDBManager::Instance()->GetStorage(fgkMainRefStorage);
 578         if(mainRefSto) mainRefSto->QueryCDB(GetCurrentRun());
 579
 580         // Loop on detectors in the configuration
 581         TIter iter(fConfig->GetDetectors());
 582         TObjString* aDetector = 0;
 583
 584         while ((aDetector = (TObjString*) iter.Next()))
 585         {
 586                 fCurrentDetector = aDetector->String();
 587
 588                 if (ContinueProcessing() == kFALSE) continue;
 589
 590                 AliInfo(Form("\n\n \t\t\t****** run %d - %s: START  ******",
 591                                                 GetCurrentRun(), aDetector->GetName()));
 592
 593
 594                 Int_t pid = fork();
 595
 596                 if (pid < 0)
 597                 {
 598                         Log("SHUTTLE", "ERROR: Forking failed");
 599                 }
 600                 else if (pid > 0)
 601                 {
 602                         // parent
 603                         AliInfo(Form("In parent process of %d - %s: Starting monitoring",
 604                                                         GetCurrentRun(), aDetector->GetName()));
 605
 606                         Long_t begin = time(0);
 607
 608                         int status; // to be used with waitpid, on purpose an int (not Int_t)!
 609                         while (waitpid(pid, &status, WNOHANG) == 0)
 610                         {
 611                                 Long_t expiredTime = time(0) - begin;
 612
 613                                 if (expiredTime > fConfig->GetPPTimeOut())
 614                                 {
 615                                         Log("SHUTTLE", Form("Process time out. Run time: %d seconds. Killing...",
 616                                                                 expiredTime));
 617
 618                                         kill(pid, 9);
 619
 620                                         hasError = kTRUE;
 621
 622                                         gSystem->Sleep(1000);
 623                                 }
 624                                 else
 625                                 {
 626                                         if (expiredTime % 60 == 0)
 627                                         Log("SHUTTLE", Form("Checked process. Run time: %d seconds.",
 628                                                                 expiredTime));
 629                                         gSystem->Sleep(1000);
 630                                 }
 631                         }
 632
 633                         AliInfo(Form("In parent process of %d - %s: Client has terminated.",
 634                                                                 GetCurrentRun(), aDetector->GetName()));
 635
 636                         if (WIFEXITED(status))
 637                         {
 638                                 Int_t returnCode = WEXITSTATUS(status);
 639
 640                                 Log("SHUTTLE", Form("The return code is %d", returnCode));
 641
 642                                 if (returnCode != 0)
 643                                 hasError = kTRUE;
 644                         }
 645                 }
 646                 else if (pid == 0)
 647                 {
 648                         // client
 649                         AliInfo(Form("In client process of %d - %s", GetCurrentRun(), aDetector->GetName()));
 650
 651                         UInt_t result = ProcessCurrentDetector();
 652
 653                         Int_t returnCode = 0; // will be set to 1 in case of an error
 654
 655                         if (!result)
 656                         {
 657                                 returnCode = 1;
 658                                 AliInfo(Form("\n \t\t\t****** run %d - %s: PREPROCESSOR ERROR ****** \n\n",
 659                                                         GetCurrentRun(), aDetector->GetName()));
 660                         }
 661                         else if (result == 2)
 662                         {
 663                                 AliInfo(Form("\n \t\t\t****** run %d - %s: STORAGE ERROR ****** \n\n",
 664                                                         GetCurrentRun(), aDetector->GetName()));
 665                         } else
 666                         {
 667                                 AliInfo(Form("\n \t\t\t****** run %d - %s: DONE ****** \n\n",
 668                                                         GetCurrentRun(), aDetector->GetName()));
 669                         }
 670
 671                         if (result > 0)
 672                         {
 673                                 // Process successful: Update time_processed field in FXS logbooks!
 674                                 if (fFXSCalled[kDAQ])
 675                                 {
 676                                         if (UpdateDAQTable() == kFALSE)
 677                                         returnCode = 1;
 678                                         fFXSlist[kDAQ].Clear();
 679                                 }
 680                                 //if(fFXSCalled[kDCS]) {
 681                                 //  if (UpdateDCSTable(aDetector->GetName()) == kFALSE)
 682                                 //    returnCode = 1;
 683                                 //  fFXSlist[kDCS].Clear();
 684                                 //}
 685                                 if (fFXSCalled[kHLT])
 686                                 {
 687                                         if (UpdateHLTTable() == kFALSE)
 688                                         returnCode = 1;
 689                                         fFXSlist[kHLT].Clear();
 690                                 }
 691                         }
 692
 693                         AliInfo(Form("Client process of %d - %s is exiting now with %d.",
 694                                                         GetCurrentRun(), aDetector->GetName(), returnCode));
 695
 696                         // the client exits here
 697                         gSystem->Exit(returnCode);
 698
 699                         AliError("We should never get here!!!");
 700                 }
 701         }
 702
 703         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^* \n",
 704                                                         GetCurrentRun()));
 705
 706         //check if shuttle is done for this run, if so update logbook
 707         TObjArray checkEntryArray;
 708         checkEntryArray.SetOwner(1);
 709         TString whereClause = Form("where run=%d", GetCurrentRun());
 710         if (!QueryShuttleLogbook(whereClause.Data(), checkEntryArray) || checkEntryArray.GetEntries() == 0) {
 711                 Log("SHUTTLE", Form("Process - Warning: Cannot check status of run %d on Shuttle logbook!",
 712                                                 GetCurrentRun()));
 713                 return hasError == kFALSE;
 714         }
 715
 716         AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
 717                                                 (checkEntryArray.At(0));
 718
 719         if (checkEntry)
 720         {
 721                 if (checkEntry->IsDone())
 722                 {
 723                         Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
 724                         UpdateShuttleLogbook("shuttle_done");
 725                 }
 726                 else
 727                 {
 728                         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
 729                         {
 730                                 if (checkEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
 731                                 {
 732                                         AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
 733                                                         checkEntry->GetRun(), GetDetName(iDet)));
 734                                         fFirstUnprocessed[iDet] = kFALSE;
 735                                 }
 736                         }
 737                 }
 738         }
 739
 740         fLogbookEntry = 0;
 741
 742         return hasError == kFALSE;
 743 }
 744
 745 //______________________________________________________________________________________________
 746 UInt_t AliShuttle::ProcessCurrentDetector()
 747 {
 748         //
 749         // Makes data retrieval just for a specific detector (fCurrentDetector).
 750         // Threre should be a configuration for this detector.
 751
 752         AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), GetCurrentRun()));
 753
 754         UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
 755
 756         TMap dcsMap;
 757         dcsMap.SetOwner(1);
 758
 759         Bool_t aDCSError = kFALSE;
 760         fGridError = kFALSE;
 761
 762         // TODO Test only... I've added a flag that allows to
 763         // exclude DCS archive DB query
 764         if (!fgkProcessDCS)
 765         {
 766                 AliInfo("Skipping DCS processing!");
 767                 aDCSError = kFALSE;
 768         } else {
 769                 TString host(fConfig->GetDCSHost(fCurrentDetector));
 770                 Int_t port = fConfig->GetDCSPort(fCurrentDetector);
 771
 772                 // Retrieval of Aliases
 773                 TObjString* anAlias = 0;
 774                 TIter iterAliases(fConfig->GetDCSAliases(fCurrentDetector));
 775                 while ((anAlias = (TObjString*) iterAliases.Next()))
 776                 {
 777                         TObjArray *valueSet = new TObjArray();
 778                         valueSet->SetOwner(1);
 779
 780                         AliInfo("Querying DCS archive DB (Aliases)...");
 781                         aDCSError = (GetValueSet(host, port, anAlias->String(), valueSet, kAlias) == 0);
 782
 783                         if(!aDCSError)
 784                         {
 785                                 dcsMap.Add(anAlias->Clone(), valueSet);
 786                         } else {
 787                                 Log(fCurrentDetector,
 788                                         Form("ProcessCurrentDetector - Error while retrieving alias %s",
 789                                                 anAlias->GetName()));
 790                                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
 791                                 dcsMap.DeleteAll();
 792                                 return 0;
 793                         }
 794                 }
 795
 796                 // Retrieval of Data Points
 797                 TObjString* aDP = 0;
 798                 TIter iterDP(fConfig->GetDCSDataPoints(fCurrentDetector));
 799                 while ((aDP = (TObjString*) iterDP.Next()))
 800                 {
 801                         TObjArray *valueSet = new TObjArray();
 802                         valueSet->SetOwner(1);
 803                         AliInfo("Querying DCS archive DB (Data Points)...");
 804                         aDCSError = (GetValueSet(host, port, aDP->String(), valueSet, kDP) == 0);
 805
 806                         if(!aDCSError)
 807                         {
 808                                 dcsMap.Add(aDP->Clone(), valueSet);
 809                         } else {
 810                                 Log(fCurrentDetector,
 811                                         Form("ProcessCurrentDetector - Error while retrieving data point %s",
 812                                                 aDP->GetName()));
 813                                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
 814                                 dcsMap.DeleteAll();
 815                                 return 0;
 816                         }
 817                 }
 818         }
 819
 820         // DCS Archive DB processing successful. Call Preprocessor!
 821         UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
 822
 823         AliPreprocessor* aPreprocessor =
 824                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
 825
 826         aPreprocessor->Initialize(GetCurrentRun(), GetCurrentStartTime(), GetCurrentEndTime());
 827         UInt_t aPPResult = aPreprocessor->Process(&dcsMap);
 828
 829         UInt_t returnValue = 0;
 830         if (aPPResult == 0) { // Preprocessor error
 831                 UpdateShuttleStatus(AliShuttleStatus::kPPError);
 832                 returnValue = 0;
 833         } else if (fGridError == kFALSE) { // process and Grid storage ok!
 834                 UpdateShuttleStatus(AliShuttleStatus::kDone);
 835                 UpdateShuttleLogbook(fCurrentDetector, "DONE");
 836                 Log(fCurrentDetector.Data(),
 837                         "ProcessCurrentDetector - Preprocessor and Grid storage ended successfully");
 838                 returnValue = 1;
 839         } else { // Grid storage error (process ok, but object put in local storage)
 840                 UpdateShuttleStatus(AliShuttleStatus::kStoreFailed);
 841                 returnValue = 2;
 842         }
 843
 844         dcsMap.DeleteAll();
 845
 846         return returnValue;
 847 }
 848
 849 //______________________________________________________________________________________________
 850 Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
 851                 TObjArray& entries)
 852 {
 853 // Query DAQ's Shuttle logbook and fills detector status object.
 854 // Call QueryRunParameters to query DAQ logbook for run parameters.
 855
 856         entries.SetOwner(1);
 857
 858         // check connection, in case connect
 859         if(!Connect(3)) return kFALSE;
 860
 861         TString sqlQuery;
 862         sqlQuery = Form("select * from logbook_shuttle %s order by run", whereClause);
 863
 864         TSQLResult* aResult = fServer[3]->Query(sqlQuery);
 865         if (!aResult) {
 866                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
 867                 return kFALSE;
 868         }
 869
 870         AliDebug(2,Form("Query = %s", sqlQuery.Data()));
 871
 872         if(aResult->GetRowCount() == 0) {
 873 //              if(sqlQuery.EndsWith("where shuttle_done=0 order by run")){
 874 //                      Log("SHUTTLE", "QueryShuttleLogbook - All runs in Shuttle Logbook are already DONE");
 875 //                      delete aResult;
 876 //                      return kTRUE;
 877 //              } else {
 878                         AliInfo("No entries in Shuttle Logbook match request");
 879                         delete aResult;
 880                         return kTRUE;
 881 //              }
 882         }
 883
 884         // TODO Check field count!
 885         const UInt_t nCols = 22;
 886         if (aResult->GetFieldCount() != (Int_t) nCols) {
 887                 AliError("Invalid SQL result field number!");
 888                 delete aResult;
 889                 return kFALSE;
 890         }
 891
 892         TSQLRow* aRow;
 893         while ((aRow = aResult->Next())) {
 894                 TString runString(aRow->GetField(0), aRow->GetFieldLength(0));
 895                 Int_t run = runString.Atoi();
 896
 897                 AliShuttleLogbookEntry *entry = QueryRunParameters(run);
 898                 if (!entry)
 899                         continue;
 900
 901                 // loop on detectors
 902                 for(UInt_t ii = 0; ii < nCols; ii++)
 903                         entry->SetDetectorStatus(aResult->GetFieldName(ii), aRow->GetField(ii));
 904
 905                 entries.AddLast(entry);
 906                 delete aRow;
 907         }
 908
 909 //      if(sqlQuery.EndsWith("where shuttle_done=0 order by run"))
 910 //              Log("SHUTTLE", Form("QueryShuttleLogbook - Found %d unprocessed runs in Shuttle Logbook",
 911 //                                                      entries.GetEntriesFast()));
 912         delete aResult;
 913         return kTRUE;
 914 }
 915
 916 //______________________________________________________________________________________________
 917 AliShuttleLogbookEntry* AliShuttle::QueryRunParameters(Int_t run)
 918 {
 919         //
 920         // Retrieve run parameters written in the DAQ logbook and sets them into AliShuttleLogbookEntry object
 921         //
 922
 923         // check connection, in case connect
 924         if (!Connect(3))
 925                 return 0;
 926
 927         TString sqlQuery;
 928         sqlQuery.Form("select * from %s where run=%d", fConfig->GetDAQlbTable(), run);
 929
 930         TSQLResult* aResult = fServer[3]->Query(sqlQuery);
 931         if (!aResult) {
 932                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
 933                 return 0;
 934         }
 935
 936         if (aResult->GetRowCount() == 0) {
 937                 Log("SHUTTLE", Form("QueryRunParameters - No entry in DAQ Logbook for run %d. Skipping", run));
 938                 delete aResult;
 939                 return 0;
 940         }
 941
 942         if (aResult->GetRowCount() > 1) {
 943                 AliError(Form("More than one entry in DAQ Logbook for run %d. Skipping", run));
 944                 delete aResult;
 945                 return 0;
 946         }
 947
 948         TSQLRow* aRow = aResult->Next();
 949         if (!aRow)
 950         {
 951                 AliError(Form("Could not retrieve row for run %d. Skipping", run));
 952                 delete aResult;
 953                 return 0;
 954         }
 955
 956         AliShuttleLogbookEntry* entry = new AliShuttleLogbookEntry(run);
 957
 958         for (Int_t ii = 0; ii < aResult->GetFieldCount(); ii++)
 959                 entry->SetRunParameter(aResult->GetFieldName(ii), aRow->GetField(ii));
 960
 961         UInt_t startTime = entry->GetStartTime();
 962         UInt_t endTime = entry->GetEndTime();
 963
 964         if (!startTime || !endTime || startTime > endTime) {
 965                 Log("SHUTTLE",
 966                         Form("QueryRunParameters - Invalid parameters for Run %d: startTime = %d, endTime = %d",
 967                                 run, startTime, endTime));
 968                 delete entry;
 969                 delete aRow;
 970                 delete aResult;
 971                 return 0;
 972         }
 973
 974         delete aRow;
 975         delete aResult;
 976
 977         return entry;
 978 }
 979
 980 //______________________________________________________________________________________________
 981 Bool_t AliShuttle::TryToStoreAgain()
 982 {
 983   // Called in case the detector failed to store the object in Grid OCDB
 984   // It tries to store the object again, if it does not find more recent and overlapping objects
 985   // Calls underlying TryToStoreAgain(const char*) function twice, for OCDB and Reference storage.
 986
 987         AliInfo("Trying to store OCDB data again...");
 988         Bool_t resultCDB = TryToStoreAgain(fgkMainCDB);
 989
 990         AliInfo("Trying to store reference data again...");
 991         Bool_t resultRef = TryToStoreAgain(fgkMainRefStorage);
 992
 993         return resultCDB && resultRef;
 994 }
 995
 996 //______________________________________________________________________________________________
 997 Bool_t AliShuttle::TryToStoreAgain(TString& gridURI)
 998 {
 999   // Called by TryToStoreAgain(), performs actual storage retry
1000
1001         TObjArray* gridIds=0;
1002
1003         Bool_t result = kTRUE;
1004
1005         const char* type = 0;
1006         TString backupURI;
1007         if(gridURI == fgkMainCDB) {
1008                 type = "OCDB";
1009                 backupURI = fgkLocalCDB;
1010         } else if(gridURI == fgkMainRefStorage) {
1011                 type = "reference";
1012                 backupURI = fgkLocalRefStorage;
1013         } else {
1014                 AliError(Form("Invalid storage URI: %s", gridURI.Data()));
1015                 return kFALSE;
1016         }
1017
1018         AliCDBManager* man = AliCDBManager::Instance();
1019
1020         AliCDBStorage *gridSto = man->GetStorage(gridURI);
1021         if(!gridSto) {
1022                 Log(fCurrentDetector.Data(),
1023                         Form("TryToStoreAgain - cannot activate main %s storage", type));
1024                 return kFALSE;
1025         }
1026
1027         gridIds = gridSto->GetQueryCDBList();
1028
1029         // get objects previously stored in local CDB
1030         AliCDBStorage *backupSto = man->GetStorage(backupURI);
1031         AliCDBPath aPath(GetOfflineDetName(fCurrentDetector.Data()),"*","*");
1032         // Local objects were stored with current run as Grid version!
1033         TList* localEntries = backupSto->GetAll(aPath.GetPath(), GetCurrentRun(), GetCurrentRun());
1034         localEntries->SetOwner(1);
1035
1036         // loop on local stored objects
1037         TIter localIter(localEntries);
1038         AliCDBEntry *aLocEntry = 0;
1039         while((aLocEntry = dynamic_cast<AliCDBEntry*> (localIter.Next()))){
1040                 aLocEntry->SetOwner(1);
1041                 AliCDBId aLocId = aLocEntry->GetId();
1042                 aLocEntry->SetVersion(-1);
1043                 aLocEntry->SetSubVersion(-1);
1044
1045                 // loop on Grid valid Id's
1046                 Bool_t store = kTRUE;
1047                 TIter gridIter(gridIds);
1048                 AliCDBId* aGridId = 0;
1049                 while((aGridId = dynamic_cast<AliCDBId*> (gridIter.Next()))){
1050                         // If local object is valid up to infinity we store it only if it is
1051                         // the first unprocessed run!
1052                         if (aLocId.GetLastRun() == AliCDBRunRange::Infinity())
1053                         {
1054                                 if (!fFirstUnprocessed[GetDetPos(fCurrentDetector)])
1055                                 {
1056                                         Log(fCurrentDetector.Data(),
1057                                                 ("TryToStoreAgain - This object has validity infinite but "
1058                                                  "there are previous unprocessed runs!"));
1059                                         continue;
1060                                 } else {
1061                                         break;
1062                                 }
1063                         }
1064                         if(aGridId->GetPath() != aLocId.GetPath()) continue;
1065                         // skip all objects valid up to infinity
1066                         if(aGridId->GetLastRun() == AliCDBRunRange::Infinity()) continue;
1067                         // if we get here, it means there's already some more recent object stored on Grid!
1068                         store = kFALSE;
1069                         break;
1070                 }
1071
1072                 if(!store){
1073                         Log(fCurrentDetector.Data(),
1074                                 Form("TryToStoreAgain - A more recent object already exists in %s storage: <%s>",
1075                                         type, aGridId->ToString().Data()));
1076                         // removing local filename...
1077                         // TODO maybe it's better not to remove it, it was not copied to the Grid!
1078                         TString filename;
1079                         backupSto->IdToFilename(aLocId, filename);
1080                         AliInfo(Form("Removing local file %s", filename.Data()));
1081                         gSystem->Exec(Form("rm %s",filename.Data()));
1082                         continue;
1083                 }
1084
1085                 // If we get here, the file can be stored!
1086                 Bool_t storeOk = gridSto->Put(aLocEntry);
1087                 if(storeOk){
1088                         Log(fCurrentDetector.Data(),
1089                                 Form("TryToStoreAgain - Object <%s> successfully put into %s storage",
1090                                         aLocId.ToString().Data(), type));
1091
1092                         // removing local filename...
1093                         TString filename;
1094                         backupSto->IdToFilename(aLocId, filename);
1095                         AliInfo(Form("Removing local file %s", filename.Data()));
1096                         gSystem->Exec(Form("rm %s", filename.Data()));
1097                         continue;
1098                 } else  {
1099                         Log(fCurrentDetector.Data(),
1100                                 Form("TryToStoreAgain - Grid %s storage of object <%s> failed again",
1101                                         type, aLocId.ToString().Data()));
1102                         result = kFALSE;
1103                 }
1104         }
1105         localEntries->Clear();
1106
1107         return result;
1108 }
1109
1110 //______________________________________________________________________________________________
1111 Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* entry,
1112                                 TObjArray* valueSet, DCSType type)
1113 {
1114 // Retrieve all "entry" data points from the DCS server
1115 // host, port: TSocket connection parameters
1116 // entry: name of the alias or data point
1117 // valueSet: array of retrieved AliDCSValue's
1118 // type: kAlias or kDP
1119
1120         AliDCSClient client(host, port, fTimeout, fRetries);
1121         if (!client.IsConnected())
1122         {
1123                 return kFALSE;
1124         }
1125
1126         Int_t result=0;
1127
1128         if (type == kAlias)
1129         {
1130                 result = client.GetAliasValues(entry,
1131                         GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
1132         } else
1133         if (type == kDP)
1134         {
1135                 result = client.GetDPValues(entry,
1136                         GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
1137         }
1138
1139         if (result < 0)
1140         {
1141                 Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get '%s'! Reason: %s",
1142                         entry, AliDCSClient::GetErrorString(result)));
1143
1144                 if (result == AliDCSClient::fgkServerError)
1145                 {
1146                         Log(fCurrentDetector.Data(), Form("GetValueSet - Server error: %s",
1147                                 client.GetServerError().Data()));
1148                 }
1149
1150                 return kFALSE;
1151         }
1152
1153         return kTRUE;
1154 }
1155
1156 //______________________________________________________________________________________________
1157 const char* AliShuttle::GetFile(Int_t system, const char* detector,
1158                 const char* id, const char* source)
1159 {
1160 // Get calibration file from file exchange servers
1161 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
1162
1163         switch(system){
1164                 case kDAQ:
1165                         return GetDAQFileName(detector, id, source);
1166                         break;
1167                 case kDCS:
1168                         return GetDCSFileName(detector, id, source);
1169                         break;
1170                 case kHLT:
1171                         return GetHLTFileName(detector, id, source);
1172                         break;
1173                 default:
1174                         AliError(Form("No valid system index: %d",system));
1175         }
1176
1177         return 0;
1178 }
1179
1180 //______________________________________________________________________________________________
1181 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
1182 {
1183 // Get sources producing the condition file Id from file exchange servers
1184 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
1185
1186         switch(system){
1187                 case kDAQ:
1188                         return GetDAQFileSources(detector, id);
1189                         break;
1190                 case kDCS:
1191                         return GetDCSFileSources(detector, id);
1192                         break;
1193                 case kHLT:
1194                         return GetHLTFileSources(detector, id);
1195                         break;
1196                 default:
1197                         AliError(Form("No valid system index: %d",system));
1198         }
1199
1200         return NULL;
1201 }
1202
1203 //______________________________________________________________________________________________
1204 Bool_t AliShuttle::Connect(Int_t system)
1205 {
1206 // Connect to MySQL Server of the system's FXS MySQL databases
1207 // DAQ Logbook, Shuttle Logbook and DAQ FXS db are on the same host
1208
1209         // check connection: if already connected return
1210         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
1211
1212         TString dbHost, dbUser, dbPass, dbName;
1213
1214         if (system < 3) // FXS db servers
1215         {
1216                 dbHost = Form("mysql://%s:%d", fConfig->GetFXSdbHost(system), fConfig->GetFXSdbPort(system));
1217                 dbUser = fConfig->GetFXSdbUser(system);
1218                 dbPass = fConfig->GetFXSdbPass(system);
1219                 dbName =   fConfig->GetFXSdbName(system);
1220         } else { // Run & Shuttle logbook servers
1221         // TODO Will the Shuttle logbook server be the same as the Run logbook server ???
1222                 dbHost = Form("mysql://%s:%d", fConfig->GetDAQlbHost(), fConfig->GetDAQlbPort());
1223                 dbUser = fConfig->GetDAQlbUser();
1224                 dbPass = fConfig->GetDAQlbPass();
1225                 dbName =   fConfig->GetDAQlbDB();
1226         }
1227
1228         fServer[system] = TSQLServer::Connect(dbHost.Data(), dbUser.Data(), dbPass.Data());
1229         if (!fServer[system] || !fServer[system]->IsConnected()) {
1230                 if(system < 3)
1231                 {
1232                 AliError(Form("Can't establish connection to FXS database for %s",
1233                                         AliShuttleInterface::GetSystemName(system)));
1234                 } else {
1235                 AliError("Can't establish connection to Run logbook.");
1236                 }
1237                 if(fServer[system]) delete fServer[system];
1238                 return kFALSE;
1239         }
1240
1241         // Get tables
1242         // TODO in the configuration should the table name be there too?
1243         TSQLResult* aResult=0;
1244         switch(system){
1245                 case kDAQ:
1246                         aResult = fServer[kDAQ]->GetTables(dbName.Data());
1247                         break;
1248                 case kDCS:
1249                         //aResult = fServer[kDCS]->GetTables(dbName.Data());
1250                         break;
1251                 case kHLT:
1252                         aResult = fServer[kHLT]->GetTables(dbName.Data());
1253                         break;
1254                 default:
1255                         aResult = fServer[3]->GetTables(dbName.Data());
1256                         break;
1257         }
1258
1259         delete aResult;
1260         return kTRUE;
1261 }
1262
1263 //______________________________________________________________________________________________
1264 const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source)
1265 {
1266 // Retrieves a file from the DAQ FXS.
1267 // First queris the DAQ FXS database for the DAQ file name, using the run, detector, id and source info
1268 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
1269 // run: current run being processed (given by Logbook entry fLogbookEntry)
1270 // detector: the Preprocessor name
1271 // id: provided as a parameter by the Preprocessor
1272 // source: provided by the Preprocessor through GetFileSources function
1273
1274         // check connection, in case connect
1275         if (!Connect(kDAQ))
1276         {
1277                 Log(detector, "GetDAQFileName - Couldn't connect to DAQ FXS database");
1278                 return 0;
1279         }
1280
1281         // Query preparation
1282         TString sqlQueryStart = Form("select filePath from %s where", fConfig->GetFXSdbTable(kDAQ));
1283         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
1284                                 GetCurrentRun(), detector, id, source);
1285         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1286
1287         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1288
1289         // Query execution
1290         TSQLResult* aResult = 0;
1291         aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1292         if (!aResult) {
1293                 Log(detector, Form("GetDAQFileName - Can't execute SQL query for: id = %s, source = %s",
1294                                 id, source));
1295                 return 0;
1296         }
1297
1298         if(aResult->GetRowCount() == 0)
1299         {
1300                 Log(detector,
1301                         Form("GetDAQFileName - No entry in FXS table for: id = %s, source = %s",
1302                                 id, source));
1303                 delete aResult;
1304                 return 0;
1305         }
1306
1307         if (aResult->GetRowCount() > 1) {
1308                 Log(detector,
1309                         Form("GetDAQFileName - More than one entry in FXS table for: id = %s, source = %s",
1310                                 id, source));
1311                 delete aResult;
1312                 return 0;
1313         }
1314
1315         TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
1316
1317         if (!aRow){
1318                 Log(detector, Form("GetDAQFileName - Empty set result from query: id = %s, source = %s",
1319                                 id, source));
1320                 delete aResult;
1321                 return 0;
1322         }
1323
1324         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
1325
1326         delete aResult;
1327         delete aRow;
1328
1329         AliDebug(2, Form("filePath = %s",filePath.Data()));
1330
1331         // retrieved file is renamed to make it unique
1332         TString localFileName = Form("DAQ_%s_%d_%s_%s.shuttle",
1333                                         detector, GetCurrentRun(), id, source);
1334
1335         // file retrieval from DAQ FXS
1336         Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
1337         if(!result) {
1338                 Log(detector, Form("GetDAQFileName - Copy of file %s from DAQ FXS failed", filePath.Data()));
1339                 return 0;
1340         } else {
1341                 AliInfo(Form("File %s copied from DAQ FXS into %s/%s",
1342                         filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
1343         }
1344
1345         fFXSCalled[kDAQ]=kTRUE;
1346         TObjString *fileParams = new TObjString(Form("%s#!?!#%s", id, source));
1347         fFXSlist[kDAQ].Add(fileParams);
1348
1349         static TString fullLocalFileName;
1350         fullLocalFileName = TString::Format("%s/%s", fgkShuttleTempDir, localFileName.Data());
1351
1352         AliInfo(Form("fullLocalFileName = %s", fullLocalFileName.Data()));
1353
1354         return fullLocalFileName.Data();
1355
1356 }
1357
1358 //______________________________________________________________________________________________
1359 Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName)
1360 {
1361 // Copies file from DAQ FXS to local Shuttle machine
1362
1363         // check temp directory: trying to cd to temp; if it does not exist, create it
1364         AliDebug(2, Form("Copy file %s from DAQ FXS into %s/%s",
1365                         daqFileName, fgkShuttleTempDir, localFileName));
1366
1367         void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
1368         if (dir == NULL) {
1369                 if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
1370                         AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
1371                         return kFALSE;
1372                 }
1373
1374         } else {
1375                 gSystem->FreeDirectory(dir);
1376         }
1377
1378         TString baseDAQFXSFolder = "FES";
1379         TString command = Form("scp -oPort=%d -2 %s@%s:%s/%s %s/%s",
1380                 fConfig->GetFXSPort(kDAQ),
1381                 fConfig->GetFXSUser(kDAQ),
1382                 fConfig->GetFXSHost(kDAQ),
1383                 baseDAQFXSFolder.Data(),
1384                 daqFileName,
1385                 fgkShuttleTempDir,
1386                 localFileName);
1387
1388         AliDebug(2, Form("%s",command.Data()));
1389
1390         UInt_t nRetries = 0;
1391         UInt_t maxRetries = 3;
1392
1393         // copy!! if successful TSystem::Exec returns 0
1394         while(nRetries++ < maxRetries) {
1395                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
1396                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
1397         }
1398
1399         return kFALSE;
1400
1401 }
1402
1403 //______________________________________________________________________________________________
1404 TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id)
1405 {
1406 // Retrieves list of DAQ sources of file Id
1407
1408         // check connection, in case connect
1409         if(!Connect(kDAQ)){
1410                 Log(detector, "GetDAQFileSources - Couldn't connect to DAQ FXS database");
1411                 return 0;
1412         }
1413
1414         // Query preparation
1415         TString sqlQueryStart = Form("select DAQsource from %s where", fConfig->GetFXSdbTable(kDAQ));
1416         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
1417                                 GetCurrentRun(), detector, id);
1418         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1419
1420         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1421
1422         // Query execution
1423         TSQLResult* aResult;
1424         aResult = fServer[kDAQ]->Query(sqlQuery);
1425         if (!aResult) {
1426                 Log(detector, Form("GetDAQFileSources - Can't execute SQL query for id: %s", id));
1427                 return 0;
1428         }
1429
1430         if (aResult->GetRowCount() == 0) {
1431                 Log(detector,
1432                         Form("GetDAQFileSources - No entry in FXS table for id: %s", id));
1433                 delete aResult;
1434                 return 0;
1435         }
1436
1437         TSQLRow* aRow;
1438         TList *list = new TList();
1439         list->SetOwner(1);
1440
1441         while((aRow = aResult->Next())){
1442
1443                 TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
1444                 AliDebug(2, Form("daqSource = %s", daqSource.Data()));
1445                 list->Add(new TObjString(daqSource));
1446                 delete aRow;
1447         }
1448         delete aResult;
1449
1450         return list;
1451
1452 }
1453
1454 //______________________________________________________________________________________________
1455 const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1456 // Retrieves a file from the DCS FXS.
1457
1458 return "You're in DCS";
1459
1460 }
1461
1462 //______________________________________________________________________________________________
1463 TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
1464 // Retrieves file sources from the DCS FXS.
1465
1466 return NULL;
1467
1468 }
1469
1470 //______________________________________________________________________________________________
1471 const char* AliShuttle::GetHLTFileName(const char* detector, const char* id, const char* source){
1472 // Retrieves a file from the HLT FXS.
1473 // First queris the HLT FXS database for the HLT file name, using the run, detector, id and source info
1474 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
1475 // run: current run being processed (given by Logbook entry fLogbookEntry)
1476 // detector: the Preprocessor name
1477 // id: provided as a parameter by the Preprocessor
1478 // source: provided by the Preprocessor through GetFileSources function
1479
1480         // check connection, in case connect
1481         if (!Connect(kHLT))
1482         {
1483                 Log(detector, "GetHLTFileName - Couldn't connect to HLT FXS database");
1484                 return 0;
1485         }
1486
1487         // Query preparation
1488         TString sqlQueryStart = Form("select filePath,fileSize,fileChecksum from %s where",
1489                                                                                 fConfig->GetFXSdbTable(kHLT));
1490         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DDLnumbers=\"%s\"",
1491                                 GetCurrentRun(), detector, id, source);
1492         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1493
1494         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1495
1496         // Query execution
1497         TSQLResult* aResult = 0;
1498         aResult = dynamic_cast<TSQLResult*> (fServer[kHLT]->Query(sqlQuery));
1499         if (!aResult) {
1500                 Log(detector, Form("GetHLTFileName - Can't execute SQL query for: id = %s, source = %s",
1501                                 id, source));
1502                 return 0;
1503         }
1504
1505         if(aResult->GetRowCount() == 0)
1506         {
1507                 Log(detector,
1508                         Form("GetHLTFileName - No entry in FXS table for: id = %s, source = %s",
1509                                 id, source));
1510                 delete aResult;
1511                 return 0;
1512         }
1513
1514         if (aResult->GetRowCount() > 1) {
1515                 Log(detector,
1516                         Form("GetHLTFileName - More than one entry in FXS table for: id = %s, source = %s",
1517                                 id, source));
1518                 delete aResult;
1519                 return 0;
1520         }
1521
1522         if (aResult->GetFieldCount() != 3) {
1523                 Log(detector,
1524                         Form("GetHLTFileName - Wrong field count in FXS table for: id = %s, source = %s",
1525                                 id, source));
1526                 delete aResult;
1527                 return 0;
1528         }
1529
1530         TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
1531
1532         if (!aRow){
1533                 Log(detector, Form("GetHLTFileName - Empty set result from query: id = %s, source = %s",
1534                                 id, source));
1535                 delete aResult;
1536                 return 0;
1537         }
1538
1539         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
1540         TString fileSize(aRow->GetField(1), aRow->GetFieldLength(1));
1541         TString fileMd5Sum(aRow->GetField(2), aRow->GetFieldLength(2));
1542
1543         delete aResult;
1544         delete aRow;
1545
1546         AliDebug(2, Form("filePath = %s",filePath.Data()));
1547
1548         // The full file path in HLT FXS is runNb/DET/DDLnumber/filePath
1549 //      TString fullFilePath = Form("%d/%s/%s/%s", GetCurrentRun(), detector, source, filePath.Data());
1550
1551         // retrieved file is renamed to make it unique
1552         TString localFileName = Form("HLT_%s_%d_%s_%s.shuttle",
1553                                         detector, GetCurrentRun(), id, source);
1554
1555         // file retrieval from HLT FXS
1556         Bool_t result = RetrieveHLTFile(filePath.Data(), localFileName.Data());
1557         if(!result)
1558         {
1559                 Log(detector, Form("GetHLTFileName - Copy of file %s from HLT FXS failed", filePath.Data()));
1560                 return 0;
1561         } else {
1562                 AliInfo(Form("File %s copied from HLT FXS into %s/%s",
1563                         filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
1564         }
1565
1566         // compare md5sum of local file with the one stored in the HLT DB
1567         Int_t md5Comp = gSystem->Exec(Form("md5sum %s/%s |grep %s 2>&1 > /dev/null",
1568                                                 fgkShuttleTempDir, localFileName.Data(), fileMd5Sum.Data()));
1569
1570         if (md5Comp != 0)
1571         {
1572                 Log(detector, Form("GetHLTFileName - md5sum of file %s does not match with local copy!", filePath.Data()));
1573                 return 0;
1574         }
1575
1576         fFXSCalled[kHLT]=kTRUE;
1577         TObjString *fileParams = new TObjString(Form("%s#!?!#%s", id, source));
1578         fFXSlist[kHLT].Add(fileParams);
1579
1580         static TString fullLocalFileName;
1581         fullLocalFileName = TString::Format("%s/%s", fgkShuttleTempDir, localFileName.Data());
1582
1583         AliInfo(Form("fullLocalFileName = %s", fullLocalFileName.Data()));
1584
1585         return fullLocalFileName.Data();
1586
1587 }
1588
1589 //______________________________________________________________________________________________
1590 Bool_t AliShuttle::RetrieveHLTFile(const char* hltFileName, const char* localFileName)
1591 {
1592 // Copies file from HLT FXS to local Shuttle machine
1593
1594         // check temp directory: trying to cd to temp; if it does not exist, create it
1595         AliDebug(2, Form("Copy file %s from HLT FXS into %s/%s",
1596                         hltFileName, fgkShuttleTempDir, localFileName));
1597
1598         void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
1599         if (dir == NULL) {
1600                 if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
1601                         AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
1602                         return kFALSE;
1603                 }
1604
1605         } else {
1606                 gSystem->FreeDirectory(dir);
1607         }
1608
1609         TString baseHLTFXSFolder = "~";
1610         TString command = Form("scp -oPort=%d %s@%s:%s/%s %s/%s",
1611                 fConfig->GetFXSPort(kHLT),
1612                 fConfig->GetFXSUser(kHLT),
1613                 fConfig->GetFXSHost(kHLT),
1614                 baseHLTFXSFolder.Data(),
1615                 hltFileName,
1616                 fgkShuttleTempDir,
1617                 localFileName);
1618
1619         AliDebug(2, Form("%s",command.Data()));
1620
1621         UInt_t nRetries = 0;
1622         UInt_t maxRetries = 3;
1623
1624         // copy!! if successful TSystem::Exec returns 0
1625         while(nRetries++ < maxRetries) {
1626                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
1627                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
1628         }
1629
1630         return kFALSE;
1631
1632 }
1633
1634 //______________________________________________________________________________________________
1635 TList* AliShuttle::GetHLTFileSources(const char* detector, const char* id){
1636 // Retrieves list of HLT sources (DDLnumbers) of file Id
1637
1638         // check connection, in case connect
1639         if(!Connect(kHLT)){
1640                 Log(detector, "GetHLTFileSources - Couldn't connect to HLT FXS database");
1641                 return 0;
1642         }
1643
1644         // Query preparation
1645         TString sqlQueryStart = Form("select DDLnumbers from %s where", fConfig->GetFXSdbTable(kHLT));
1646         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
1647                                 GetCurrentRun(), detector, id);
1648         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1649
1650         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1651
1652         // Query execution
1653         TSQLResult* aResult;
1654         aResult = fServer[kHLT]->Query(sqlQuery);
1655         if (!aResult) {
1656                 Log(detector, Form("GetHLTFileSources - Can't execute SQL query for id: %s", id));
1657                 return 0;
1658         }
1659
1660         if (aResult->GetRowCount() == 0) {
1661                 Log(detector,
1662                         Form("GetHLTFileSources - No entry in FXS table for id: %s", id));
1663                 delete aResult;
1664                 return 0;
1665         }
1666
1667         TSQLRow* aRow;
1668         TList *list = new TList();
1669         list->SetOwner(1);
1670
1671         while((aRow = aResult->Next())){
1672
1673                 TString ddlNumbers(aRow->GetField(0), aRow->GetFieldLength(0));
1674                 AliDebug(2, Form("DDLnumbers = %s", ddlNumbers.Data()));
1675                 list->Add(new TObjString(ddlNumbers));
1676                 delete aRow;
1677         }
1678         delete aResult;
1679
1680         return list;
1681
1682 }
1683
1684 //______________________________________________________________________________________________
1685 Bool_t AliShuttle::UpdateDAQTable()
1686 {
1687 // Update DAQ table filling time_processed field in all rows corresponding to current run and detector
1688
1689         // check connection, in case connect
1690         if(!Connect(kDAQ)){
1691                 Log(fCurrentDetector, "UpdateDAQTable - Couldn't connect to DAQ FXS database");
1692                 return kFALSE;
1693         }
1694
1695         TTimeStamp now; // now
1696
1697         // Loop on FXS list entries
1698         TIter iter(&fFXSlist[kDAQ]);
1699         TObjString *aFXSentry=0;
1700         while((aFXSentry = dynamic_cast<TObjString*> (iter.Next()))){
1701                 TString aFXSentrystr = aFXSentry->String();
1702                 TObjArray *aFXSarray = aFXSentrystr.Tokenize("#!?!#");
1703                 if(!aFXSarray || aFXSarray->GetEntries() != 2 ) {
1704                         Log(fCurrentDetector, Form("UpdateDAQTable - error updating FXS entry. Check string: <%s>",
1705                                 aFXSentrystr.Data()));
1706                         if(aFXSarray) delete aFXSarray;
1707                         return kFALSE;
1708                 }
1709                 const char* fileId = ((TObjString*) aFXSarray->At(0))->GetName();
1710                 const char* daqSource = ((TObjString*) aFXSarray->At(1))->GetName();
1711                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
1712                         GetCurrentRun(), fCurrentDetector.Data(), fileId, daqSource);
1713
1714                 delete aFXSarray;
1715
1716                 TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(kDAQ),
1717                                                         now.GetSec(), whereClause.Data());
1718
1719                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1720
1721                 // Query execution
1722                 TSQLResult* aResult;
1723                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1724                 if (!aResult) {
1725                         Log(fCurrentDetector, Form("UpdateDAQTable - Can't execute SQL query <%s>", sqlQuery.Data()));
1726                         return kFALSE;
1727                 }
1728                 delete aResult;
1729         }
1730
1731         return kTRUE;
1732 }
1733
1734 //______________________________________________________________________________________________
1735 Bool_t AliShuttle::UpdateHLTTable()
1736 {
1737 // Update HLT table filling time_processed field in all rows corresponding to current run and detector
1738
1739         // check connection, in case connect
1740         if(!Connect(kHLT)){
1741                 Log(fCurrentDetector, "UpdateHLTTable - Couldn't connect to HLT FXS database");
1742                 return kFALSE;
1743         }
1744
1745         TTimeStamp now; // now
1746
1747         // Loop on FXS list entries
1748         TIter iter(&fFXSlist[kHLT]);
1749         TObjString *aFXSentry=0;
1750         while((aFXSentry = dynamic_cast<TObjString*> (iter.Next()))){
1751                 TString aFXSentrystr = aFXSentry->String();
1752                 TObjArray *aFXSarray = aFXSentrystr.Tokenize("#!?!#");
1753                 if(!aFXSarray || aFXSarray->GetEntries() != 2 ) {
1754                         Log(fCurrentDetector, Form("UpdateHLTTable - error updating FXS entry. Check string: <%s>",
1755                                 aFXSentrystr.Data()));
1756                         if(aFXSarray) delete aFXSarray;
1757                         return kFALSE;
1758                 }
1759                 const char* fileId = ((TObjString*) aFXSarray->At(0))->GetName();
1760                 const char* hltSource = ((TObjString*) aFXSarray->At(1))->GetName();
1761                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DDLnumbers=\"%s\";",
1762                         GetCurrentRun(), fCurrentDetector.Data(), fileId, hltSource);
1763
1764                 delete aFXSarray;
1765
1766                 TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(kHLT),
1767                                                         now.GetSec(), whereClause.Data());
1768
1769                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1770
1771                 // Query execution
1772                 TSQLResult* aResult;
1773                 aResult = dynamic_cast<TSQLResult*> (fServer[kHLT]->Query(sqlQuery));
1774                 if (!aResult) {
1775                         Log(fCurrentDetector, Form("UpdateHLTTable - Can't execute SQL query <%s>", sqlQuery.Data()));
1776                         return kFALSE;
1777                 }
1778                 delete aResult;
1779         }
1780
1781         return kTRUE;
1782 }
1783
1784 //______________________________________________________________________________________________
1785 Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status)
1786 {
1787 // Update Shuttle logbook filling detector or shuttle_done column
1788 // ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
1789
1790         // check connection, in case connect
1791         if(!Connect(3)){
1792                 Log("SHUTTLE", "UpdateShuttleLogbook - Couldn't connect to DAQ Logbook.");
1793                 return kFALSE;
1794         }
1795
1796         TString detName(detector);
1797         TString setClause;
1798         if(detName == "shuttle_done") {
1799                 setClause = "set shuttle_done=1";
1800         } else {
1801                 TString statusStr(status);
1802                 if(statusStr.Contains("done", TString::kIgnoreCase) ||
1803                    statusStr.Contains("failed", TString::kIgnoreCase)){
1804                         setClause = Form("set %s=\"%s\"", detector, status);
1805                 } else {
1806                         Log("SHUTTLE",
1807                                 Form("UpdateShuttleLogbook - Invalid status <%s> for detector %s",
1808                                         status, detector));
1809                         return kFALSE;
1810                 }
1811         }
1812
1813         TString whereClause = Form("where run=%d", GetCurrentRun());
1814
1815         TString sqlQuery = Form("update logbook_shuttle %s %s",
1816                                         setClause.Data(), whereClause.Data());
1817
1818         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1819
1820         // Query execution
1821         TSQLResult* aResult;
1822         aResult = dynamic_cast<TSQLResult*> (fServer[3]->Query(sqlQuery));
1823         if (!aResult) {
1824                 Log("SHUTTLE", Form("UpdateShuttleLogbook - Can't execute query <%s>", sqlQuery.Data()));
1825                 return kFALSE;
1826         }
1827         delete aResult;
1828
1829         return kTRUE;
1830 }
1831
1832 //______________________________________________________________________________________________
1833 Int_t AliShuttle::GetCurrentRun() const
1834 {
1835 // Get current run from logbook entry
1836
1837         return fLogbookEntry ? fLogbookEntry->GetRun() : -1;
1838 }
1839
1840 //______________________________________________________________________________________________
1841 UInt_t AliShuttle::GetCurrentStartTime() const
1842 {
1843 // get current start time
1844
1845         return fLogbookEntry ? fLogbookEntry->GetStartTime() : 0;
1846 }
1847
1848 //______________________________________________________________________________________________
1849 UInt_t AliShuttle::GetCurrentEndTime() const
1850 {
1851 // get current end time from logbook entry
1852
1853         return fLogbookEntry ? fLogbookEntry->GetEndTime() : 0;
1854 }
1855
1856 //______________________________________________________________________________________________
1857 void AliShuttle::Log(const char* detector, const char* message)
1858 {
1859 // Fill log string with a message
1860
1861         void* dir = gSystem->OpenDirectory(fgkShuttleLogDir);
1862         if (dir == NULL) {
1863                 if (gSystem->mkdir(fgkShuttleLogDir, kTRUE)) {
1864                         AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
1865                         return;
1866                 }
1867
1868         } else {
1869                 gSystem->FreeDirectory(dir);
1870         }
1871
1872         TString toLog = Form("%s (%d): %s - ", TTimeStamp(time(0)).AsString("s"), getpid(), detector);
1873         if(GetCurrentRun()>=0 ) toLog += Form("run %d - ", GetCurrentRun());
1874         toLog += Form("%s", message);
1875
1876         AliInfo(toLog.Data());
1877
1878         TString fileName;
1879         fileName.Form("%s/%s.log", fgkShuttleLogDir, detector);
1880         gSystem->ExpandPathName(fileName);
1881
1882         ofstream logFile;
1883         logFile.open(fileName, ofstream::out | ofstream::app);
1884
1885         if (!logFile.is_open()) {
1886                 AliError(Form("Could not open file %s", fileName.Data()));
1887                 return;
1888         }
1889
1890         logFile << toLog.Data() << "\n";
1891
1892         logFile.close();
1893 }
1894
1895 //______________________________________________________________________________________________
1896 Bool_t AliShuttle::Collect(Int_t run)
1897 {
1898 //
1899 // Collects conditions data for all UNPROCESSED run written to DAQ LogBook in case of run = -1 (default)
1900 // If a dedicated run is given this run is processed
1901 //
1902 // In operational mode, this is the Shuttle function triggered by the EOR signal.
1903 //
1904
1905         if (run == -1)
1906                 Log("SHUTTLE","Collect - Shuttle called. Collecting conditions data for unprocessed runs");
1907         else
1908                 Log("SHUTTLE", Form("Collect - Shuttle called. Collecting conditions data for run %d", run));
1909
1910         SetLastAction("Starting");
1911
1912         TString whereClause("where shuttle_done=0");
1913         if (run != -1)
1914                 whereClause += Form(" and run=%d", run);
1915
1916         TObjArray shuttleLogbookEntries;
1917         if (!QueryShuttleLogbook(whereClause, shuttleLogbookEntries))
1918         {
1919                 Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
1920                 return kFALSE;
1921         }
1922
1923         if (shuttleLogbookEntries.GetEntries() == 0)
1924         {
1925                 if (run == -1)
1926                         Log("SHUTTLE","Collect - Found no UNPROCESSED runs in Shuttle logbook");
1927                 else
1928                         Log("SHUTTLE", Form("Collect - Run %d is already DONE "
1929                                                 "or it does not exist in Shuttle logbook", run));
1930                 return kTRUE;
1931         }
1932
1933         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
1934                 fFirstUnprocessed[iDet] = kTRUE;
1935
1936         if (run != -1)
1937         {
1938                 // query Shuttle logbook for earlier runs, check if some detectors are unprocessed,
1939                 // flag them into fFirstUnprocessed array
1940                 TString whereClause(Form("where shuttle_done=0 and run < %d", run));
1941                 TObjArray tmpLogbookEntries;
1942                 if (!QueryShuttleLogbook(whereClause, tmpLogbookEntries))
1943                 {
1944                         Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
1945                         return kFALSE;
1946                 }
1947
1948                 TIter iter(&tmpLogbookEntries);
1949                 AliShuttleLogbookEntry* anEntry = 0;
1950                 while ((anEntry = dynamic_cast<AliShuttleLogbookEntry*> (iter.Next())))
1951                 {
1952                         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
1953                         {
1954                                 if (anEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
1955                                 {
1956                                         AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
1957                                                         anEntry->GetRun(), GetDetName(iDet)));
1958                                         fFirstUnprocessed[iDet] = kFALSE;
1959                                 }
1960                         }
1961
1962                 }
1963
1964         }
1965
1966         if (!RetrieveConditionsData(shuttleLogbookEntries))
1967         {
1968                 Log("SHUTTLE", "Collect - Process of at least one run failed");
1969                 return kFALSE;
1970         }
1971
1972         return kTRUE;
1973 }
1974
1975 //______________________________________________________________________________________________
1976 Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
1977 {
1978 // Retrieve conditions data for all runs that aren't processed yet
1979
1980         Bool_t hasError = kFALSE;
1981
1982         TIter iter(&dateEntries);
1983         AliShuttleLogbookEntry* anEntry;
1984
1985         while ((anEntry = (AliShuttleLogbookEntry*) iter.Next())){
1986                 if (!Process(anEntry)){
1987                         hasError = kTRUE;
1988                 }
1989         }
1990
1991         return hasError == kFALSE;
1992 }
1993
1994 //______________________________________________________________________________________________
1995 ULong_t AliShuttle::GetTimeOfLastAction() const
1996 {
1997         ULong_t tmp;
1998
1999         fMonitoringMutex->Lock();
2000
2001         tmp = fLastActionTime;
2002
2003         fMonitoringMutex->UnLock();
2004
2005         return tmp;
2006 }
2007
2008 //______________________________________________________________________________________________
2009 const TString AliShuttle::GetLastAction() const
2010 {
2011         // returns a string description of the last action
2012
2013         TString tmp;
2014
2015         fMonitoringMutex->Lock();
2016
2017         tmp = fLastAction;
2018
2019         fMonitoringMutex->UnLock();
2020
2021         return tmp;
2022 }
2023
2024 //______________________________________________________________________________________________
2025 void AliShuttle::SetLastAction(const char* action)
2026 {
2027         // updates the monitoring variables
2028
2029         fMonitoringMutex->Lock();
2030
2031         fLastAction = action;
2032         fLastActionTime = time(0);
2033
2034         fMonitoringMutex->UnLock();
2035 }
2036
2037 //______________________________________________________________________________________________
2038 const char* AliShuttle::GetRunParameter(const char* param)
2039 {
2040 // returns run parameter read from DAQ logbook
2041
2042         if(!fLogbookEntry) {
2043                 AliError("No logbook entry!");
2044                 return 0;
2045         }
2046
2047         return fLogbookEntry->GetRunParameter(param);
2048 }
2049
2050 //______________________________________________________________________________________________
2051 Bool_t AliShuttle::SendMail()
2052 {
2053 // sends a mail to the subdetector expert in case of preprocessor error
2054
2055         void* dir = gSystem->OpenDirectory(fgkShuttleLogDir);
2056         if (dir == NULL)
2057         {
2058                 if (gSystem->mkdir(fgkShuttleLogDir, kTRUE))
2059                 {
2060                         AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
2061                         return kFALSE;
2062                 }
2063
2064         } else {
2065                 gSystem->FreeDirectory(dir);
2066         }
2067
2068         TString bodyFileName;
2069         bodyFileName.Form("%s/mail.body", fgkShuttleLogDir);
2070         gSystem->ExpandPathName(bodyFileName);
2071
2072         ofstream mailBody;
2073         mailBody.open(bodyFileName, ofstream::out);
2074
2075         if (!mailBody.is_open())
2076         {
2077                 AliError(Form("Could not open mail body file %s", bodyFileName.Data()));
2078                 return kFALSE;
2079         }
2080
2081         TString to="";
2082         TIter iterExperts(fConfig->GetResponsibles(fCurrentDetector));
2083         TObjString *anExpert=0;
2084         while ((anExpert = (TObjString*) iterExperts.Next()))
2085         {
2086                 to += Form("%s,", anExpert->GetName());
2087         }
2088         to.Remove(to.Length()-1);
2089         AliInfo(Form("to: %s",to.Data()));
2090
2091         TString cc="alberto.colla@cern.ch";
2092
2093         TString subject = Form("%s Shuttle preprocessor error in run %d !",
2094                                 fCurrentDetector.Data(), GetCurrentRun());
2095         AliInfo(Form("subject: %s", subject.Data()));
2096
2097         TString body = Form("Dear %s expert(s), \n\n", fCurrentDetector.Data());
2098         body += Form("SHUTTLE just detected that your preprocessor "
2099                         "exited with ERROR state in run %d !!\n\n", GetCurrentRun());
2100         body += Form("Please check %s status on the web page asap!\n\n", fCurrentDetector.Data());
2101         body += Form("The last 10 lines of %s log file are following:\n\n");
2102
2103         AliInfo(Form("Body begin: %s", body.Data()));
2104
2105         mailBody << body.Data();
2106         mailBody.close();
2107         mailBody.open(bodyFileName, ofstream::out | ofstream::app);
2108
2109         TString logFileName = Form("%s/%s.log", fgkShuttleLogDir, fCurrentDetector.Data());
2110         TString tailCommand = Form("tail -n 10 %s >> %s", logFileName.Data(), bodyFileName.Data());
2111         if (gSystem->Exec(tailCommand.Data()))
2112         {
2113                 mailBody << Form("%s log file not found ...\n\n", fCurrentDetector.Data());
2114         }
2115
2116         TString endBody = Form("------------------------------------------------------\n\n");
2117         endBody += Form("In case of problems please contact the SHUTTLE core team!\n\n");
2118         endBody += "Please do not answer this message directly, it is automatically generated!\n\n";
2119         endBody += "Sincerely yours,\n\n \t\t\tthe SHUTTLE\n";
2120
2121         AliInfo(Form("Body end: %s", endBody.Data()));
2122
2123         mailBody << endBody.Data();
2124
2125         mailBody.close();
2126
2127         // send mail!
2128         TString mailCommand = Form("mail -s \"%s\" -c %s %s < %s",
2129                                                 subject.Data(),
2130                                                 cc.Data(),
2131                                                 to.Data(),
2132                                                 bodyFileName.Data());
2133         AliInfo(Form("mail command: %s", mailCommand.Data()));
2134
2135         Bool_t result = gSystem->Exec(mailCommand.Data());
2136
2137         return result == 0;
2138 }