SHUTTLE/AliShuttle.cxx

   1 /**************************************************************************
   2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
   3  *                                                                        *
   4  * Author: The ALICE Off-line Project.                                    *
   5  * Contributors are mentioned in the code where appropriate.              *
   6  *                                                                        *
   7  * Permission to use, copy, modify and distribute this software and its   *
   8  * documentation strictly for non-commercial purposes is hereby granted   *
   9  * without fee, provided that the above copyright notice appears in all   *
  10  * copies and that both the copyright notice and this permission notice   *
  11  * appear in the supporting documentation. The authors make no claims     *
  12  * about the suitability of this software for any purpose. It is          *
  13  * provided "as is" without express or implied warranty.                  *
  14  **************************************************************************/
  15
  16 /*
  17 $Log$
  18 Revision 1.10  2006/07/20 09:54:40  jgrosseo
  19 introducing status management: The processing per subdetector is divided into several steps,
  20 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
  21 can keep track of the number of failures and skips further processing after a certain threshold is
  22 exceeded. These thresholds can be configured in LDAP.
  23
  24 Revision 1.9  2006/07/19 10:09:55  jgrosseo
  25 new configuration, accesst to DAQ FES (Alberto)
  26
  27 Revision 1.8  2006/07/11 12:44:36  jgrosseo
  28 adding parameters for extended validity range of data produced by preprocessor
  29
  30 Revision 1.7  2006/07/10 14:37:09  jgrosseo
  31 small fix + todo comment
  32
  33 Revision 1.6  2006/07/10 13:01:41  jgrosseo
  34 enhanced storing of last sucessfully processed run (alberto)
  35
  36 Revision 1.5  2006/07/04 14:59:57  jgrosseo
  37 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
  38
  39 Revision 1.4  2006/06/12 09:11:16  jgrosseo
  40 coding conventions (Alberto)
  41
  42 Revision 1.3  2006/06/06 14:26:40  jgrosseo
  43 o) removed files that were moved to STEER
  44 o) shuttle updated to follow the new interface (Alberto)
  45
  46 Revision 1.2  2006/03/07 07:52:34  hristov
  47 New version (B.Yordanov)
  48
  49 Revision 1.6  2005/11/19 17:19:14  byordano
  50 RetrieveDATEEntries and RetrieveConditionsData added
  51
  52 Revision 1.5  2005/11/19 11:09:27  byordano
  53 AliShuttle declaration added
  54
  55 Revision 1.4  2005/11/17 17:47:34  byordano
  56 TList changed to TObjArray
  57
  58 Revision 1.3  2005/11/17 14:43:23  byordano
  59 import to local CVS
  60
  61 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
  62 Initial import as subdirectory in AliRoot
  63
  64 Revision 1.2  2005/09/13 08:41:15  byordano
  65 default startTime endTime added
  66
  67 Revision 1.4  2005/08/30 09:13:02  byordano
  68 some docs added
  69
  70 Revision 1.3  2005/08/29 21:15:47  byordano
  71 some docs added
  72
  73 */
  74
  75 //
  76 // This class is the main manager for AliShuttle.
  77 // It organizes the data retrieval from DCS and call the
  78 // interface methods of AliPreprocessor.
  79 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
  80 // data for its set of aliases is retrieved. If there is registered
  81 // AliPreprocessor for this detector then it will be used
  82 // accroding to the schema (see AliPreprocessor).
  83 // If there isn't registered AliPreprocessor than the retrieved
  84 // data is stored automatically to the undelying AliCDBStorage.
  85 // For detSpec is used the alias name.
  86 //
  87
  88 #include "AliShuttle.h"
  89
  90 #include "AliCDBManager.h"
  91 #include "AliCDBStorage.h"
  92 #include "AliCDBId.h"
  93 #include "AliCDBEntry.h"
  94 #include "AliShuttleConfig.h"
  95 #include "AliDCSClient.h"
  96 #include "AliLog.h"
  97 #include "AliPreprocessor.h"
  98 #include "AliDefaultPreprocessor.h"
  99 #include "AliShuttleStatus.h"
 100
 101 #include <TSystem.h>
 102 #include <TObject.h>
 103 #include <TString.h>
 104 #include <TTimeStamp.h>
 105 #include <TObjString.h>
 106 #include <TSQLServer.h>
 107 #include <TSQLResult.h>
 108 #include <TSQLRow.h>
 109
 110 #include <fstream>
 111
 112 ClassImp(AliShuttle)
 113
 114 TString AliShuttle::fgkLocalUri("local://$ALICE_ROOT/SHUTTLE/ShuttleCDB");
 115 const char* AliShuttle::fgkShuttleTempDir = "$ALICE_ROOT/SHUTTLE/temp";
 116 const char* AliShuttle::fgkShuttleLogDir = "$ALICE_ROOT/SHUTTLE/log";
 117
 118 const char* AliShuttle::fgkDetectorName[AliShuttle::fgkNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
 119         "PHOS", "CPV", "RICH", "EMCAL", "MUON_TRK", "MUON_TRG", "FMD", "ZDC", "PMD", "START", "VZERO"};
 120
 121 const char* AliShuttle::fgkDetectorCode[AliShuttle::fgkNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
 122         "PHS", "CPV", "HMP", "EMC", "MCH", "MTR", "FMD", "ZDC", "PMD", "T00", "V00"};
 123
 124 //______________________________________________________________________________________________
 125 AliShuttle::AliShuttle(const AliShuttleConfig* config,
 126                 UInt_t timeout, Int_t retries):
 127         fConfig(config),
 128         fTimeout(timeout),
 129         fRetries(retries), fCurrentRun(-1), fCurrentStartTime(0),
 130         fCurrentEndTime(0),
 131   fStatusEntry(0)
 132 {
 133         //
 134         // config: AliShuttleConfig used
 135         // timeout: timeout used for AliDCSClient connection
 136         // retries: the number of retries in case of connection error.
 137         //
 138
 139         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
 140         for(int iSys=0;iSys<3;iSys++) {
 141                 fServer[iSys]=0;
 142                 fFESlist[iSys].SetOwner(kTRUE);
 143         }
 144 }
 145
 146 //______________________________________________________________________
 147 AliShuttle::AliShuttle(const AliShuttle& /*other*/):
 148 AliShuttleInterface()
 149 {
 150 // copy constructor (not implemented)
 151
 152 }
 153
 154 //______________________________________________________________________
 155 AliShuttle &AliShuttle::operator=(const AliShuttle& /*other*/)
 156 {
 157 // assignment operator (not implemented)
 158
 159 return *this;
 160 }
 161
 162 //______________________________________________________________________________________________
 163 AliShuttle::~AliShuttle()
 164 {
 165 // destructor
 166
 167         fPreprocessorMap.DeleteAll();
 168         for(int iSys=0;iSys<3;iSys++)
 169                 if(fServer[iSys]) {
 170                         fServer[iSys]->Close();
 171                         delete fServer[iSys];
 172                 }
 173 }
 174
 175 //______________________________________________________________________________________________
 176 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
 177 {
 178         //
 179         // Registers new AliPreprocessor.
 180         // It uses GetName() for indentificator of the pre processor.
 181         // The pre processor is registered it there isn't any other
 182         // with the same identificator (GetName()).
 183         //
 184
 185         if (fPreprocessorMap.GetValue(preprocessor->GetName())) {
 186                 AliWarning(Form("AliPreprocessor %s is already registered!",
 187                         preprocessor->GetName()));
 188                 return;
 189         }
 190
 191         fPreprocessorMap.Add(new TObjString(preprocessor->GetName()), preprocessor);
 192 }
 193
 194 //______________________________________________________________________________________________
 195 UInt_t AliShuttle::Store(const char* detector,
 196                 TObject* object, AliCDBMetaData* metaData, Int_t /*validityStart*/, Bool_t /*validityInfinite*/)
 197 {
 198         // store data into CDB
 199   //
 200   // validityStart is the start validity of the data, if not 0 GetCurrentRun() - validityStart is taken
 201   // validityInfinite defines if the data is valid until new data arrives (e.g. for calibration runs)
 202   //
 203         // returns 0 if fail
 204         //         1 if stored in main (Grid) storage
 205         //         2 if stored in backup (Local) storage
 206
 207   // TODO implement use of two parameters
 208
 209   // TODO shouldn't the path be given by the preprocessor???
 210         AliCDBId id(AliCDBPath(detector, "DCS", "Data"),
 211                 GetCurrentRun(), GetCurrentRun());
 212
 213         UInt_t result = 0;
 214         if (!(AliCDBManager::Instance()->IsDefaultStorageSet())) {
 215                 Log(detector, "No CDB storage set!");
 216         } else {
 217                 result = (UInt_t) AliCDBManager::Instance()->Put(object, id, metaData);
 218         }
 219         if(!result) {
 220
 221                 Log(detector, "Error while storing object in main storage!");
 222                 AliError("local storage will be used!");
 223
 224                 result = AliCDBManager::Instance()->GetStorage(fgkLocalUri)
 225                                         ->Put(object, id, metaData);
 226
 227                 if(result) {
 228                         result = 2;
 229                 }else{
 230                         Log(detector, "Can't store data!");
 231                 }
 232         }
 233         return result;
 234
 235 }
 236
 237 //______________________________________________________________________________________________
 238 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
 239 {
 240   // Reads the AliShuttleStatus from the CDB
 241
 242   if (fStatusEntry)
 243   {
 244     delete fStatusEntry;
 245     fStatusEntry = 0;
 246   }
 247
 248   fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalURI())
 249       ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), fCurrentRun);
 250
 251   if (!fStatusEntry)
 252     return 0;
 253
 254   TObject* anObject = fStatusEntry->GetObject();
 255   if (anObject == NULL || anObject->IsA() != AliShuttleStatus::Class())
 256   {
 257     AliError("Invalid object stored to CDB!");
 258     return 0;
 259   }
 260
 261   AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (anObject);
 262   return status;
 263 }
 264
 265 //______________________________________________________________________________________________
 266 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
 267 {
 268   // writes the status for one subdetector
 269
 270   if (fStatusEntry)
 271   {
 272     delete fStatusEntry;
 273     fStatusEntry = 0;
 274   }
 275
 276   AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), fCurrentRun, fCurrentRun);
 277
 278   fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
 279
 280   UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalUri)->Put(fStatusEntry);
 281
 282   if (!result)
 283   {
 284     AliError(Form("WriteShuttleStatus for %s, run %d failed", fCurrentDetector.Data(), fCurrentRun));
 285     return kFALSE;
 286   }
 287
 288   return kTRUE;
 289 }
 290
 291 //______________________________________________________________________________________________
 292 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
 293 {
 294   // changes the AliShuttleStatus for the given detector and run to the given status
 295
 296   if (!fStatusEntry)
 297   {
 298     AliError("UNEXPECTED: fStatusEntry empty");
 299     return;
 300   }
 301
 302   TObject* anObject = fStatusEntry->GetObject();
 303   AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (anObject);
 304
 305   if (!status)
 306   {
 307     AliError("UNEXPECTED: status could not be read from current CDB entry");
 308     return;
 309   }
 310
 311   Log("SHUTTLE", Form("%s: Changing state from %s to %s", fCurrentDetector.Data(),
 312                                 status->GetStatusName(), status->GetStatusName(newStatus)));
 313
 314   status->SetStatus(newStatus);
 315   if (increaseCount)
 316     status->IncreaseCount();
 317
 318   AliCDBManager::Instance()->GetStorage(fgkLocalUri)->Put(fStatusEntry);
 319 }
 320
 321 //______________________________________________________________________________________________
 322 Bool_t AliShuttle::ContinueProcessing()
 323 {
 324   // this function reads the AliShuttleStatus information from CDB and
 325   // checks if the processing should be continued
 326   // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
 327
 328   AliShuttleStatus* status = ReadShuttleStatus();
 329   if (!status)
 330   {
 331     // first time
 332
 333     Log("SHUTTLE", Form("%s: Processing first time.", fCurrentDetector.Data()));
 334     status = new AliShuttleStatus(AliShuttleStatus::kStarted);
 335     return WriteShuttleStatus(status);
 336   }
 337
 338   if (status->GetStatus() == AliShuttleStatus::kDone)
 339   {
 340     Log("SHUTTLE", Form("%s already done for run %d", fCurrentDetector.Data(), fCurrentRun));
 341     return kFALSE;
 342   }
 343
 344   if (status->GetStatus() == AliShuttleStatus::kFailed)
 345   {
 346     Log("SHUTTLE", Form("%s already in failed state for run %d", fCurrentDetector.Data(), fCurrentRun));
 347     return kFALSE;
 348   }
 349
 350   // if we get here, there is a restart
 351
 352   // abort conditions
 353   if (status->GetStatus() == AliShuttleStatus::kPPStarted && status->GetCount() >= fConfig->GetMaxPPRetries() ||
 354       status->GetCount() >= fConfig->GetMaxRetries())
 355   {
 356     Log("SHUTTLE", Form("%s, run %d failed too often, %d times, status %s. Skipping processing.",
 357                 fCurrentDetector.Data(), fCurrentRun, status->GetCount(), status->GetStatusName()));
 358
 359     return kFALSE;
 360   }
 361
 362   Log("SHUTTLE", Form("Restart of %s, run %d. Got stuck before in %s, count %d",
 363                 fCurrentDetector.Data(), fCurrentRun, status->GetStatusName(), status->GetCount()));
 364
 365   UpdateShuttleStatus(AliShuttleStatus::kStarted, kTRUE);
 366
 367   return kTRUE;
 368 }
 369
 370 //______________________________________________________________________________________________
 371 Bool_t AliShuttle::Process(Int_t run, UInt_t startTime, UInt_t endTime)
 372 {
 373         //
 374         // Makes data retrieval for all detectors in the configuration.
 375         // run: is the run number used
 376         // startTime: is the run start time
 377         // endTime: is the run end time
 378         // Returns kFALSE in case of error occured and kTRUE otherwise
 379         //
 380
 381         AliInfo(Form("\n\n ^*^*^*^*^*^* Processing run %d ^*^*^*^*^*^*", run));
 382
 383         // Initialization
 384         Bool_t hasError = kFALSE;
 385         for(Int_t iSys=0;iSys<3;iSys++) fFESCalled[iSys]=kFALSE;
 386
 387         fCurrentRun = run;
 388         fCurrentStartTime = startTime;
 389         fCurrentEndTime = endTime;
 390
 391         // Loop on detectors in the configuration
 392         TIter iter(fConfig->GetDetectors());
 393         TObjString* aDetector;
 394
 395         while ((aDetector = (TObjString*) iter.Next())) {
 396                 fCurrentDetector = aDetector->String();
 397
 398                 Bool_t detectorError=kFALSE;
 399                 if (!fConfig->HostProcessDetector(fCurrentDetector)) continue;
 400
 401                 if (ContinueProcessing() == kFALSE) continue;
 402
 403                 if(!Process()) {
 404                         hasError = kTRUE;
 405                         detectorError=kTRUE;
 406                         continue;
 407                 }
 408                 AliInfo(Form("Process ended successfully for detector %s!",aDetector->GetName()));
 409
 410                 // Process successful: Update time_processed field in FES logbooks!
 411                 if(fFESCalled[kDAQ]) {
 412                         hasError = (UpdateDAQTable() == kFALSE);
 413                         fFESlist[kDAQ].Clear();
 414                 }
 415                 //if(fFESCalled[kDCS]) {
 416                 //      hasError = UpdateDCSTable(aDetector->GetName());
 417                 //      fFESlist[kDCS].Clear();
 418                 //}
 419                 //if(fFESCalled[kHLT]) {
 420                 //      hasError = UpdateHLTTable(aDetector->GetName());
 421                 //      fFESlist[kHLT].Clear();
 422                 //}
 423
 424                 UpdateShuttleStatus(AliShuttleStatus::kDone);
 425         }
 426
 427         fCurrentRun = -1;
 428         fCurrentStartTime = 0;
 429         fCurrentEndTime = 0;
 430
 431         return hasError == kFALSE;
 432 }
 433
 434 //______________________________________________________________________________________________
 435 Bool_t AliShuttle::Process()
 436 {
 437         //
 438         // Makes data retrieval just for one specific detector.
 439         // Threre should be a configuration for this detector.
 440         // run: is the run number used
 441         // startTime: is the run start time
 442         // endTime: is the run end time
 443         // detector: detector for which the retrieval will be made
 444         // Returns kFALSE in case of error occured and kTRUE otherwise
 445         //
 446
 447         AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), fCurrentRun));
 448
 449         if (!fConfig->HasDetector(fCurrentDetector)) {
 450                 Log(fCurrentDetector, "There isn't any configuration for %s !");
 451                 UpdateShuttleStatus(AliShuttleStatus::kFailed);
 452                 return kFALSE;
 453         }
 454
 455         UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
 456
 457         TString host(fConfig->GetDCSHost(fCurrentDetector));
 458         Int_t port = fConfig->GetDCSPort(fCurrentDetector);
 459
 460         TIter iter(fConfig->GetDCSAliases(fCurrentDetector));
 461         TObjString* anAlias;
 462         TMap aliasMap;
 463
 464         Bool_t hasError = kFALSE;
 465         Bool_t result=kFALSE;
 466
 467         while ((anAlias = (TObjString*) iter.Next())) {
 468                 TObjArray valueSet;
 469                 result = GetValueSet(host, port, anAlias->String(), valueSet);
 470                 //AliInfo(Form("Port = %d",port));
 471                 //result = kTRUE;
 472                 if(result) {
 473                         aliasMap.Add(anAlias->Clone(), valueSet.Clone());
 474                 }else{
 475                         TString message = Form("Error while retrieving alias %s !",
 476                                         anAlias->GetName());
 477                         Log(fCurrentDetector, message.Data());
 478                         hasError = kTRUE;
 479                 }
 480         }
 481
 482         if (hasError)
 483         {
 484                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
 485                 return kFALSE;
 486         }
 487
 488   UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
 489
 490   AliPreprocessor* aPreprocessor =
 491                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
 492         if(aPreprocessor)
 493         {
 494                 aPreprocessor->Initialize(fCurrentRun, fCurrentStartTime, fCurrentEndTime);
 495                 hasError = (aPreprocessor->Process(&aliasMap) == 0);
 496         }else{
 497     // TODO default behaviour?
 498                 AliInfo(Form("No Preprocessor for %s: storing TMap of DP arrays into CDB!", fCurrentDetector.Data()));
 499                 AliCDBMetaData metaData;
 500                 AliDCSValue dcsValue(fCurrentStartTime, fCurrentEndTime);
 501                 metaData.SetResponsible(Form("Duck, Donald"));
 502                 metaData.SetProperty("StartEndTime", &dcsValue);
 503                 metaData.SetComment("Automatically stored by Shuttle!");
 504                 hasError = (Store(fCurrentDetector, &aliasMap, &metaData) == 0);
 505         }
 506
 507   if (hasError)
 508     UpdateShuttleStatus(AliShuttleStatus::kPPError);
 509   else
 510     UpdateShuttleStatus(AliShuttleStatus::kPPDone);
 511
 512         aliasMap.Delete();
 513
 514         return hasError == kFALSE;
 515 }
 516
 517 //______________________________________________________________________________________________
 518 Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* alias,
 519                                 TObjArray& valueSet)
 520 {
 521 // Retrieve all "alias" data points from the DCS server
 522 // host, port: TSocket connection parameters
 523 // alias: name of the alias
 524 // valueSet: array of retrieved AliDCSValue's
 525
 526         AliDCSClient client(host, port, fTimeout, fRetries);
 527         if (!client.IsConnected()) {
 528                 return kFALSE;
 529         }
 530
 531         Int_t result = client.GetAliasValues(alias,
 532                 GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
 533
 534         if (result < 0) {
 535                 AliError(Form("Can't get '%s'! Reason: %s",
 536                         alias, AliDCSClient::GetErrorString(result)));
 537
 538                 if (result == AliDCSClient::fgkServerError) {
 539                         AliError(Form("Server error: %s",
 540                                 client.GetServerError().Data()));
 541                 }
 542
 543                 return kFALSE;
 544         }
 545
 546         return kTRUE;
 547 }
 548
 549 //______________________________________________________________________________________________
 550 const char* AliShuttle::GetFile(Int_t system, const char* detector,
 551                 const char* id, const char* source)
 552 {
 553 // Get calibration file from file exchange servers
 554 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
 555
 556         switch(system){
 557                 case kDAQ:
 558                         return GetDAQFileName(detector, id, source);
 559                         break;
 560                 case kDCS:
 561                         return GetDCSFileName(detector, id, source);
 562                         break;
 563                 case kHLT:
 564                         return GetHLTFileName(detector, id, source);
 565                         break;
 566                 default:
 567                         AliError(Form("No valid system index: %d",system));
 568         }
 569
 570         return 0;
 571 }
 572
 573 //______________________________________________________________________________________________
 574 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
 575 {
 576 // Get sources producing the condition file Id from file exchange servers
 577 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
 578
 579         switch(system){
 580                 case kDAQ:
 581                         return GetDAQFileSources(detector, id);
 582                         break;
 583                 case kDCS:
 584                         return GetDCSFileSources(detector, id);
 585                         break;
 586                 case kHLT:
 587                         return GetHLTFileSources(detector, id);
 588                         break;
 589                 default:
 590                         AliError(Form("No valid system index: %d",system));
 591         }
 592
 593         return NULL;
 594 }
 595
 596 //______________________________________________________________________________________________
 597 Bool_t AliShuttle::Connect(Int_t system){
 598 // Connect to MySQL Server of the system's FES logbook
 599
 600         // check connection: if already connected return
 601         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
 602
 603         TString aFESlbHost= Form("mysql://%s", fConfig->GetFESlbHost(system));
 604
 605         fServer[system] = TSQLServer::Connect(aFESlbHost,
 606                         fConfig->GetFESlbUser(system),
 607                         fConfig->GetFESlbPass(system));
 608         if (!fServer[system] || !fServer[system]->IsConnected()) {
 609                 AliError(Form("Can't establish connection to FES logbook for %s !",fkSystemNames[system]));
 610                 return kFALSE;
 611         }
 612
 613         // Get tables
 614         // TODO in the configuration should the table name be there too?
 615         switch(system){
 616                 case kDAQ:
 617                         fServer[kDAQ]->GetTables("REFSYSLOG");
 618                         break;
 619                 case kDCS:
 620                         //fServer[kDCS]->GetTables("REFSYSLOG");
 621                         break;
 622                 case kHLT:
 623                         //fServer[kHLT]->GetTables("REFSYSLOG");
 624                         break;
 625                 default:
 626                         break;
 627         }
 628
 629         return kTRUE;
 630 }
 631
 632 //______________________________________________________________________________________________
 633 const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source){
 634 // Retrieves a file from the DAQ FES.
 635 // First queris the DAQ logbook_fs for the DAQ file name, using the run, detector, id and source info
 636 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
 637 // run: current run being processed (fCurrentRun)
 638 // detector: comes from the Preprocessor name (must be converted into detector code with GetDetCode)
 639 // id: provided as a parameter by the Preprocessor
 640 // source: provided by the Preprocessor through GetFileSources function
 641
 642         // check connection, in case connect
 643         if(!Connect(kDAQ)){
 644                 Log(detector, "GetDAQFileName: Couldn't connect to DAQ Logbook !");
 645                 return 0;
 646         }
 647
 648         // Query preparation
 649         TString sqlQueryStart = "select filePath from logbook_fs where";
 650         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
 651                                 fCurrentRun, GetDetCode(detector), id, source);
 652         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
 653
 654         AliInfo(Form("SQL query: \n%s",sqlQuery.Data()));
 655
 656         // Query execution
 657         TSQLResult* aResult;
 658         aResult = fServer[kDAQ]->Query(sqlQuery);
 659         if (!aResult) {
 660                 Log(detector, Form("Can't execute query <%s>!", sqlQuery.Data()));
 661                 return 0;
 662         }
 663
 664         if (aResult->GetRowCount() == 0) {
 665                 Log(detector,
 666                         Form("GetDAQFileName: No result from SQL query <%s>!", sqlQuery.Data()));
 667                 delete aResult;
 668                 return 0;
 669         }
 670
 671         if (aResult->GetRowCount() >1) {
 672                 Log(detector,
 673                         Form("GetDAQFileName: More than one row resulting from SQL query <%s>!", sqlQuery.Data()));
 674                 delete aResult;
 675                 return 0;
 676         }
 677
 678         TSQLRow* aRow = aResult->Next();
 679
 680         if(!aRow){
 681                 Log(detector, Form("GetDAQFileName: Empty set result from query <%s>!", sqlQuery.Data()));
 682                 delete aResult;
 683                 return 0;
 684         }
 685
 686         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
 687
 688         delete aResult;
 689
 690         AliInfo(Form("filePath = %s",filePath.Data()));
 691
 692         // retrieved file is renamed to make it unique
 693         TString localFileName = Form("%s_%d_%s_%s.shuttle",
 694                                         detector, fCurrentRun, id, source);
 695
 696         // file retrieval from DAQ FES
 697         Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
 698         if(!result) {
 699                 Log(detector, Form("copying file %s from DAQ FES failed!", filePath.Data()));
 700                 return 0;
 701         } else {
 702                 AliInfo(Form("File %s copied from DAQ FES into %s/%s !",
 703                         filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
 704         }
 705
 706
 707         fFESCalled[kDAQ]=kTRUE;
 708         TObjString *fileParams = new TObjString(Form("%s_!?!_%s", id, source));
 709         fFESlist[kDAQ].Add(fileParams);
 710
 711         return localFileName.Data();
 712
 713 }
 714
 715 //______________________________________________________________________________________________
 716 Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName){
 717
 718         // check temp directory: trying to cd to temp; if it does not exist, create it
 719         AliInfo(Form("Copy file %s from DAQ FES into folder %s and rename it as %s",
 720                         daqFileName,fgkShuttleTempDir, localFileName));
 721
 722         void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
 723         if (dir == NULL) {
 724                 if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
 725                         AliError(Form("Can't open directory <%s>!", fgkShuttleTempDir));
 726                         return kFALSE;
 727                 }
 728
 729         } else {
 730                 gSystem->FreeDirectory(dir);
 731         }
 732
 733         TString baseDAQFESFolder = "DAQ";
 734         TString command = Form("scp %s@%s:%s/%s %s/%s",
 735                 fConfig->GetFESUser(kDAQ),
 736                 fConfig->GetFESHost(kDAQ),
 737                 baseDAQFESFolder.Data(),
 738                 daqFileName,
 739                 fgkShuttleTempDir,
 740                 localFileName);
 741
 742         AliInfo(Form("%s",command.Data()));
 743
 744         UInt_t nRetries = 0;
 745         UInt_t maxRetries = 3;
 746
 747         // copy!! if successful TSystem::Exec returns 0
 748         while(nRetries++ < maxRetries) {
 749                 AliInfo(Form("Trying to copy file. Retry # %d", nRetries));
 750                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
 751         }
 752
 753         return kFALSE;
 754
 755 }
 756
 757 //______________________________________________________________________________________________
 758 TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id){
 759 // Retrieves a file from the DCS FES.
 760
 761         // check connection, in case connect
 762         if(!Connect(kDAQ)){
 763                 Log(detector, "GetDAQFileName: Couldn't connect to DAQ Logbook !");
 764                 return 0;
 765         }
 766
 767         // Query preparation
 768         TString sqlQueryStart = "select DAQsource from logbook_fs where";
 769         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
 770                                 fCurrentRun, GetDetCode(detector), id);
 771         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
 772
 773         AliInfo(Form("SQL query: \n%s",sqlQuery.Data()));
 774
 775         // Query execution
 776         TSQLResult* aResult;
 777         aResult = fServer[kDAQ]->Query(sqlQuery);
 778         if (!aResult) {
 779                 Log(detector, Form("GetDAQFileSources: Can't execute query <%s>!", sqlQuery.Data()));
 780                 return 0;
 781         }
 782
 783         if (aResult->GetRowCount() == 0) {
 784                 Log(detector,
 785                         Form("GetDAQFileSources: No result from SQL query <%s>!", sqlQuery.Data()));
 786                 delete aResult;
 787                 return 0;
 788         }
 789
 790         TSQLRow* aRow;
 791         TList *list = new TList();
 792         list->SetOwner(1);
 793
 794         while((aRow = aResult->Next())){
 795
 796                 TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
 797                 AliInfo(Form("daqSource = %s", daqSource.Data()));
 798                 list->Add(new TObjString(daqSource));
 799         }
 800         delete aResult;
 801
 802         return list;
 803
 804 }
 805
 806 //______________________________________________________________________________________________
 807 Bool_t AliShuttle::UpdateDAQTable(){
 808 // Update DAQ table filling time_processed field in all rows corresponding to current run and detector
 809
 810         // check connection, in case connect
 811         if(!Connect(kDAQ)){
 812                 Log(fCurrentDetector, "UpdateDAQTable: Couldn't connect to DAQ Logbook !");
 813                 return kFALSE;
 814         }
 815
 816         TTimeStamp now; // now
 817
 818         // Loop on FES list entries
 819         TIter iter(&fFESlist[kDAQ]);
 820         TObjString *aFESentry=0;
 821         while((aFESentry = dynamic_cast<TObjString*> (iter.Next()))){
 822                 TString aFESentrystr = aFESentry->String();
 823                 TObjArray *aFESarray = aFESentrystr.Tokenize("_!?!_");
 824                 if(!aFESarray || aFESarray->GetEntries() != 2 ) {
 825                         Log(fCurrentDetector,Form("UpdateDAQTable: error updating FES entry! string = %s",
 826                                 aFESentrystr.Data()));
 827                         if(aFESarray) delete aFESarray;
 828                         return kFALSE;
 829                 }
 830                 const char* fileId = ((TObjString*) aFESarray->At(0))->GetName();
 831                 const char* daqSource = ((TObjString*) aFESarray->At(1))->GetName();
 832                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
 833                         fCurrentRun,GetDetCode(fCurrentDetector), fileId, daqSource);
 834
 835                 delete aFESarray;
 836
 837                 TString sqlQuery = Form("update logbook_fs set time_processed=%d %s", now.GetSec(), whereClause.Data());
 838
 839                 AliInfo(Form("SQL query: \n%s",sqlQuery.Data()));
 840
 841                 // Query execution
 842                 TSQLResult* aResult;
 843                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
 844                 if (!aResult) {
 845                         Log(fCurrentDetector, Form("UpdateDAQTable: Can't execute query <%s>!", sqlQuery.Data()));
 846                         return kFALSE;
 847                 }
 848                 delete aResult;
 849
 850                 // check result - TODO Is it necessary?
 851                 sqlQuery = Form("select time_processed from logbook_fs %s", whereClause.Data());
 852                 AliInfo(Form(" CHECK - SQL query: \n%s",sqlQuery.Data()));
 853
 854                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
 855                 if (!aResult) {
 856                         AliWarning("Can't check result!");
 857                         continue;
 858                 }
 859
 860         if (aResult->GetRowCount() == 0) {
 861                 Log(fCurrentDetector,
 862                         Form("GetDAQFileName: No result from SQL query <%s>!", sqlQuery.Data()));
 863                 delete aResult;
 864                 //return 0;
 865         }
 866
 867         if (aResult->GetRowCount() >1) {
 868                 Log(fCurrentDetector,
 869                         Form("GetDAQFileName: More than one row resulting from SQL query <%s>!", sqlQuery.Data()));
 870                 delete aResult;
 871                 //return 0;
 872         }
 873
 874                 TSQLRow *row = dynamic_cast<TSQLRow*> (aResult->Next());
 875                 TString processedTimeString(row->GetField(0), row->GetFieldLength(0));
 876                 Int_t processedTime = processedTimeString.Atoi();
 877                 if(processedTime != now.GetSec()){
 878                         Log(fCurrentDetector, Form("UpdateDAQTable: Update table error: processed_time=%d, now=%d !",
 879                                 processedTime, now.GetSec()));
 880                         delete aResult;
 881                         return kFALSE;
 882                 }
 883
 884                 delete aResult;
 885
 886         }
 887
 888         return kTRUE;
 889 }
 890
 891 //______________________________________________________________________________________________
 892 const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
 893 // Retrieves a file from the DCS FES.
 894
 895 return "You're in DCS";
 896
 897 }
 898
 899 //______________________________________________________________________________________________
 900 TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
 901 // Retrieves a file from the DCS FES.
 902
 903 return NULL;
 904
 905 }
 906
 907 //______________________________________________________________________________________________
 908 const char* AliShuttle::GetHLTFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
 909 // Retrieves a file from the HLT FES.
 910
 911 return "You're in HLT";
 912
 913 }
 914
 915 //______________________________________________________________________________________________
 916 TList* AliShuttle::GetHLTFileSources(const char* /*detector*/, const char* /*id*/){
 917 // Retrieves a file from the HLT FES.
 918
 919 return NULL;
 920
 921 }
 922
 923 //______________________________________________________________________________________________
 924 const char* AliShuttle::GetDetCode(const char* detector){
 925 // Return detector code
 926
 927         for(int iDet=0; iDet < fgkNDetectors; iDet++){
 928                 if(!strcmp(fgkDetectorName[iDet], detector)) return fgkDetectorCode[iDet];
 929         }
 930
 931         return 0;
 932 }
 933
 934 //______________________________________________________________________________________________
 935 void AliShuttle::Log(const char* detector, const char* message)
 936 {
 937 // Fill log string with a message
 938
 939   TString toLog = Form("%s: %s, run %d - %s", TTimeStamp(time(0)).AsString(),
 940     detector, GetCurrentRun(), message);
 941   AliInfo(toLog.Data());
 942
 943   TString fileName;
 944   fileName.Form("%s/%s.log", fgkShuttleLogDir, detector);
 945   gSystem->ExpandPathName(fileName);
 946
 947   ofstream logFile;
 948   logFile.open(fileName, ofstream::out | ofstream::app);
 949
 950   if (!logFile.is_open())
 951   {
 952     AliError(Form("Could not open file %s", fileName.Data()));
 953     return;
 954   }
 955
 956   logFile << toLog.Data() << "\n";
 957
 958   logFile.close();
 959 }