SHUTTLE/AliShuttle.cxx

   1 /**************************************************************************
   2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
   3  *                                                                        *
   4  * Author: The ALICE Off-line Project.                                    *
   5  * Contributors are mentioned in the code where appropriate.              *
   6  *                                                                        *
   7  * Permission to use, copy, modify and distribute this software and its   *
   8  * documentation strictly for non-commercial purposes is hereby granted   *
   9  * without fee, provided that the above copyright notice appears in all   *
  10  * copies and that both the copyright notice and this permission notice   *
  11  * appear in the supporting documentation. The authors make no claims     *
  12  * about the suitability of this software for any purpose. It is          *
  13  * provided "as is" without express or implied warranty.                  *
  14  **************************************************************************/
  15
  16 /*
  17 $Log$
  18 Revision 1.39  2007/04/17 12:43:57  acolla
  19 Correction in StoreOCDB; change of text in mail to detector expert
  20
  21 Revision 1.38  2007/04/12 08:26:18  jgrosseo
  22 updated comment
  23
  24 Revision 1.37  2007/04/10 16:53:14  jgrosseo
  25 redirecting sub detector stdout, stderr to sub detector log file
  26
  27 Revision 1.35  2007/04/04 16:26:38  acolla
  28 1. Re-organization of function calls in TestPreprocessor to make it more meaningful.
  29 2. Added missing dependency in test preprocessors.
  30 3. in AliShuttle.cxx: processing time and memory consumption info on a single line.
  31
  32 Revision 1.34  2007/04/04 10:33:36  jgrosseo
  33 1) Storing of files to the Grid is now done _after_ your preprocessors succeeded. This is transparent, which means that you can still use the same functions (Store, StoreReferenceData) to store files to the Grid. However, the Shuttle first stores them locally and transfers them after the preprocessor finished. The return code of these two functions has changed from UInt_t to Bool_t which gives you the success of the storing.
  34 In case of an error with the Grid, the Shuttle will retry the storing later, the preprocessor does not need to be run again.
  35
  36 2) The meaning of the return code of the preprocessor has changed. 0 is now success and any other value means failure. This value is stored in the log and you can use it to keep details about the error condition.
  37
  38 3) New function StoreReferenceFile to _directly_ store a file (without opening it) to the reference storage.
  39
  40 4) The memory usage of the preprocessor is monitored. If it exceeds 2 GB it is terminated.
  41
  42 5) New function AliPreprocessor::ProcessDCS(). If you do not need to have DCS data in all cases, you can skip the processing by implemting this function and returning kFALSE under certain conditions. E.g. if there is a certain run type.
  43 If you always need DCS data (like before), you do not need to implement it.
  44
  45 6) The run type has been added to the monitoring page
  46
  47 Revision 1.33  2007/04/03 13:56:01  acolla
  48 Grid Storage at the end of preprocessing. Added virtual method to disable DCS query according to the
  49 run type.
  50
  51 Revision 1.32  2007/02/28 10:41:56  acolla
  52 Run type field added in SHUTTLE framework. Run type is read from "run type" logbook and retrieved by
  53 AliPreprocessor::GetRunType() function.
  54 Added some ldap definition files.
  55
  56 Revision 1.30  2007/02/13 11:23:21  acolla
  57 Moved getters and setters of Shuttle's main OCDB/Reference, local
  58 OCDB/Reference, temp and log folders to AliShuttleInterface
  59
  60 Revision 1.27  2007/01/30 17:52:42  jgrosseo
  61 adding monalisa monitoring
  62
  63 Revision 1.26  2007/01/23 19:20:03  acolla
  64 Removed old ldif files, added TOF, MCH ldif files. Added some options in
  65 AliShuttleConfig::Print. Added in Ali Shuttle: SetShuttleTempDir and
  66 SetShuttleLogDir
  67
  68 Revision 1.25  2007/01/15 19:13:52  acolla
  69 Moved some AliInfo to AliDebug in SendMail function
  70
  71 Revision 1.21  2006/12/07 08:51:26  jgrosseo
  72 update (alberto):
  73 table, db names in ldap configuration
  74 added GRP preprocessor
  75 DCS data can also be retrieved by data point
  76
  77 Revision 1.20  2006/11/16 16:16:48  jgrosseo
  78 introducing strict run ordering flag
  79 removed giving preprocessor name to preprocessor, they have to know their name themselves ;-)
  80
  81 Revision 1.19  2006/11/06 14:23:04  jgrosseo
  82 major update (Alberto)
  83 o) reading of run parameters from the logbook
  84 o) online offline naming conversion
  85 o) standalone DCSclient package
  86
  87 Revision 1.18  2006/10/20 15:22:59  jgrosseo
  88 o) Adding time out to the execution of the preprocessors: The Shuttle forks and the parent process monitors the child
  89 o) Merging Collect, CollectAll, CollectNew function
  90 o) Removing implementation of empty copy constructors (declaration still there!)
  91
  92 Revision 1.17  2006/10/05 16:20:55  jgrosseo
  93 adapting to new CDB classes
  94
  95 Revision 1.16  2006/10/05 15:46:26  jgrosseo
  96 applying to the new interface
  97
  98 Revision 1.15  2006/10/02 16:38:39  jgrosseo
  99 update (alberto):
 100 fixed memory leaks
 101 storing of objects that failed to be stored to the grid before
 102 interfacing of shuttle status table in daq system
 103
 104 Revision 1.14  2006/08/29 09:16:05  jgrosseo
 105 small update
 106
 107 Revision 1.13  2006/08/15 10:50:00  jgrosseo
 108 effc++ corrections (alberto)
 109
 110 Revision 1.12  2006/08/08 14:19:29  jgrosseo
 111 Update to shuttle classes (Alberto)
 112
 113 - Possibility to set the full object's path in the Preprocessor's and
 114 Shuttle's  Store functions
 115 - Possibility to extend the object's run validity in the same classes
 116 ("startValidity" and "validityInfinite" parameters)
 117 - Implementation of the StoreReferenceData function to store reference
 118 data in a dedicated CDB storage.
 119
 120 Revision 1.11  2006/07/21 07:37:20  jgrosseo
 121 last run is stored after each run
 122
 123 Revision 1.10  2006/07/20 09:54:40  jgrosseo
 124 introducing status management: The processing per subdetector is divided into several steps,
 125 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
 126 can keep track of the number of failures and skips further processing after a certain threshold is
 127 exceeded. These thresholds can be configured in LDAP.
 128
 129 Revision 1.9  2006/07/19 10:09:55  jgrosseo
 130 new configuration, accesst to DAQ FES (Alberto)
 131
 132 Revision 1.8  2006/07/11 12:44:36  jgrosseo
 133 adding parameters for extended validity range of data produced by preprocessor
 134
 135 Revision 1.7  2006/07/10 14:37:09  jgrosseo
 136 small fix + todo comment
 137
 138 Revision 1.6  2006/07/10 13:01:41  jgrosseo
 139 enhanced storing of last sucessfully processed run (alberto)
 140
 141 Revision 1.5  2006/07/04 14:59:57  jgrosseo
 142 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
 143
 144 Revision 1.4  2006/06/12 09:11:16  jgrosseo
 145 coding conventions (Alberto)
 146
 147 Revision 1.3  2006/06/06 14:26:40  jgrosseo
 148 o) removed files that were moved to STEER
 149 o) shuttle updated to follow the new interface (Alberto)
 150
 151 Revision 1.2  2006/03/07 07:52:34  hristov
 152 New version (B.Yordanov)
 153
 154 Revision 1.6  2005/11/19 17:19:14  byordano
 155 RetrieveDATEEntries and RetrieveConditionsData added
 156
 157 Revision 1.5  2005/11/19 11:09:27  byordano
 158 AliShuttle declaration added
 159
 160 Revision 1.4  2005/11/17 17:47:34  byordano
 161 TList changed to TObjArray
 162
 163 Revision 1.3  2005/11/17 14:43:23  byordano
 164 import to local CVS
 165
 166 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
 167 Initial import as subdirectory in AliRoot
 168
 169 Revision 1.2  2005/09/13 08:41:15  byordano
 170 default startTime endTime added
 171
 172 Revision 1.4  2005/08/30 09:13:02  byordano
 173 some docs added
 174
 175 Revision 1.3  2005/08/29 21:15:47  byordano
 176 some docs added
 177
 178 */
 179
 180 //
 181 // This class is the main manager for AliShuttle.
 182 // It organizes the data retrieval from DCS and call the
 183 // interface methods of AliPreprocessor.
 184 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
 185 // data for its set of aliases is retrieved. If there is registered
 186 // AliPreprocessor for this detector then it will be used
 187 // accroding to the schema (see AliPreprocessor).
 188 // If there isn't registered AliPreprocessor than the retrieved
 189 // data is stored automatically to the undelying AliCDBStorage.
 190 // For detSpec is used the alias name.
 191 //
 192
 193 #include "AliShuttle.h"
 194
 195 #include "AliCDBManager.h"
 196 #include "AliCDBStorage.h"
 197 #include "AliCDBId.h"
 198 #include "AliCDBRunRange.h"
 199 #include "AliCDBPath.h"
 200 #include "AliCDBEntry.h"
 201 #include "AliShuttleConfig.h"
 202 #include "DCSClient/AliDCSClient.h"
 203 #include "AliLog.h"
 204 #include "AliPreprocessor.h"
 205 #include "AliShuttleStatus.h"
 206 #include "AliShuttleLogbookEntry.h"
 207
 208 #include <TSystem.h>
 209 #include <TObject.h>
 210 #include <TString.h>
 211 #include <TTimeStamp.h>
 212 #include <TObjString.h>
 213 #include <TSQLServer.h>
 214 #include <TSQLResult.h>
 215 #include <TSQLRow.h>
 216 #include <TMutex.h>
 217 #include <TSystemDirectory.h>
 218 #include <TSystemFile.h>
 219 #include <TFileMerger.h>
 220 #include <TGrid.h>
 221 #include <TGridResult.h>
 222
 223 #include <TMonaLisaWriter.h>
 224
 225 #include <fstream>
 226
 227 #include <sys/types.h>
 228 #include <sys/wait.h>
 229
 230 ClassImp(AliShuttle)
 231
 232 //______________________________________________________________________________________________
 233 AliShuttle::AliShuttle(const AliShuttleConfig* config,
 234                 UInt_t timeout, Int_t retries):
 235 fConfig(config),
 236 fTimeout(timeout), fRetries(retries),
 237 fPreprocessorMap(),
 238 fLogbookEntry(0),
 239 fCurrentDetector(),
 240 fStatusEntry(0),
 241 fMonitoringMutex(0),
 242 fLastActionTime(0),
 243 fLastAction(),
 244 fMonaLisa(0),
 245 fTestMode(kNone),
 246 fReadTestMode(kFALSE),
 247 fOutputRedirected(kFALSE)
 248 {
 249         //
 250         // config: AliShuttleConfig used
 251         // timeout: timeout used for AliDCSClient connection
 252         // retries: the number of retries in case of connection error.
 253         //
 254
 255         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
 256         for(int iSys=0;iSys<4;iSys++) {
 257                 fServer[iSys]=0;
 258                 if (iSys < 3)
 259                         fFXSlist[iSys].SetOwner(kTRUE);
 260         }
 261         fPreprocessorMap.SetOwner(kTRUE);
 262
 263         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
 264                 fFirstUnprocessed[iDet] = kFALSE;
 265
 266         fMonitoringMutex = new TMutex();
 267 }
 268
 269 //______________________________________________________________________________________________
 270 AliShuttle::~AliShuttle()
 271 {
 272         //
 273         // destructor
 274         //
 275
 276         fPreprocessorMap.DeleteAll();
 277         for(int iSys=0;iSys<4;iSys++)
 278                 if(fServer[iSys]) {
 279                         fServer[iSys]->Close();
 280                         delete fServer[iSys];
 281                         fServer[iSys] = 0;
 282                 }
 283
 284         if (fStatusEntry){
 285                 delete fStatusEntry;
 286                 fStatusEntry = 0;
 287         }
 288
 289         if (fMonitoringMutex)
 290         {
 291                 delete fMonitoringMutex;
 292                 fMonitoringMutex = 0;
 293         }
 294 }
 295
 296 //______________________________________________________________________________________________
 297 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
 298 {
 299         //
 300         // Registers new AliPreprocessor.
 301         // It uses GetName() for indentificator of the pre processor.
 302         // The pre processor is registered it there isn't any other
 303         // with the same identificator (GetName()).
 304         //
 305
 306         const char* detName = preprocessor->GetName();
 307         if(GetDetPos(detName) < 0)
 308                 AliFatal(Form("********** !!!!! Invalid detector name: %s !!!!! **********", detName));
 309
 310         if (fPreprocessorMap.GetValue(detName)) {
 311                 AliWarning(Form("AliPreprocessor %s is already registered!", detName));
 312                 return;
 313         }
 314
 315         fPreprocessorMap.Add(new TObjString(detName), preprocessor);
 316 }
 317 //______________________________________________________________________________________________
 318 Bool_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
 319                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
 320 {
 321         // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
 322         // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
 323         // using this function. Use StoreReferenceData instead!
 324         // It calls StoreLocally function which temporarily stores the data locally; when the preprocessor
 325         // finishes the data are transferred to the main storage (Grid).
 326
 327         return StoreLocally(fgkLocalCDB, path, object, metaData, validityStart, validityInfinite);
 328 }
 329
 330 //______________________________________________________________________________________________
 331 Bool_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData)
 332 {
 333         // Stores a CDB object in the storage for reference data. This objects will not be available during
 334         // offline reconstrunction. Use this function for reference data only!
 335         // It calls StoreLocally function which temporarily stores the data locally; when the preprocessor
 336         // finishes the data are transferred to the main storage (Grid).
 337
 338         return StoreLocally(fgkLocalRefStorage, path, object, metaData);
 339 }
 340
 341 //______________________________________________________________________________________________
 342 Bool_t AliShuttle::StoreLocally(const TString& localUri,
 343                         const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
 344                         Int_t validityStart, Bool_t validityInfinite)
 345 {
 346         // Store object temporarily in local storage. Parameters are passed by Store and StoreReferenceData functions.
 347         // when the preprocessor finishes the data are transferred to the main storage (Grid).
 348         // The parameters are:
 349         //   1) Uri of the backup storage (Local)
 350         //   2) the object's path.
 351         //   3) the object to be stored
 352         //   4) the metaData to be associated with the object
 353         //   5) the validity start run number w.r.t. the current run,
 354         //      if the data is valid only for this run leave the default 0
 355         //   6) specifies if the calibration data is valid for infinity (this means until updated),
 356         //      typical for calibration runs, the default is kFALSE
 357         //
 358         // returns 0 if fail, 1 otherwise
 359
 360         if (fTestMode & kErrorStorage)
 361         {
 362                 Log(fCurrentDetector, "StoreLocally - In TESTMODE - Simulating error while storing locally");
 363                 return kFALSE;
 364         }
 365
 366         const char* cdbType = (localUri == fgkLocalCDB) ? "CDB" : "Reference";
 367
 368         Int_t firstRun = GetCurrentRun() - validityStart;
 369         if(firstRun < 0) {
 370                 AliWarning("First valid run happens to be less than 0! Setting it to 0.");
 371                 firstRun=0;
 372         }
 373
 374         Int_t lastRun = -1;
 375         if(validityInfinite) {
 376                 lastRun = AliCDBRunRange::Infinity();
 377         } else {
 378                 lastRun = GetCurrentRun();
 379         }
 380
 381         // Version is set to current run, it will be used later to transfer data to Grid
 382         AliCDBId id(path, firstRun, lastRun, GetCurrentRun(), -1);
 383
 384         if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
 385                 TObjString runUsed = Form("%d", GetCurrentRun());
 386                 metaData->SetProperty("RunUsed(TObjString)", runUsed.Clone());
 387         }
 388
 389         Bool_t result = kFALSE;
 390
 391         if (!(AliCDBManager::Instance()->GetStorage(localUri))) {
 392                 Log("SHUTTLE", Form("StoreLocally - Cannot activate local %s storage", cdbType));
 393         } else {
 394                 result = AliCDBManager::Instance()->GetStorage(localUri)
 395                                         ->Put(object, id, metaData);
 396         }
 397
 398         if(!result) {
 399
 400                 Log(fCurrentDetector, Form("StoreLocally - Can't store object <%s>!", id.ToString().Data()));
 401         }
 402
 403         return result;
 404 }
 405
 406 //______________________________________________________________________________________________
 407 Bool_t AliShuttle::StoreOCDB()
 408 {
 409         //
 410         // Called when preprocessor ends successfully or when previous storage attempt failed (kStoreError status)
 411         // Calls underlying StoreOCDB(const char*) function twice, for OCDB and Reference storage.
 412         // Then calls StoreRefFilesToGrid to store reference files.
 413         //
 414
 415         if (fTestMode & kErrorGrid)
 416         {
 417                 Log("SHUTTLE", "StoreOCDB - In TESTMODE - Simulating error while storing in the Grid");
 418                 Log(fCurrentDetector, "StoreOCDB - In TESTMODE - Simulating error while storing in the Grid");
 419                 return kFALSE;
 420         }
 421
 422         AliInfo("Storing OCDB data ...");
 423         Bool_t resultCDB = StoreOCDB(fgkMainCDB);
 424
 425         AliInfo("Storing reference data ...");
 426         Bool_t resultRef = StoreOCDB(fgkMainRefStorage);
 427
 428         AliInfo("Storing reference files ...");
 429         Bool_t resultRefFiles = StoreRefFilesToGrid();
 430
 431         return resultCDB && resultRef && resultRefFiles;
 432 }
 433
 434 //______________________________________________________________________________________________
 435 Bool_t AliShuttle::StoreOCDB(const TString& gridURI)
 436 {
 437         //
 438         // Called by StoreOCDB(), performs actual storage to the main OCDB and reference storages (Grid)
 439         //
 440
 441         TObjArray* gridIds=0;
 442
 443         Bool_t result = kTRUE;
 444
 445         const char* type = 0;
 446         TString localURI;
 447         if(gridURI == fgkMainCDB) {
 448                 type = "OCDB";
 449                 localURI = fgkLocalCDB;
 450         } else if(gridURI == fgkMainRefStorage) {
 451                 type = "reference";
 452                 localURI = fgkLocalRefStorage;
 453         } else {
 454                 AliError(Form("Invalid storage URI: %s", gridURI.Data()));
 455                 return kFALSE;
 456         }
 457
 458         AliCDBManager* man = AliCDBManager::Instance();
 459
 460         AliCDBStorage *gridSto = man->GetStorage(gridURI);
 461         if(!gridSto) {
 462                 Log("SHUTTLE",
 463                         Form("StoreOCDB - cannot activate main %s storage", type));
 464                 return kFALSE;
 465         }
 466
 467         gridIds = gridSto->GetQueryCDBList();
 468
 469         // get objects previously stored in local CDB
 470         AliCDBStorage *localSto = man->GetStorage(localURI);
 471         if(!localSto) {
 472                 Log("SHUTTLE",
 473                         Form("StoreOCDB - cannot activate local %s storage", type));
 474                 return kFALSE;
 475         }
 476         AliCDBPath aPath(GetOfflineDetName(fCurrentDetector.Data()),"*","*");
 477         // Local objects were stored with current run as Grid version!
 478         TList* localEntries = localSto->GetAll(aPath.GetPath(), GetCurrentRun(), GetCurrentRun());
 479         localEntries->SetOwner(1);
 480
 481         // loop on local stored objects
 482         TIter localIter(localEntries);
 483         AliCDBEntry *aLocEntry = 0;
 484         while((aLocEntry = dynamic_cast<AliCDBEntry*> (localIter.Next()))){
 485                 aLocEntry->SetOwner(1);
 486                 AliCDBId aLocId = aLocEntry->GetId();
 487                 aLocEntry->SetVersion(-1);
 488                 aLocEntry->SetSubVersion(-1);
 489
 490                 // If local object is valid up to infinity we store it only if it is
 491                 // the first unprocessed run!
 492                 if (aLocId.GetLastRun() == AliCDBRunRange::Infinity() &&
 493                         !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
 494                 {
 495                         Log("SHUTTLE", Form("StoreOCDB - %s: object %s has validity infinite but "
 496                                                 "there are previous unprocessed runs!",
 497                                                 fCurrentDetector.Data(), aLocId.GetPath().Data()));
 498                         continue;
 499                 }
 500
 501                 // loop on Grid valid Id's
 502                 Bool_t store = kTRUE;
 503                 TIter gridIter(gridIds);
 504                 AliCDBId* aGridId = 0;
 505                 while((aGridId = dynamic_cast<AliCDBId*> (gridIter.Next()))){
 506                         if(aGridId->GetPath() != aLocId.GetPath()) continue;
 507                         // skip all objects valid up to infinity
 508                         if(aGridId->GetLastRun() == AliCDBRunRange::Infinity()) continue;
 509                         // if we get here, it means there's already some more recent object stored on Grid!
 510                         store = kFALSE;
 511                         break;
 512                 }
 513
 514                 // If we get here, the file can be stored!
 515                 Bool_t storeOk = gridSto->Put(aLocEntry);
 516                 if(!store || storeOk){
 517
 518                         if (!store)
 519                         {
 520                                 Log(fCurrentDetector.Data(),
 521                                         Form("StoreOCDB - A more recent object already exists in %s storage: <%s>",
 522                                                 type, aGridId->ToString().Data()));
 523                         } else {
 524                                 Log("SHUTTLE",
 525                                         Form("StoreOCDB - Object <%s> successfully put into %s storage",
 526                                                 aLocId.ToString().Data(), type));
 527                         }
 528
 529                         // removing local filename...
 530                         TString filename;
 531                         localSto->IdToFilename(aLocId, filename);
 532                         AliInfo(Form("Removing local file %s", filename.Data()));
 533                         RemoveFile(filename.Data());
 534                         continue;
 535                 } else  {
 536                         Log("SHUTTLE",
 537                                 Form("StoreOCDB - Grid %s storage of object <%s> failed",
 538                                         type, aLocId.ToString().Data()));
 539                         result = kFALSE;
 540                 }
 541         }
 542         localEntries->Clear();
 543
 544         return result;
 545 }
 546
 547 //______________________________________________________________________________________________
 548 Bool_t AliShuttle::StoreReferenceFile(const char* detector, const char* localFile, const char* gridFileName)
 549 {
 550         //
 551         // Stores reference file directly (without opening it). This function stores the file locally.
 552         //
 553         // The file is stored under the following location:
 554         // <base folder of local reference storage>/<DET>/<RUN#>_<gridFileName>
 555         // where <gridFileName> is the second parameter given to the function
 556         //
 557
 558         if (fTestMode & kErrorStorage)
 559         {
 560                 Log(fCurrentDetector, "StoreReferenceFile - In TESTMODE - Simulating error while storing locally");
 561                 return kFALSE;
 562         }
 563
 564         AliCDBManager* man = AliCDBManager::Instance();
 565         AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
 566
 567         TString localBaseFolder = sto->GetBaseFolder();
 568
 569         TString targetDir;
 570         targetDir.Form("%s/%s", localBaseFolder.Data(), detector);
 571
 572         TString target;
 573         target.Form("%s/%d_%s", targetDir.Data(), GetCurrentRun(), gridFileName);
 574
 575         Int_t result = gSystem->GetPathInfo(targetDir, 0, (Long64_t*) 0, 0, 0);
 576         if (result)
 577         {
 578                 result = gSystem->mkdir(targetDir, kTRUE);
 579                 if (result != 0)
 580                 {
 581                         Log("SHUTTLE", Form("StoreReferenceFile - Error creating base directory %s", targetDir.Data()));
 582                         return kFALSE;
 583                 }
 584         }
 585
 586         result = gSystem->CopyFile(localFile, target);
 587
 588         if (result == 0)
 589         {
 590                 Log("SHUTTLE", Form("StoreReferenceFile - Stored file %s locally to %s", localFile, target.Data()));
 591                 return kTRUE;
 592         }
 593         else
 594         {
 595                 Log("SHUTTLE", Form("StoreReferenceFile - Storing file %s locally to %s failed", localFile, target.Data()));
 596                 return kFALSE;
 597         }
 598 }
 599
 600 //______________________________________________________________________________________________
 601 Bool_t AliShuttle::StoreRefFilesToGrid()
 602 {
 603         //
 604         // Transfers the reference file to the Grid.
 605         //
 606         // The files are stored under the following location:
 607         // <base folder of reference storage>/<DET>/<RUN#>_<gridFileName>
 608         //
 609
 610         AliCDBManager* man = AliCDBManager::Instance();
 611         AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
 612         if (!sto)
 613                 return kFALSE;
 614         TString localBaseFolder = sto->GetBaseFolder();
 615
 616         TString dir;
 617         dir.Form("%s/%s", localBaseFolder.Data(), GetOfflineDetName(fCurrentDetector));
 618
 619         AliCDBStorage* gridSto = man->GetStorage(fgkMainRefStorage);
 620         if (!gridSto)
 621                 return kFALSE;
 622         TString gridBaseFolder = gridSto->GetBaseFolder();
 623         TString alienDir;
 624         alienDir.Form("%s%s", gridBaseFolder.Data(), GetOfflineDetName(fCurrentDetector));
 625
 626         if (!gGrid)
 627                 return kFALSE;
 628
 629         TString begin;
 630         begin.Form("%d_", GetCurrentRun());
 631
 632         TSystemDirectory* baseDir = new TSystemDirectory("/", dir);
 633         if (!baseDir)
 634                 return kTRUE;
 635
 636         TList* dirList            = baseDir->GetListOfFiles();
 637         if (!dirList)
 638         {
 639                 delete baseDir;
 640                 return kTRUE;
 641         }
 642
 643         Int_t nDirs               = dirList->GetEntries();
 644
 645         Bool_t success = kTRUE;
 646         Bool_t first = kTRUE;
 647
 648         for (Int_t iDir=0; iDir<nDirs; ++iDir)
 649         {
 650                 TSystemFile* entry = dynamic_cast<TSystemFile*> (dirList->At(iDir));
 651                 if (!entry)
 652                         continue;
 653
 654                 if (entry->IsDirectory())
 655                         continue;
 656
 657                 TString fileName(entry->GetName());
 658                 if (!fileName.BeginsWith(begin))
 659                         continue;
 660
 661                 if (first)
 662                 {
 663                         first = kFALSE;
 664                         // check that DET folder exists, otherwise create it
 665                         TGridResult* result = gGrid->Ls(alienDir.Data(), "a");
 666
 667                         if (!result)
 668                                 return kFALSE;
 669
 670                         if (!result->GetFileName(0))
 671                         {
 672                                 if (!gGrid->Mkdir(alienDir.Data(),"",0))
 673                                 {
 674                                         Log("SHUTTLE", Form("StoreRefFilesToGrid - Cannot create directory %s",
 675                                                         alienDir.Data()));
 676                                         delete baseDir;
 677                                         return kFALSE;
 678                                 }
 679
 680                         }
 681                 }
 682
 683                 TString fullLocalPath;
 684                 fullLocalPath.Form("%s/%s", dir.Data(), fileName.Data());
 685
 686                 TString fullGridPath;
 687                 fullGridPath.Form("alien://%s/%s", alienDir.Data(), fileName.Data());
 688
 689                 Log("SHUTTLE", Form("StoreRefFilesToGrid - Copying local file %s to %s", fullLocalPath.Data(), fullGridPath.Data()));
 690
 691                 TFileMerger fileMerger;
 692                 Bool_t result = fileMerger.Cp(fullLocalPath, fullGridPath);
 693
 694                 if (result)
 695                 {
 696                         Log("SHUTTLE", Form("StoreRefFilesToGrid - Copying local file %s to %s succeeded", fullLocalPath.Data(), fullGridPath.Data()));
 697                         RemoveFile(fullLocalPath);
 698                 }
 699                 else
 700                 {
 701                         Log("SHUTTLE", Form("StoreRefFilesToGrid - Copying local file %s to %s failed", fullLocalPath.Data(), fullGridPath.Data()));
 702                         success = kFALSE;
 703                 }
 704         }
 705
 706         delete baseDir;
 707
 708         return success;
 709 }
 710
 711 //______________________________________________________________________________________________
 712 void AliShuttle::CleanLocalStorage(const TString& uri)
 713 {
 714         //
 715         // Called in case the preprocessor is declared failed. Remove remaining objects from the local storages.
 716         //
 717
 718         const char* type = 0;
 719         if(uri == fgkLocalCDB) {
 720                 type = "OCDB";
 721         } else if(uri == fgkLocalRefStorage) {
 722                 type = "reference";
 723         } else {
 724                 AliError(Form("Invalid storage URI: %s", uri.Data()));
 725                 return;
 726         }
 727
 728         AliCDBManager* man = AliCDBManager::Instance();
 729
 730         // open local storage
 731         AliCDBStorage *localSto = man->GetStorage(uri);
 732         if(!localSto) {
 733                 Log("SHUTTLE",
 734                         Form("CleanLocalStorage - cannot activate local %s storage", type));
 735                 return;
 736         }
 737
 738         TString filename(Form("%s/%s/*/Run*_v%d_s*.root",
 739                 localSto->GetBaseFolder().Data(), fCurrentDetector.Data(), GetCurrentRun()));
 740
 741         AliInfo(Form("filename = %s", filename.Data()));
 742
 743         AliInfo(Form("Removing remaining local files from run %d and detector %s ...",
 744                 GetCurrentRun(), fCurrentDetector.Data()));
 745
 746         RemoveFile(filename.Data());
 747
 748 }
 749
 750 //______________________________________________________________________________________________
 751 void AliShuttle::RemoveFile(const char* filename)
 752 {
 753         //
 754         // removes local file
 755         //
 756
 757         TString command(Form("rm -f %s", filename));
 758
 759         Int_t result = gSystem->Exec(command.Data());
 760         if(result != 0)
 761         {
 762                 Log("SHUTTLE", Form("RemoveFile - %s: Cannot remove file %s!",
 763                         fCurrentDetector.Data(), filename));
 764         }
 765 }
 766
 767 //______________________________________________________________________________________________
 768 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
 769 {
 770         //
 771         // Reads the AliShuttleStatus from the CDB
 772         //
 773
 774         if (fStatusEntry){
 775                 delete fStatusEntry;
 776                 fStatusEntry = 0;
 777         }
 778
 779         fStatusEntry = AliCDBManager::Instance()->GetStorage(GetLocalCDB())
 780                 ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), GetCurrentRun());
 781
 782         if (!fStatusEntry) return 0;
 783         fStatusEntry->SetOwner(1);
 784
 785         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
 786         if (!status) {
 787                 AliError("Invalid object stored to CDB!");
 788                 return 0;
 789         }
 790
 791         return status;
 792 }
 793
 794 //______________________________________________________________________________________________
 795 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
 796 {
 797         //
 798         // writes the status for one subdetector
 799         //
 800
 801         if (fStatusEntry){
 802                 delete fStatusEntry;
 803                 fStatusEntry = 0;
 804         }
 805
 806         Int_t run = GetCurrentRun();
 807
 808         AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), run, run);
 809
 810         fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
 811         fStatusEntry->SetOwner(1);
 812
 813         UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
 814
 815         if (!result) {
 816                 Log("SHUTTLE", Form("WriteShuttleStatus - Failed for %s, run %d",
 817                                                 fCurrentDetector.Data(), run));
 818                 return kFALSE;
 819         }
 820
 821         SendMLInfo();
 822
 823         return kTRUE;
 824 }
 825
 826 //______________________________________________________________________________________________
 827 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
 828 {
 829         //
 830         // changes the AliShuttleStatus for the given detector and run to the given status
 831         //
 832
 833         if (!fStatusEntry){
 834                 AliError("UNEXPECTED: fStatusEntry empty");
 835                 return;
 836         }
 837
 838         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
 839
 840         if (!status){
 841                 Log("SHUTTLE", "UNEXPECTED: status could not be read from current CDB entry");
 842                 return;
 843         }
 844
 845         TString actionStr = Form("UpdateShuttleStatus - %s: Changing state from %s to %s",
 846                                 fCurrentDetector.Data(),
 847                                 status->GetStatusName(),
 848                                 status->GetStatusName(newStatus));
 849         Log("SHUTTLE", actionStr);
 850         SetLastAction(actionStr);
 851
 852         status->SetStatus(newStatus);
 853         if (increaseCount) status->IncreaseCount();
 854
 855         AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
 856
 857         SendMLInfo();
 858 }
 859
 860 //______________________________________________________________________________________________
 861 void AliShuttle::SendMLInfo()
 862 {
 863         //
 864         // sends ML information about the current status of the current detector being processed
 865         //
 866
 867         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
 868
 869         if (!status){
 870                 Log("SHUTTLE", "SendMLInfo - UNEXPECTED: status could not be read from current CDB entry");
 871                 return;
 872         }
 873
 874         TMonaLisaText  mlStatus(Form("%s_status", fCurrentDetector.Data()), status->GetStatusName());
 875         TMonaLisaValue mlRetryCount(Form("%s_count", fCurrentDetector.Data()), status->GetCount());
 876
 877         TList mlList;
 878         mlList.Add(&mlStatus);
 879         mlList.Add(&mlRetryCount);
 880
 881         fMonaLisa->SendParameters(&mlList);
 882 }
 883
 884 //______________________________________________________________________________________________
 885 Bool_t AliShuttle::ContinueProcessing()
 886 {
 887         // this function reads the AliShuttleStatus information from CDB and
 888         // checks if the processing should be continued
 889         // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
 890
 891         if (!fConfig->HostProcessDetector(fCurrentDetector)) return kFALSE;
 892
 893         AliPreprocessor* aPreprocessor =
 894                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
 895         if (!aPreprocessor)
 896         {
 897                 AliInfo(Form("%s: no preprocessor registered", fCurrentDetector.Data()));
 898                 return kFALSE;
 899         }
 900
 901         AliShuttleLogbookEntry::Status entryStatus =
 902                 fLogbookEntry->GetDetectorStatus(fCurrentDetector);
 903
 904         if(entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
 905                 AliInfo(Form("ContinueProcessing - %s is %s",
 906                                 fCurrentDetector.Data(),
 907                                 fLogbookEntry->GetDetectorStatusName(entryStatus)));
 908                 return kFALSE;
 909         }
 910
 911         // if we get here, according to Shuttle logbook subdetector is in UNPROCESSED state
 912
 913         // check if current run is first unprocessed run for current detector
 914         if (fConfig->StrictRunOrder(fCurrentDetector) &&
 915                 !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
 916         {
 917                 if (fTestMode == kNone)
 918                 {
 919                         Log("SHUTTLE", Form("ContinueProcessing - %s requires strict run ordering but this is not the first unprocessed run!"));
 920                         return kFALSE;
 921                 }
 922                 else
 923                 {
 924                         Log("SHUTTLE", Form("ContinueProcessing - In TESTMODE - Although %s requires strict run ordering and this is not the first unprocessed run, the SHUTTLE continues"));
 925                 }
 926         }
 927
 928         AliShuttleStatus* status = ReadShuttleStatus();
 929         if (!status) {
 930                 // first time
 931                 Log("SHUTTLE", Form("ContinueProcessing - %s: Processing first time",
 932                                 fCurrentDetector.Data()));
 933                 status = new AliShuttleStatus(AliShuttleStatus::kStarted);
 934                 return WriteShuttleStatus(status);
 935         }
 936
 937         // The following two cases shouldn't happen if Shuttle Logbook was correctly updated.
 938         // If it happens it may mean Logbook updating failed... let's do it now!
 939         if (status->GetStatus() == AliShuttleStatus::kDone ||
 940             status->GetStatus() == AliShuttleStatus::kFailed){
 941                 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s. Updating Shuttle Logbook",
 942                                         fCurrentDetector.Data(),
 943                                         status->GetStatusName(status->GetStatus())));
 944                 UpdateShuttleLogbook(fCurrentDetector.Data(),
 945                                         status->GetStatusName(status->GetStatus()));
 946                 return kFALSE;
 947         }
 948
 949         if (status->GetStatus() == AliShuttleStatus::kStoreError) {
 950                 Log("SHUTTLE",
 951                         Form("ContinueProcessing - %s: Grid storage of one or more objects failed. Trying again now",
 952                                 fCurrentDetector.Data()));
 953                 UpdateShuttleStatus(AliShuttleStatus::kStoreStarted);
 954                 if (StoreOCDB()){
 955                         Log("SHUTTLE", Form("ContinueProcessing - %s: all objects successfully stored into main storage",
 956                                 fCurrentDetector.Data()));
 957                         UpdateShuttleStatus(AliShuttleStatus::kDone);
 958                         UpdateShuttleLogbook(fCurrentDetector.Data(), "DONE");
 959                 } else {
 960                         Log("SHUTTLE",
 961                                 Form("ContinueProcessing - %s: Grid storage failed again",
 962                                         fCurrentDetector.Data()));
 963                         UpdateShuttleStatus(AliShuttleStatus::kStoreError);
 964                 }
 965                 return kFALSE;
 966         }
 967
 968         // if we get here, there is a restart
 969         Bool_t cont = kFALSE;
 970
 971         // abort conditions
 972         if (status->GetCount() >= fConfig->GetMaxRetries()) {
 973                 Log("SHUTTLE", Form("ContinueProcessing - %s failed %d times in status %s - "
 974                                 "Updating Shuttle Logbook", fCurrentDetector.Data(),
 975                                 status->GetCount(), status->GetStatusName()));
 976                 UpdateShuttleLogbook(fCurrentDetector.Data(), "FAILED");
 977                 UpdateShuttleStatus(AliShuttleStatus::kFailed);
 978
 979                 // there may still be objects in local OCDB and reference storage
 980                 // and FXS databases may be not updated: do it now!
 981
 982                 // TODO Currently disabled, we want to keep files in case of failure!
 983                 // CleanLocalStorage(fgkLocalCDB);
 984                 // CleanLocalStorage(fgkLocalRefStorage);
 985                 // UpdateTableFailCase();
 986
 987                 // Send mail to detector expert!
 988                 AliInfo(Form("Sending mail to %s expert...", fCurrentDetector.Data()));
 989                 if (!SendMail())
 990                         Log("SHUTTLE", Form("ContinueProcessing - Could not send mail to %s expert",
 991                                         fCurrentDetector.Data()));
 992
 993         } else {
 994                 Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. "
 995                                 "Aborted before with %s. Retry number %d.", fCurrentDetector.Data(),
 996                                 status->GetStatusName(), status->GetCount()));
 997                 Bool_t increaseCount = kTRUE;
 998                 if (status->GetStatus() == AliShuttleStatus::kDCSError || status->GetStatus() == AliShuttleStatus::kDCSStarted)
 999                         increaseCount = kFALSE;
1000                 UpdateShuttleStatus(AliShuttleStatus::kStarted, increaseCount);
1001                 cont = kTRUE;
1002         }
1003
1004         return cont;
1005 }
1006
1007 //______________________________________________________________________________________________
1008 Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
1009 {
1010         //
1011         // Makes data retrieval for all detectors in the configuration.
1012         // entry: Shuttle logbook entry, contains run paramenters and status of detectors
1013         // (Unprocessed, Inactive, Failed or Done).
1014         // Returns kFALSE in case of error occured and kTRUE otherwise
1015         //
1016
1017         if (!entry) return kFALSE;
1018
1019         fLogbookEntry = entry;
1020
1021         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^* \n",
1022                                         GetCurrentRun()));
1023
1024         // create ML instance that monitors this run
1025         fMonaLisa = new TMonaLisaWriter(Form("%d", GetCurrentRun()), "SHUTTLE", "aliendb1.cern.ch");
1026         // disable monitoring of other parameters that come e.g. from TFile
1027         gMonitoringWriter = 0;
1028
1029         // Send the information to ML
1030         TMonaLisaText  mlStatus("SHUTTLE_status", "Processing");
1031         TMonaLisaText  mlRunType("SHUTTLE_runtype", Form("%s (%s)", entry->GetRunType(), entry->GetRunParameter("log")));
1032
1033         TList mlList;
1034         mlList.Add(&mlStatus);
1035         mlList.Add(&mlRunType);
1036
1037         fMonaLisa->SendParameters(&mlList);
1038
1039         if (fLogbookEntry->IsDone())
1040         {
1041                 Log("SHUTTLE","Process - Shuttle is already DONE. Updating logbook");
1042                 UpdateShuttleLogbook("shuttle_done");
1043                 fLogbookEntry = 0;
1044                 return kTRUE;
1045         }
1046
1047         // read test mode if flag is set
1048         if (fReadTestMode)
1049         {
1050                 fTestMode = kNone;
1051                 TString logEntry(entry->GetRunParameter("log"));
1052                 //printf("log entry = %s\n", logEntry.Data());
1053                 TString searchStr("Testmode: ");
1054                 Int_t pos = logEntry.Index(searchStr.Data());
1055                 //printf("%d\n", pos);
1056                 if (pos >= 0)
1057                 {
1058                         TSubString subStr = logEntry(pos + searchStr.Length(), logEntry.Length());
1059                         //printf("%s\n", subStr.String().Data());
1060                         TString newStr(subStr.Data());
1061                         TObjArray* token = newStr.Tokenize(' ');
1062                         if (token)
1063                         {
1064                                 //token->Print();
1065                                 TObjString* tmpStr = dynamic_cast<TObjString*> (token->First());
1066                                 if (tmpStr)
1067                                 {
1068                                         Int_t testMode = tmpStr->String().Atoi();
1069                                         if (testMode > 0)
1070                                         {
1071                                                 Log("SHUTTLE", Form("Enabling test mode %d", testMode));
1072                                                 SetTestMode((TestMode) testMode);
1073                                         }
1074                                 }
1075                                 delete token;
1076                         }
1077                 }
1078         }
1079
1080         Log("SHUTTLE", Form("The test mode flag is %d", (Int_t) fTestMode));
1081
1082         fLogbookEntry->Print("all");
1083
1084         // Initialization
1085         Bool_t hasError = kFALSE;
1086
1087         AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
1088         if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
1089         AliCDBStorage *mainRefSto = AliCDBManager::Instance()->GetStorage(fgkMainRefStorage);
1090         if(mainRefSto) mainRefSto->QueryCDB(GetCurrentRun());
1091
1092         // Loop on detectors in the configuration
1093         TIter iter(fConfig->GetDetectors());
1094         TObjString* aDetector = 0;
1095
1096         while ((aDetector = (TObjString*) iter.Next()))
1097         {
1098                 fCurrentDetector = aDetector->String();
1099
1100                 if (ContinueProcessing() == kFALSE) continue;
1101
1102                 AliInfo(Form("\n\n \t\t\t****** run %d - %s: START  ******",
1103                                                 GetCurrentRun(), aDetector->GetName()));
1104
1105                 for(Int_t iSys=0;iSys<3;iSys++) fFXSCalled[iSys]=kFALSE;
1106
1107                 Log(fCurrentDetector.Data(), "Starting processing");
1108
1109                 Int_t pid = fork();
1110
1111                 if (pid < 0)
1112                 {
1113                         Log("SHUTTLE", "ERROR: Forking failed");
1114                 }
1115                 else if (pid > 0)
1116                 {
1117                         // parent
1118                         AliInfo(Form("In parent process of %d - %s: Starting monitoring",
1119                                                         GetCurrentRun(), aDetector->GetName()));
1120
1121                         Long_t begin = time(0);
1122
1123                         int status; // to be used with waitpid, on purpose an int (not Int_t)!
1124                         while (waitpid(pid, &status, WNOHANG) == 0)
1125                         {
1126                                 Long_t expiredTime = time(0) - begin;
1127
1128                                 if (expiredTime > fConfig->GetPPTimeOut())
1129                                 {
1130                                         TString tmp;
1131                                         tmp.Form("Process of %s time out. Run time: %d seconds. Killing...",
1132                                                                 fCurrentDetector.Data(), expiredTime);
1133                                         Log("SHUTTLE", tmp);
1134                                         Log(fCurrentDetector, tmp);
1135
1136                                         kill(pid, 9);
1137
1138                                         UpdateShuttleStatus(AliShuttleStatus::kPPTimeOut);
1139                                         hasError = kTRUE;
1140
1141                                         gSystem->Sleep(1000);
1142                                 }
1143                                 else
1144                                 {
1145                                         gSystem->Sleep(1000);
1146
1147                                         TString checkStr;
1148                                         checkStr.Form("ps -o vsize --pid %d | tail -n 1", pid);
1149                                         FILE* pipe = gSystem->OpenPipe(checkStr, "r");
1150                                         if (!pipe)
1151                                         {
1152                                                 Log("SHUTTLE", Form("Error: Could not open pipe to %s", checkStr.Data()));
1153                                                 continue;
1154                                         }
1155
1156                                         char buffer[100];
1157                                         if (!fgets(buffer, 100, pipe))
1158                                         {
1159                                                 Log("SHUTTLE", "Error: ps did not return anything");
1160                                                 gSystem->ClosePipe(pipe);
1161                                                 continue;
1162                                         }
1163                                         gSystem->ClosePipe(pipe);
1164
1165                                         //Log("SHUTTLE", Form("ps returned %s", buffer));
1166
1167                                         Int_t mem = 0;
1168                                         if ((sscanf(buffer, "%d\n", &mem) != 1) || !mem)
1169                                         {
1170                                                 Log("SHUTTLE", "Error: Could not parse output of ps");
1171                                                 continue;
1172                                         }
1173
1174                                         if (expiredTime % 60 == 0)
1175                                                 Log("SHUTTLE", Form("%s: Checking process. Run time: %d seconds - Memory consumption: %d KB",
1176                                                                 fCurrentDetector.Data(), expiredTime, mem));
1177
1178                                         if (mem > fConfig->GetPPMaxMem())
1179                                         {
1180                                                 TString tmp;
1181                                                 tmp.Form("Process exceeds maximum allowed memory (%d KB > %d KB). Killing...",
1182                                                         mem, fConfig->GetPPMaxMem());
1183                                                 Log("SHUTTLE", tmp);
1184                                                 Log(fCurrentDetector, tmp);
1185
1186                                                 kill(pid, 9);
1187
1188                                                 UpdateShuttleStatus(AliShuttleStatus::kPPOutOfMemory);
1189                                                 hasError = kTRUE;
1190
1191                                                 gSystem->Sleep(1000);
1192                                         }
1193                                 }
1194                         }
1195
1196                         AliInfo(Form("In parent process of %d - %s: Client has terminated.",
1197                                                                 GetCurrentRun(), aDetector->GetName()));
1198
1199                         if (WIFEXITED(status))
1200                         {
1201                                 Int_t returnCode = WEXITSTATUS(status);
1202
1203                                 Log("SHUTTLE", Form("%s: the return code is %d", fCurrentDetector.Data(),
1204                                                                                 returnCode));
1205
1206                                 if (returnCode == 0) hasError = kTRUE;
1207                         }
1208                 }
1209                 else if (pid == 0)
1210                 {
1211                         // client
1212                         AliInfo(Form("In client process of %d - %s", GetCurrentRun(), aDetector->GetName()));
1213
1214                         AliInfo("Redirecting output...");
1215
1216                         if ((freopen(GetLogFileName(fCurrentDetector), "w", stdout)) == 0)
1217                         {
1218                                 Log("SHUTTLE", "Could not freopen stdout");
1219                         }
1220                         else
1221                         {
1222                                 fOutputRedirected = kTRUE;
1223                                 if ((dup2(fileno(stdout), fileno(stderr))) < 0)
1224                                         Log("SHUTTLE", "Could not redirect stderr");
1225
1226                         }
1227
1228                         Bool_t success = ProcessCurrentDetector();
1229                         if (success) // Preprocessor finished successfully!
1230                         {
1231                                 // Update time_processed field in FXS DB
1232                                 if (UpdateTable() == kFALSE)
1233                                         Log("SHUTTLE", Form("Process - %s: Could not update FXS databases!"));
1234
1235                                 // Transfer the data from local storage to main storage (Grid)
1236                                 UpdateShuttleStatus(AliShuttleStatus::kStoreStarted);
1237                                 if (StoreOCDB() == kFALSE)
1238                                 {
1239                                         AliInfo(Form("\n \t\t\t****** run %d - %s: STORAGE ERROR ****** \n\n",
1240                                                         GetCurrentRun(), aDetector->GetName()));
1241                                         UpdateShuttleStatus(AliShuttleStatus::kStoreError);
1242                                         success = kFALSE;
1243                                 } else {
1244                                         AliInfo(Form("\n \t\t\t****** run %d - %s: DONE ****** \n\n",
1245                                                         GetCurrentRun(), aDetector->GetName()));
1246                                         UpdateShuttleStatus(AliShuttleStatus::kDone);
1247                                         UpdateShuttleLogbook(fCurrentDetector, "DONE");
1248                                 }
1249                         }
1250
1251                         for (UInt_t iSys=0; iSys<3; iSys++)
1252                         {
1253                                 if (fFXSCalled[iSys]) fFXSlist[iSys].Clear();
1254                         }
1255
1256                         AliInfo(Form("Client process of %d - %s is exiting now with %d.",
1257                                                         GetCurrentRun(), aDetector->GetName(), success));
1258
1259                         // the client exits here
1260                         gSystem->Exit(success);
1261
1262                         AliError("We should never get here!!!");
1263                 }
1264         }
1265
1266         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^* \n",
1267                                                         GetCurrentRun()));
1268
1269         //check if shuttle is done for this run, if so update logbook
1270         TObjArray checkEntryArray;
1271         checkEntryArray.SetOwner(1);
1272         TString whereClause = Form("where run=%d", GetCurrentRun());
1273         if (!QueryShuttleLogbook(whereClause.Data(), checkEntryArray) || checkEntryArray.GetEntries() == 0) {
1274                 Log("SHUTTLE", Form("Process - Warning: Cannot check status of run %d on Shuttle logbook!",
1275                                                 GetCurrentRun()));
1276                 return hasError == kFALSE;
1277         }
1278
1279         AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
1280                                                 (checkEntryArray.At(0));
1281
1282         if (checkEntry)
1283         {
1284                 if (checkEntry->IsDone())
1285                 {
1286                         Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
1287                         UpdateShuttleLogbook("shuttle_done");
1288                 }
1289                 else
1290                 {
1291                         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
1292                         {
1293                                 if (checkEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
1294                                 {
1295                                         AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
1296                                                         checkEntry->GetRun(), GetDetName(iDet)));
1297                                         fFirstUnprocessed[iDet] = kFALSE;
1298                                 }
1299                         }
1300                 }
1301         }
1302
1303         // remove ML instance
1304         delete fMonaLisa;
1305         fMonaLisa = 0;
1306
1307         fLogbookEntry = 0;
1308
1309         return hasError == kFALSE;
1310 }
1311
1312 //______________________________________________________________________________________________
1313 Bool_t AliShuttle::ProcessCurrentDetector()
1314 {
1315         //
1316         // Makes data retrieval just for a specific detector (fCurrentDetector).
1317         // Threre should be a configuration for this detector.
1318
1319         AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), GetCurrentRun()));
1320
1321         TMap dcsMap;
1322         dcsMap.SetOwner(1);
1323
1324         Bool_t aDCSError = kFALSE;
1325
1326         // call preprocessor
1327         AliPreprocessor* aPreprocessor =
1328                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
1329
1330         aPreprocessor->Initialize(GetCurrentRun(), GetCurrentStartTime(), GetCurrentEndTime());
1331
1332         Bool_t processDCS = aPreprocessor->ProcessDCS();
1333
1334         if (!processDCS || (fTestMode & kSkipDCS))
1335         {
1336                 Log(fCurrentDetector, "In TESTMODE - Skipping DCS processing!");
1337         }
1338         else if (fTestMode & kErrorDCS)
1339         {
1340                 Log(fCurrentDetector, "In TESTMODE - Simulating DCS error");
1341                 UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
1342                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
1343                 return kFALSE;
1344         } else {
1345
1346                 UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
1347
1348                 TString host(fConfig->GetDCSHost(fCurrentDetector));
1349                 Int_t port = fConfig->GetDCSPort(fCurrentDetector);
1350
1351                 // Retrieval of Aliases
1352                 TObjString* anAlias = 0;
1353                 Int_t iAlias = 1;
1354                 Int_t nTotAliases= ((TMap*)fConfig->GetDCSAliases(fCurrentDetector))->GetEntries();
1355                 TIter iterAliases(fConfig->GetDCSAliases(fCurrentDetector));
1356                 while ((anAlias = (TObjString*) iterAliases.Next()))
1357                 {
1358                         TObjArray *valueSet = new TObjArray();
1359                         valueSet->SetOwner(1);
1360
1361                         if (((iAlias-1) % 500) == 0 || iAlias == nTotAliases)
1362                                 AliInfo(Form("Querying DCS archive: alias %s (%d of %d)",
1363                                                 anAlias->GetName(), iAlias++, nTotAliases));
1364                         aDCSError = (GetValueSet(host, port, anAlias->String(), valueSet, kAlias) == 0);
1365
1366                         if(!aDCSError)
1367                         {
1368                                 dcsMap.Add(anAlias->Clone(), valueSet);
1369                         } else {
1370                                 Log(fCurrentDetector,
1371                                         Form("ProcessCurrentDetector - Error while retrieving alias %s",
1372                                                 anAlias->GetName()));
1373                                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
1374                                 dcsMap.DeleteAll();
1375                                 return kFALSE;
1376                         }
1377                 }
1378
1379                 // Retrieval of Data Points
1380                 TObjString* aDP = 0;
1381                 Int_t iDP = 0;
1382                 Int_t nTotDPs= ((TMap*)fConfig->GetDCSDataPoints(fCurrentDetector))->GetEntries();
1383                 TIter iterDP(fConfig->GetDCSDataPoints(fCurrentDetector));
1384                 while ((aDP = (TObjString*) iterDP.Next()))
1385                 {
1386                         TObjArray *valueSet = new TObjArray();
1387                         valueSet->SetOwner(1);
1388                         if (((iDP-1) % 500) == 0 || iDP == nTotDPs)
1389                                 AliInfo(Form("Querying DCS archive: DP %s (%d of %d)",
1390                                                 aDP->GetName(), iDP++, nTotDPs));
1391                         aDCSError = (GetValueSet(host, port, aDP->String(), valueSet, kDP) == 0);
1392
1393                         if(!aDCSError)
1394                         {
1395                                 dcsMap.Add(aDP->Clone(), valueSet);
1396                         } else {
1397                                 Log(fCurrentDetector,
1398                                         Form("ProcessCurrentDetector - Error while retrieving data point %s",
1399                                                 aDP->GetName()));
1400                                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
1401                                 dcsMap.DeleteAll();
1402                                 return kFALSE;
1403                         }
1404                 }
1405         }
1406
1407         // DCS Archive DB processing successful. Call Preprocessor!
1408         UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
1409
1410         UInt_t returnValue = aPreprocessor->Process(&dcsMap);
1411
1412         if (returnValue > 0) // Preprocessor error!
1413         {
1414                 Log(fCurrentDetector, Form("Preprocessor failed. Process returned %d.", returnValue));
1415                 UpdateShuttleStatus(AliShuttleStatus::kPPError);
1416                 dcsMap.DeleteAll();
1417                 return kFALSE;
1418         }
1419
1420         // preprocessor ok!
1421         UpdateShuttleStatus(AliShuttleStatus::kPPDone);
1422         Log(fCurrentDetector, Form("ProcessCurrentDetector - %s preprocessor returned success",
1423                                 fCurrentDetector.Data()));
1424
1425         dcsMap.DeleteAll();
1426
1427         return kTRUE;
1428 }
1429
1430 //______________________________________________________________________________________________
1431 Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
1432                 TObjArray& entries)
1433 {
1434         // Query DAQ's Shuttle logbook and fills detector status object.
1435         // Call QueryRunParameters to query DAQ logbook for run parameters.
1436         //
1437
1438         entries.SetOwner(1);
1439
1440         // check connection, in case connect
1441         if(!Connect(3)) return kFALSE;
1442
1443         TString sqlQuery;
1444         sqlQuery = Form("select * from %s %s order by run", fConfig->GetShuttlelbTable(), whereClause);
1445
1446         TSQLResult* aResult = fServer[3]->Query(sqlQuery);
1447         if (!aResult) {
1448                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
1449                 return kFALSE;
1450         }
1451
1452         AliDebug(2,Form("Query = %s", sqlQuery.Data()));
1453
1454         if(aResult->GetRowCount() == 0) {
1455                 AliInfo("No entries in Shuttle Logbook match request");
1456                 delete aResult;
1457                 return kTRUE;
1458         }
1459
1460         // TODO Check field count!
1461         const UInt_t nCols = 22;
1462         if (aResult->GetFieldCount() != (Int_t) nCols) {
1463                 AliError("Invalid SQL result field number!");
1464                 delete aResult;
1465                 return kFALSE;
1466         }
1467
1468         TSQLRow* aRow;
1469         while ((aRow = aResult->Next())) {
1470                 TString runString(aRow->GetField(0), aRow->GetFieldLength(0));
1471                 Int_t run = runString.Atoi();
1472
1473                 AliShuttleLogbookEntry *entry = QueryRunParameters(run);
1474                 if (!entry)
1475                         continue;
1476
1477                 // loop on detectors
1478                 for(UInt_t ii = 0; ii < nCols; ii++)
1479                         entry->SetDetectorStatus(aResult->GetFieldName(ii), aRow->GetField(ii));
1480
1481                 entries.AddLast(entry);
1482                 delete aRow;
1483         }
1484
1485         delete aResult;
1486         return kTRUE;
1487 }
1488
1489 //______________________________________________________________________________________________
1490 AliShuttleLogbookEntry* AliShuttle::QueryRunParameters(Int_t run)
1491 {
1492         //
1493         // Retrieve run parameters written in the DAQ logbook and sets them into AliShuttleLogbookEntry object
1494         //
1495
1496         // check connection, in case connect
1497         if (!Connect(3))
1498                 return 0;
1499
1500         TString sqlQuery;
1501         sqlQuery.Form("select * from %s where run=%d", fConfig->GetDAQlbTable(), run);
1502
1503         TSQLResult* aResult = fServer[3]->Query(sqlQuery);
1504         if (!aResult) {
1505                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
1506                 return 0;
1507         }
1508
1509         if (aResult->GetRowCount() == 0) {
1510                 Log("SHUTTLE", Form("QueryRunParameters - No entry in DAQ Logbook for run %d. Skipping", run));
1511                 delete aResult;
1512                 return 0;
1513         }
1514
1515         if (aResult->GetRowCount() > 1) {
1516                 AliError(Form("More than one entry in DAQ Logbook for run %d. Skipping", run));
1517                 delete aResult;
1518                 return 0;
1519         }
1520
1521         TSQLRow* aRow = aResult->Next();
1522         if (!aRow)
1523         {
1524                 AliError(Form("Could not retrieve row for run %d. Skipping", run));
1525                 delete aResult;
1526                 return 0;
1527         }
1528
1529         AliShuttleLogbookEntry* entry = new AliShuttleLogbookEntry(run);
1530
1531         for (Int_t ii = 0; ii < aResult->GetFieldCount(); ii++)
1532                 entry->SetRunParameter(aResult->GetFieldName(ii), aRow->GetField(ii));
1533
1534         UInt_t startTime = entry->GetStartTime();
1535         UInt_t endTime = entry->GetEndTime();
1536
1537         if (!startTime || !endTime || startTime > endTime) {
1538                 Log("SHUTTLE",
1539                         Form("QueryRunParameters - Invalid parameters for Run %d: startTime = %d, endTime = %d",
1540                                 run, startTime, endTime));
1541                 delete entry;
1542                 delete aRow;
1543                 delete aResult;
1544                 return 0;
1545         }
1546
1547         delete aRow;
1548         delete aResult;
1549
1550         return entry;
1551 }
1552
1553 //______________________________________________________________________________________________
1554 Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* entry,
1555                                 TObjArray* valueSet, DCSType type)
1556 {
1557         // Retrieve all "entry" data points from the DCS server
1558         // host, port: TSocket connection parameters
1559         // entry: name of the alias or data point
1560         // valueSet: array of retrieved AliDCSValue's
1561         // type: kAlias or kDP
1562
1563         AliDCSClient client(host, port, fTimeout, fRetries);
1564         if (!client.IsConnected())
1565         {
1566                 return kFALSE;
1567         }
1568
1569         Int_t result=0;
1570
1571         if (type == kAlias)
1572         {
1573                 result = client.GetAliasValues(entry,
1574                         GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
1575         } else
1576         if (type == kDP)
1577         {
1578                 result = client.GetDPValues(entry,
1579                         GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
1580         }
1581
1582         if (result < 0)
1583         {
1584                 Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get '%s'! Reason: %s",
1585                         entry, AliDCSClient::GetErrorString(result)));
1586
1587                 if (result == AliDCSClient::fgkServerError)
1588                 {
1589                         Log(fCurrentDetector.Data(), Form("GetValueSet - Server error: %s",
1590                                 client.GetServerError().Data()));
1591                 }
1592
1593                 return kFALSE;
1594         }
1595
1596         return kTRUE;
1597 }
1598
1599 //______________________________________________________________________________________________
1600 const char* AliShuttle::GetFile(Int_t system, const char* detector,
1601                 const char* id, const char* source)
1602 {
1603         // Get calibration file from file exchange servers
1604         // First queris the FXS database for the file name, using the run, detector, id and source info
1605         // then calls RetrieveFile(filename) for actual copy to local disk
1606         // run: current run being processed (given by Logbook entry fLogbookEntry)
1607         // detector: the Preprocessor name
1608         // id: provided as a parameter by the Preprocessor
1609         // source: provided by the Preprocessor through GetFileSources function
1610
1611         // check if test mode should simulate a FXS error
1612         if (fTestMode & kErrorFXSFiles)
1613         {
1614                 Log(detector, Form("GetFile - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
1615                 return 0;
1616         }
1617
1618         // check connection, in case connect
1619         if (!Connect(system))
1620         {
1621                 Log(detector, Form("GetFile - Couldn't connect to %s FXS database", GetSystemName(system)));
1622                 return 0;
1623         }
1624
1625         // Query preparation
1626         TString sourceName(source);
1627         Int_t nFields = 3;
1628         TString sqlQueryStart = Form("select filePath,size,fileChecksum from %s where",
1629                                                                 fConfig->GetFXSdbTable(system));
1630         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
1631                                                                 GetCurrentRun(), detector, id);
1632
1633         if (system == kDAQ)
1634         {
1635                 whereClause += Form(" and DAQsource=\"%s\"", source);
1636         }
1637         else if (system == kDCS)
1638         {
1639                 sourceName="none";
1640         }
1641         else if (system == kHLT)
1642         {
1643                 whereClause += Form(" and DDLnumbers=\"%s\"", source);
1644                 nFields = 3;
1645         }
1646
1647         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1648
1649         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1650
1651         // Query execution
1652         TSQLResult* aResult = 0;
1653         aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
1654         if (!aResult) {
1655                 Log(detector, Form("GetFileName - Can't execute SQL query to %s database for: id = %s, source = %s",
1656                                 GetSystemName(system), id, sourceName.Data()));
1657                 return 0;
1658         }
1659
1660         if(aResult->GetRowCount() == 0)
1661         {
1662                 Log(detector,
1663                         Form("GetFileName - No entry in %s FXS db for: id = %s, source = %s",
1664                                 GetSystemName(system), id, sourceName.Data()));
1665                 delete aResult;
1666                 return 0;
1667         }
1668
1669         if (aResult->GetRowCount() > 1) {
1670                 Log(detector,
1671                         Form("GetFileName - More than one entry in %s FXS db for: id = %s, source = %s",
1672                                 GetSystemName(system), id, sourceName.Data()));
1673                 delete aResult;
1674                 return 0;
1675         }
1676
1677         if (aResult->GetFieldCount() != nFields) {
1678                 Log(detector,
1679                         Form("GetFileName - Wrong field count in %s FXS db for: id = %s, source = %s",
1680                                 GetSystemName(system), id, sourceName.Data()));
1681                 delete aResult;
1682                 return 0;
1683         }
1684
1685         TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
1686
1687         if (!aRow){
1688                 Log(detector, Form("GetFileName - Empty set result in %s FXS db from query: id = %s, source = %s",
1689                                 GetSystemName(system), id, sourceName.Data()));
1690                 delete aResult;
1691                 return 0;
1692         }
1693
1694         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
1695         TString fileSize(aRow->GetField(1), aRow->GetFieldLength(1));
1696         TString fileChecksum(aRow->GetField(2), aRow->GetFieldLength(2));
1697
1698         delete aResult;
1699         delete aRow;
1700
1701         AliDebug(2, Form("filePath = %s; size = %s, fileChecksum = %s",
1702                                 filePath.Data(), fileSize.Data(), fileChecksum.Data()));
1703
1704         // retrieved file is renamed to make it unique
1705         TString localFileName = Form("%s_%s_%d_%s_%s.shuttle",
1706                                         GetSystemName(system), detector, GetCurrentRun(), id, sourceName.Data());
1707
1708
1709         // file retrieval from FXS
1710         UInt_t nRetries = 0;
1711         UInt_t maxRetries = 3;
1712         Bool_t result = kFALSE;
1713
1714         // copy!! if successful TSystem::Exec returns 0
1715         while(nRetries++ < maxRetries) {
1716                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
1717                 result = RetrieveFile(system, filePath.Data(), localFileName.Data());
1718                 if(!result)
1719                 {
1720                         Log(detector, Form("GetFileName - Copy of file %s from %s FXS failed",
1721                                         filePath.Data(), GetSystemName(system)));
1722                         continue;
1723                 } else {
1724                         AliInfo(Form("File %s copied from %s FXS into %s/%s",
1725                                                 filePath.Data(), GetSystemName(system),
1726                                                 GetShuttleTempDir(), localFileName.Data()));
1727                 }
1728
1729                 if (fileChecksum.Length()>0)
1730                 {
1731                         // compare md5sum of local file with the one stored in the FXS DB
1732                         Int_t md5Comp = gSystem->Exec(Form("md5sum %s/%s |grep %s 2>&1 > /dev/null",
1733                                                 GetShuttleTempDir(), localFileName.Data(), fileChecksum.Data()));
1734
1735                         if (md5Comp != 0)
1736                         {
1737                                 Log(detector, Form("GetFileName - md5sum of file %s does not match with local copy!",
1738                                                         filePath.Data()));
1739                                 result = kFALSE;
1740                                 continue;
1741                         }
1742                 } else {
1743                         Log(fCurrentDetector, Form("GetFile - md5sum of file %s not set in %s database, skipping comparison",
1744                                                         filePath.Data(), GetSystemName(system)));
1745                 }
1746                 if (result) break;
1747         }
1748
1749         if(!result) return 0;
1750
1751         fFXSCalled[system]=kTRUE;
1752         TObjString *fileParams = new TObjString(Form("%s#!?!#%s", id, sourceName.Data()));
1753         fFXSlist[system].Add(fileParams);
1754
1755         static TString fullLocalFileName;
1756         fullLocalFileName = TString::Format("%s/%s", GetShuttleTempDir(), localFileName.Data());
1757
1758         AliInfo(Form("fullLocalFileName = %s", fullLocalFileName.Data()));
1759
1760         return fullLocalFileName.Data();
1761
1762 }
1763
1764 //______________________________________________________________________________________________
1765 Bool_t AliShuttle::RetrieveFile(UInt_t system, const char* fxsFileName, const char* localFileName)
1766 {
1767         //
1768         // Copies file from FXS to local Shuttle machine
1769         //
1770
1771         // check temp directory: trying to cd to temp; if it does not exist, create it
1772         AliDebug(2, Form("Copy file %s from %s FXS into %s/%s",
1773                         GetSystemName(system), fxsFileName, GetShuttleTempDir(), localFileName));
1774
1775         void* dir = gSystem->OpenDirectory(GetShuttleTempDir());
1776         if (dir == NULL) {
1777                 if (gSystem->mkdir(GetShuttleTempDir(), kTRUE)) {
1778                         AliError(Form("Can't open directory <%s>", GetShuttleTempDir()));
1779                         return kFALSE;
1780                 }
1781
1782         } else {
1783                 gSystem->FreeDirectory(dir);
1784         }
1785
1786         TString baseFXSFolder;
1787         if (system == kDAQ)
1788         {
1789                 baseFXSFolder = "FES/";
1790         }
1791         else if (system == kDCS)
1792         {
1793                 baseFXSFolder = "";
1794         }
1795         else if (system == kHLT)
1796         {
1797                 baseFXSFolder = "~/";
1798         }
1799
1800
1801         TString command = Form("scp -oPort=%d -2 %s@%s:%s%s %s/%s",
1802                 fConfig->GetFXSPort(system),
1803                 fConfig->GetFXSUser(system),
1804                 fConfig->GetFXSHost(system),
1805                 baseFXSFolder.Data(),
1806                 fxsFileName,
1807                 GetShuttleTempDir(),
1808                 localFileName);
1809
1810         AliDebug(2, Form("%s",command.Data()));
1811
1812         Bool_t result = (gSystem->Exec(command.Data()) == 0);
1813
1814         return result;
1815 }
1816
1817 //______________________________________________________________________________________________
1818 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
1819 {
1820         //
1821         // Get sources producing the condition file Id from file exchange servers
1822         //
1823
1824         // check if test mode should simulate a FXS error
1825         if (fTestMode & kErrorFXSSources)
1826         {
1827                 Log(detector, Form("GetFileSources - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
1828                 return 0;
1829         }
1830
1831
1832         if (system == kDCS)
1833         {
1834                 AliError("DCS system has only one source of data!");
1835                 return NULL;
1836         }
1837
1838         // check connection, in case connect
1839         if (!Connect(system))
1840         {
1841                 Log(detector, Form("GetFile - Couldn't connect to %s FXS database", GetSystemName(system)));
1842                 return NULL;
1843         }
1844
1845         TString sourceName = 0;
1846         if (system == kDAQ)
1847         {
1848                 sourceName = "DAQsource";
1849         } else if (system == kHLT)
1850         {
1851                 sourceName = "DDLnumbers";
1852         }
1853
1854         TString sqlQueryStart = Form("select %s from %s where", sourceName.Data(), fConfig->GetFXSdbTable(system));
1855         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
1856                                 GetCurrentRun(), detector, id);
1857         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1858
1859         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1860
1861         // Query execution
1862         TSQLResult* aResult;
1863         aResult = fServer[system]->Query(sqlQuery);
1864         if (!aResult) {
1865                 Log(detector, Form("GetFileSources - Can't execute SQL query to %s database for id: %s",
1866                                 GetSystemName(system), id));
1867                 return 0;
1868         }
1869
1870         TList *list = new TList();
1871         list->SetOwner(1);
1872
1873         if (aResult->GetRowCount() == 0)
1874         {
1875                 Log(detector,
1876                         Form("GetFileSources - No entry in %s FXS table for id: %s", GetSystemName(system), id));
1877                 delete aResult;
1878                 return list;
1879         }
1880
1881         TSQLRow* aRow;
1882
1883         while ((aRow = aResult->Next()))
1884         {
1885
1886                 TString source(aRow->GetField(0), aRow->GetFieldLength(0));
1887                 AliDebug(2, Form("%s = %s", sourceName.Data(), source.Data()));
1888                 list->Add(new TObjString(source));
1889                 delete aRow;
1890         }
1891
1892         delete aResult;
1893
1894         return list;
1895 }
1896
1897 //______________________________________________________________________________________________
1898 Bool_t AliShuttle::Connect(Int_t system)
1899 {
1900         // Connect to MySQL Server of the system's FXS MySQL databases
1901         // DAQ Logbook, Shuttle Logbook and DAQ FXS db are on the same host
1902         //
1903
1904         // check connection: if already connected return
1905         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
1906
1907         TString dbHost, dbUser, dbPass, dbName;
1908
1909         if (system < 3) // FXS db servers
1910         {
1911                 dbHost = Form("mysql://%s:%d", fConfig->GetFXSdbHost(system), fConfig->GetFXSdbPort(system));
1912                 dbUser = fConfig->GetFXSdbUser(system);
1913                 dbPass = fConfig->GetFXSdbPass(system);
1914                 dbName =   fConfig->GetFXSdbName(system);
1915         } else { // Run & Shuttle logbook servers
1916         // TODO Will the Shuttle logbook server be the same as the Run logbook server ???
1917                 dbHost = Form("mysql://%s:%d", fConfig->GetDAQlbHost(), fConfig->GetDAQlbPort());
1918                 dbUser = fConfig->GetDAQlbUser();
1919                 dbPass = fConfig->GetDAQlbPass();
1920                 dbName =   fConfig->GetDAQlbDB();
1921         }
1922
1923         fServer[system] = TSQLServer::Connect(dbHost.Data(), dbUser.Data(), dbPass.Data());
1924         if (!fServer[system] || !fServer[system]->IsConnected()) {
1925                 if(system < 3)
1926                 {
1927                 AliError(Form("Can't establish connection to FXS database for %s",
1928                                         AliShuttleInterface::GetSystemName(system)));
1929                 } else {
1930                 AliError("Can't establish connection to Run logbook.");
1931                 }
1932                 if(fServer[system]) delete fServer[system];
1933                 return kFALSE;
1934         }
1935
1936         // Get tables
1937         TSQLResult* aResult=0;
1938         switch(system){
1939                 case kDAQ:
1940                         aResult = fServer[kDAQ]->GetTables(dbName.Data());
1941                         break;
1942                 case kDCS:
1943                         aResult = fServer[kDCS]->GetTables(dbName.Data());
1944                         break;
1945                 case kHLT:
1946                         aResult = fServer[kHLT]->GetTables(dbName.Data());
1947                         break;
1948                 default:
1949                         aResult = fServer[3]->GetTables(dbName.Data());
1950                         break;
1951         }
1952
1953         delete aResult;
1954         return kTRUE;
1955 }
1956
1957 //______________________________________________________________________________________________
1958 Bool_t AliShuttle::UpdateTable()
1959 {
1960         //
1961         // Update FXS table filling time_processed field in all rows corresponding to current run and detector
1962         //
1963
1964         Bool_t result = kTRUE;
1965
1966         for (UInt_t system=0; system<3; system++)
1967         {
1968                 if(!fFXSCalled[system]) continue;
1969
1970                 // check connection, in case connect
1971                 if (!Connect(system))
1972                 {
1973                         Log(fCurrentDetector, Form("UpdateTable - Couldn't connect to %s FXS database", GetSystemName(system)));
1974                         result = kFALSE;
1975                         continue;
1976                 }
1977
1978                 TTimeStamp now; // now
1979
1980                 // Loop on FXS list entries
1981                 TIter iter(&fFXSlist[system]);
1982                 TObjString *aFXSentry=0;
1983                 while ((aFXSentry = dynamic_cast<TObjString*> (iter.Next())))
1984                 {
1985                         TString aFXSentrystr = aFXSentry->String();
1986                         TObjArray *aFXSarray = aFXSentrystr.Tokenize("#!?!#");
1987                         if (!aFXSarray || aFXSarray->GetEntries() != 2 )
1988                         {
1989                                 Log(fCurrentDetector, Form("UpdateTable - error updating %s FXS entry. Check string: <%s>",
1990                                         GetSystemName(system), aFXSentrystr.Data()));
1991                                 if(aFXSarray) delete aFXSarray;
1992                                 result = kFALSE;
1993                                 continue;
1994                         }
1995                         const char* fileId = ((TObjString*) aFXSarray->At(0))->GetName();
1996                         const char* source = ((TObjString*) aFXSarray->At(1))->GetName();
1997
1998                         TString whereClause;
1999                         if (system == kDAQ)
2000                         {
2001                                 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
2002                                                         GetCurrentRun(), fCurrentDetector.Data(), fileId, source);
2003                         }
2004                         else if (system == kDCS)
2005                         {
2006                                 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\";",
2007                                                         GetCurrentRun(), fCurrentDetector.Data(), fileId);
2008                         }
2009                         else if (system == kHLT)
2010                         {
2011                                 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DDLnumbers=\"%s\";",
2012                                                         GetCurrentRun(), fCurrentDetector.Data(), fileId, source);
2013                         }
2014
2015                         delete aFXSarray;
2016
2017                         TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(system),
2018                                                                 now.GetSec(), whereClause.Data());
2019
2020                         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2021
2022                         // Query execution
2023                         TSQLResult* aResult;
2024                         aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
2025                         if (!aResult)
2026                         {
2027                                 Log(fCurrentDetector, Form("UpdateTable - %s db: can't execute SQL query <%s>",
2028                                                                 GetSystemName(system), sqlQuery.Data()));
2029                                 result = kFALSE;
2030                                 continue;
2031                         }
2032                         delete aResult;
2033                 }
2034         }
2035
2036         return result;
2037 }
2038
2039 //______________________________________________________________________________________________
2040 Bool_t AliShuttle::UpdateTableFailCase()
2041 {
2042         // Update FXS table filling time_processed field in all rows corresponding to current run and detector
2043         // this is called in case the preprocessor is declared failed for the current run, because
2044         // the fields are updated only in case of success
2045
2046         Bool_t result = kTRUE;
2047
2048         for (UInt_t system=0; system<3; system++)
2049         {
2050                 // check connection, in case connect
2051                 if (!Connect(system))
2052                 {
2053                         Log(fCurrentDetector, Form("UpdateTableFailCase - Couldn't connect to %s FXS database",
2054                                                         GetSystemName(system)));
2055                         result = kFALSE;
2056                         continue;
2057                 }
2058
2059                 TTimeStamp now; // now
2060
2061                 // Loop on FXS list entries
2062
2063                 TString whereClause = Form("where run=%d and detector=\"%s\";",
2064                                                 GetCurrentRun(), fCurrentDetector.Data());
2065
2066
2067                 TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(system),
2068                                                         now.GetSec(), whereClause.Data());
2069
2070                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2071
2072                 // Query execution
2073                 TSQLResult* aResult;
2074                 aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
2075                 if (!aResult)
2076                 {
2077                         Log(fCurrentDetector, Form("UpdateTableFailCase - %s db: can't execute SQL query <%s>",
2078                                                         GetSystemName(system), sqlQuery.Data()));
2079                         result = kFALSE;
2080                         continue;
2081                 }
2082                 delete aResult;
2083         }
2084
2085         return result;
2086 }
2087
2088 //______________________________________________________________________________________________
2089 Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status)
2090 {
2091         //
2092         // Update Shuttle logbook filling detector or shuttle_done column
2093         // ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
2094         //
2095
2096         // check connection, in case connect
2097         if(!Connect(3)){
2098                 Log("SHUTTLE", "UpdateShuttleLogbook - Couldn't connect to DAQ Logbook.");
2099                 return kFALSE;
2100         }
2101
2102         TString detName(detector);
2103         TString setClause;
2104         if(detName == "shuttle_done")
2105         {
2106                 setClause = "set shuttle_done=1";
2107
2108                 // Send the information to ML
2109                 TMonaLisaText  mlStatus("SHUTTLE_status", "Done");
2110
2111                 TList mlList;
2112                 mlList.Add(&mlStatus);
2113
2114                 fMonaLisa->SendParameters(&mlList);
2115         } else {
2116                 TString statusStr(status);
2117                 if(statusStr.Contains("done", TString::kIgnoreCase) ||
2118                    statusStr.Contains("failed", TString::kIgnoreCase)){
2119                         setClause = Form("set %s=\"%s\"", detector, status);
2120                 } else {
2121                         Log("SHUTTLE",
2122                                 Form("UpdateShuttleLogbook - Invalid status <%s> for detector %s",
2123                                         status, detector));
2124                         return kFALSE;
2125                 }
2126         }
2127
2128         TString whereClause = Form("where run=%d", GetCurrentRun());
2129
2130         TString sqlQuery = Form("update %s %s %s",
2131                                         fConfig->GetShuttlelbTable(), setClause.Data(), whereClause.Data());
2132
2133         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2134
2135         // Query execution
2136         TSQLResult* aResult;
2137         aResult = dynamic_cast<TSQLResult*> (fServer[3]->Query(sqlQuery));
2138         if (!aResult) {
2139                 Log("SHUTTLE", Form("UpdateShuttleLogbook - Can't execute query <%s>", sqlQuery.Data()));
2140                 return kFALSE;
2141         }
2142         delete aResult;
2143
2144         return kTRUE;
2145 }
2146
2147 //______________________________________________________________________________________________
2148 Int_t AliShuttle::GetCurrentRun() const
2149 {
2150         //
2151         // Get current run from logbook entry
2152         //
2153
2154         return fLogbookEntry ? fLogbookEntry->GetRun() : -1;
2155 }
2156
2157 //______________________________________________________________________________________________
2158 UInt_t AliShuttle::GetCurrentStartTime() const
2159 {
2160         //
2161         // get current start time
2162         //
2163
2164         return fLogbookEntry ? fLogbookEntry->GetStartTime() : 0;
2165 }
2166
2167 //______________________________________________________________________________________________
2168 UInt_t AliShuttle::GetCurrentEndTime() const
2169 {
2170         //
2171         // get current end time from logbook entry
2172         //
2173
2174         return fLogbookEntry ? fLogbookEntry->GetEndTime() : 0;
2175 }
2176
2177 //______________________________________________________________________________________________
2178 void AliShuttle::Log(const char* detector, const char* message)
2179 {
2180         //
2181         // Fill log string with a message
2182         //
2183
2184         void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
2185         if (dir == NULL) {
2186                 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE)) {
2187                         AliError(Form("Can't open directory <%s>", GetShuttleLogDir()));
2188                         return;
2189                 }
2190
2191         } else {
2192                 gSystem->FreeDirectory(dir);
2193         }
2194
2195         TString toLog = Form("%s (%d): %s - ", TTimeStamp(time(0)).AsString("s"), getpid(), detector);
2196         if (GetCurrentRun() >= 0)
2197                 toLog += Form("run %d - ", GetCurrentRun());
2198         toLog += Form("%s", message);
2199
2200         AliInfo(toLog.Data());
2201
2202         // if we redirect the log output already to the file, leave here
2203         if (fOutputRedirected && strcmp(detector, "SHUTTLE") != 0)
2204                 return;
2205
2206         TString fileName = GetLogFileName(detector);
2207
2208         gSystem->ExpandPathName(fileName);
2209
2210         ofstream logFile;
2211         logFile.open(fileName, ofstream::out | ofstream::app);
2212
2213         if (!logFile.is_open()) {
2214                 AliError(Form("Could not open file %s", fileName.Data()));
2215                 return;
2216         }
2217
2218         logFile << toLog.Data() << "\n";
2219
2220         logFile.close();
2221 }
2222
2223 //______________________________________________________________________________________________
2224 TString AliShuttle::GetLogFileName(const char* detector) const
2225 {
2226         //
2227         // returns the name of the log file for a given sub detector
2228         //
2229
2230         TString fileName;
2231
2232         if (GetCurrentRun() >= 0)
2233                 fileName.Form("%s/%s_%d.log", GetShuttleLogDir(), detector, GetCurrentRun());
2234         else
2235                 fileName.Form("%s/%s.log", GetShuttleLogDir(), detector);
2236
2237         return fileName;
2238 }
2239
2240 //______________________________________________________________________________________________
2241 Bool_t AliShuttle::Collect(Int_t run)
2242 {
2243         //
2244         // Collects conditions data for all UNPROCESSED run written to DAQ LogBook in case of run = -1 (default)
2245         // If a dedicated run is given this run is processed
2246         //
2247         // In operational mode, this is the Shuttle function triggered by the EOR signal.
2248         //
2249
2250         if (run == -1)
2251                 Log("SHUTTLE","Collect - Shuttle called. Collecting conditions data for unprocessed runs");
2252         else
2253                 Log("SHUTTLE", Form("Collect - Shuttle called. Collecting conditions data for run %d", run));
2254
2255         SetLastAction("Starting");
2256
2257         TString whereClause("where shuttle_done=0");
2258         if (run != -1)
2259                 whereClause += Form(" and run=%d", run);
2260
2261         TObjArray shuttleLogbookEntries;
2262         if (!QueryShuttleLogbook(whereClause, shuttleLogbookEntries))
2263         {
2264                 Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
2265                 return kFALSE;
2266         }
2267
2268         if (shuttleLogbookEntries.GetEntries() == 0)
2269         {
2270                 if (run == -1)
2271                         Log("SHUTTLE","Collect - Found no UNPROCESSED runs in Shuttle logbook");
2272                 else
2273                         Log("SHUTTLE", Form("Collect - Run %d is already DONE "
2274                                                 "or it does not exist in Shuttle logbook", run));
2275                 return kTRUE;
2276         }
2277
2278         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
2279                 fFirstUnprocessed[iDet] = kTRUE;
2280
2281         if (run != -1)
2282         {
2283                 // query Shuttle logbook for earlier runs, check if some detectors are unprocessed,
2284                 // flag them into fFirstUnprocessed array
2285                 TString whereClause(Form("where shuttle_done=0 and run < %d", run));
2286                 TObjArray tmpLogbookEntries;
2287                 if (!QueryShuttleLogbook(whereClause, tmpLogbookEntries))
2288                 {
2289                         Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
2290                         return kFALSE;
2291                 }
2292
2293                 TIter iter(&tmpLogbookEntries);
2294                 AliShuttleLogbookEntry* anEntry = 0;
2295                 while ((anEntry = dynamic_cast<AliShuttleLogbookEntry*> (iter.Next())))
2296                 {
2297                         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
2298                         {
2299                                 if (anEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
2300                                 {
2301                                         AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
2302                                                         anEntry->GetRun(), GetDetName(iDet)));
2303                                         fFirstUnprocessed[iDet] = kFALSE;
2304                                 }
2305                         }
2306
2307                 }
2308
2309         }
2310
2311         if (!RetrieveConditionsData(shuttleLogbookEntries))
2312         {
2313                 Log("SHUTTLE", "Collect - Process of at least one run failed");
2314                 return kFALSE;
2315         }
2316
2317         Log("SHUTTLE", "Collect - Requested run(s) successfully processed");
2318         return kTRUE;
2319 }
2320
2321 //______________________________________________________________________________________________
2322 Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
2323 {
2324         //
2325         // Retrieve conditions data for all runs that aren't processed yet
2326         //
2327
2328         Bool_t hasError = kFALSE;
2329
2330         TIter iter(&dateEntries);
2331         AliShuttleLogbookEntry* anEntry;
2332
2333         while ((anEntry = (AliShuttleLogbookEntry*) iter.Next())){
2334                 if (!Process(anEntry)){
2335                         hasError = kTRUE;
2336                 }
2337
2338                 // clean SHUTTLE temp directory
2339                 TString filename = Form("%s/*.shuttle", GetShuttleTempDir());
2340                 RemoveFile(filename.Data());
2341         }
2342
2343         return hasError == kFALSE;
2344 }
2345
2346 //______________________________________________________________________________________________
2347 ULong_t AliShuttle::GetTimeOfLastAction() const
2348 {
2349         //
2350         // Gets time of last action
2351         //
2352
2353         ULong_t tmp;
2354
2355         fMonitoringMutex->Lock();
2356
2357         tmp = fLastActionTime;
2358
2359         fMonitoringMutex->UnLock();
2360
2361         return tmp;
2362 }
2363
2364 //______________________________________________________________________________________________
2365 const TString AliShuttle::GetLastAction() const
2366 {
2367         //
2368         // returns a string description of the last action
2369         //
2370
2371         TString tmp;
2372
2373         fMonitoringMutex->Lock();
2374
2375         tmp = fLastAction;
2376
2377         fMonitoringMutex->UnLock();
2378
2379         return tmp;
2380 }
2381
2382 //______________________________________________________________________________________________
2383 void AliShuttle::SetLastAction(const char* action)
2384 {
2385         //
2386         // updates the monitoring variables
2387         //
2388
2389         fMonitoringMutex->Lock();
2390
2391         fLastAction = action;
2392         fLastActionTime = time(0);
2393
2394         fMonitoringMutex->UnLock();
2395 }
2396
2397 //______________________________________________________________________________________________
2398 const char* AliShuttle::GetRunParameter(const char* param)
2399 {
2400         //
2401         // returns run parameter read from DAQ logbook
2402         //
2403
2404         if(!fLogbookEntry) {
2405                 AliError("No logbook entry!");
2406                 return 0;
2407         }
2408
2409         return fLogbookEntry->GetRunParameter(param);
2410 }
2411
2412 //______________________________________________________________________________________________
2413 AliCDBEntry* AliShuttle::GetFromOCDB(const char* detector, const AliCDBPath& path)
2414 {
2415         //
2416         // returns object from OCDB valid for current run
2417         //
2418
2419         if (fTestMode & kErrorOCDB)
2420         {
2421                 Log(detector, "GetFromOCDB - In TESTMODE - Simulating error with OCDB");
2422                 return 0;
2423         }
2424
2425         AliCDBStorage *sto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
2426         if (!sto)
2427         {
2428                 Log(detector, "GetFromOCDB - Cannot activate main OCDB for query!");
2429                 return 0;
2430         }
2431
2432         return dynamic_cast<AliCDBEntry*> (sto->Get(path, GetCurrentRun()));
2433 }
2434
2435 //______________________________________________________________________________________________
2436 Bool_t AliShuttle::SendMail()
2437 {
2438         //
2439         // sends a mail to the subdetector expert in case of preprocessor error
2440         //
2441
2442         if (fTestMode != kNone)
2443                 return kTRUE;
2444
2445         void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
2446         if (dir == NULL)
2447         {
2448                 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE))
2449                 {
2450                         AliError(Form("Can't open directory <%s>", GetShuttleLogDir()));
2451                         return kFALSE;
2452                 }
2453
2454         } else {
2455                 gSystem->FreeDirectory(dir);
2456         }
2457
2458         TString bodyFileName;
2459         bodyFileName.Form("%s/mail.body", GetShuttleLogDir());
2460         gSystem->ExpandPathName(bodyFileName);
2461
2462         ofstream mailBody;
2463         mailBody.open(bodyFileName, ofstream::out);
2464
2465         if (!mailBody.is_open())
2466         {
2467                 AliError(Form("Could not open mail body file %s", bodyFileName.Data()));
2468                 return kFALSE;
2469         }
2470
2471         TString to="";
2472         TIter iterExperts(fConfig->GetResponsibles(fCurrentDetector));
2473         TObjString *anExpert=0;
2474         while ((anExpert = (TObjString*) iterExperts.Next()))
2475         {
2476                 to += Form("%s,", anExpert->GetName());
2477         }
2478         to.Remove(to.Length()-1);
2479         AliDebug(2, Form("to: %s",to.Data()));
2480
2481         if (to.IsNull()) {
2482                 AliInfo("List of detector responsibles not yet set!");
2483                 return kFALSE;
2484         }
2485
2486         TString cc="alberto.colla@cern.ch";
2487
2488         TString subject = Form("%s Shuttle preprocessor error in run %d !",
2489                                 fCurrentDetector.Data(), GetCurrentRun());
2490         AliDebug(2, Form("subject: %s", subject.Data()));
2491
2492         TString body = Form("Dear %s expert(s), \n\n", fCurrentDetector.Data());
2493         body += Form("SHUTTLE just detected that your preprocessor "
2494                         "exited with ERROR state in run %d!!\n\n", GetCurrentRun());
2495         body += Form("Please check %s status on the web page asap!\n\n", fCurrentDetector.Data());
2496         body += Form("The last 10 lines of %s log file are following:\n\n");
2497
2498         AliDebug(2, Form("Body begin: %s", body.Data()));
2499
2500         mailBody << body.Data();
2501         mailBody.close();
2502         mailBody.open(bodyFileName, ofstream::out | ofstream::app);
2503
2504         TString logFileName = Form("%s/%s_%d.log", GetShuttleLogDir(), fCurrentDetector.Data(), GetCurrentRun());
2505         TString tailCommand = Form("tail -n 10 %s >> %s", logFileName.Data(), bodyFileName.Data());
2506         if (gSystem->Exec(tailCommand.Data()))
2507         {
2508                 mailBody << Form("%s log file not found ...\n\n", fCurrentDetector.Data());
2509         }
2510
2511         TString endBody = Form("------------------------------------------------------\n\n");
2512         endBody += Form("In case of problems please contact the SHUTTLE core team.\n\n");
2513         endBody += "Please do not answer this message directly, it is automatically generated.\n\n";
2514         endBody += "Sincerely yours,\n\n \t\t\tthe SHUTTLE\n";
2515
2516         AliDebug(2, Form("Body end: %s", endBody.Data()));
2517
2518         mailBody << endBody.Data();
2519
2520         mailBody.close();
2521
2522         // send mail!
2523         TString mailCommand = Form("mail -s \"%s\" -c %s %s < %s",
2524                                                 subject.Data(),
2525                                                 cc.Data(),
2526                                                 to.Data(),
2527                                                 bodyFileName.Data());
2528         AliDebug(2, Form("mail command: %s", mailCommand.Data()));
2529
2530         Bool_t result = gSystem->Exec(mailCommand.Data());
2531
2532         return result == 0;
2533 }
2534
2535 //______________________________________________________________________________________________
2536 const char* AliShuttle::GetRunType()
2537 {
2538         //
2539         // returns run type read from "run type" logbook
2540         //
2541
2542         if(!fLogbookEntry) {
2543                 AliError("No logbook entry!");
2544                 return 0;
2545         }
2546
2547         return fLogbookEntry->GetRunType();
2548 }
2549
2550 //______________________________________________________________________________________________
2551 void AliShuttle::SetShuttleTempDir(const char* tmpDir)
2552 {
2553         //
2554         // sets Shuttle temp directory
2555         //
2556
2557         fgkShuttleTempDir = gSystem->ExpandPathName(tmpDir);
2558 }
2559
2560 //______________________________________________________________________________________________
2561 void AliShuttle::SetShuttleLogDir(const char* logDir)
2562 {
2563         //
2564         // sets Shuttle log directory
2565         //
2566
2567         fgkShuttleLogDir = gSystem->ExpandPathName(logDir);
2568 }