SHUTTLE/AliShuttle.cxx

   1 /**************************************************************************
   2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
   3  *                                                                        *
   4  * Author: The ALICE Off-line Project.                                    *
   5  * Contributors are mentioned in the code where appropriate.              *
   6  *                                                                        *
   7  * Permission to use, copy, modify and distribute this software and its   *
   8  * documentation strictly for non-commercial purposes is hereby granted   *
   9  * without fee, provided that the above copyright notice appears in all   *
  10  * copies and that both the copyright notice and this permission notice   *
  11  * appear in the supporting documentation. The authors make no claims     *
  12  * about the suitability of this software for any purpose. It is          *
  13  * provided "as is" without express or implied warranty.                  *
  14  **************************************************************************/
  15
  16 /*
  17 $Log$
  18 Revision 1.63  2007/11/02 10:53:16  acolla
  19 Protection added to AliShuttle::CopyFileLocally
  20
  21 Revision 1.62  2007/10/31 18:23:13  acolla
  22 Furter developement on the Shuttle:
  23
  24 - Shuttle now connects to the Grid as alidaq. The OCDB and Reference folders
  25 are now built from /alice/data, e.g.:
  26 /alice/data/2007/LHC07a/OCDB
  27
  28 the year and LHC period are taken from the Shuttle.
  29 Raw metadata files are stored by GRP to:
  30 /alice/data/2007/LHC07a/<runNb>/Raw/RunMetadata.root
  31
  32 - Shuttle sends a mail to DCS experts each time DP retrieval fails.
  33
  34 Revision 1.61  2007/10/30 20:33:51  acolla
  35 Improved managing of temporary folders, which weren't correctly handled.
  36 Resolved bug introduced in StoreReferenceFile, which caused SPD preprocessor fail.
  37
  38 Revision 1.60  2007/10/29 18:06:16  acolla
  39
  40 New function StoreRunMetadataFile added to preprocessor and Shuttle interface
  41 This function can be used by GRP only. It stores raw data tags merged file to the
  42 raw data folder (e.g. /alice/data/2008/LHC08a/000099999/Raw).
  43
  44 KNOWN ISSUES:
  45
  46 1. Shuttle cannot write to /alice/data/ because it belongs to alidaq. Tag file is stored in /alice/simulation/... for the time being.
  47 2. Due to a bug in TAlien::Mkdir, the creation of a folder in recursive mode (-p option) does not work. The problem
  48 has been corrected in the root package on the Shuttle machine.
  49
  50 Revision 1.59  2007/10/05 12:40:55  acolla
  51
  52 Result error code added to AliDCSClient data members (it was "lost" with the new implementation of TMap* GetAliasValues and GetDPValues).
  53
  54 Revision 1.58  2007/09/28 15:27:40  acolla
  55
  56 AliDCSClient "multiSplit" option added in the DCS configuration
  57 in AliDCSMessage: variable MAX_BODY_SIZE set to 500000
  58
  59 Revision 1.57  2007/09/27 16:53:13  acolla
  60 Detectors can have more than one AMANDA server. SHUTTLE queries the servers sequentially,
  61 merges the dcs aliases/DPs in one TMap and sends it to the preprocessor.
  62
  63 Revision 1.56  2007/09/14 16:46:14  jgrosseo
  64 1) Connect and Close are called before and after each query, so one can
  65 keep the same AliDCSClient object.
  66 2) The splitting of a query is moved to GetDPValues/GetAliasValues.
  67 3) Splitting interval can be specified in constructor
  68
  69 Revision 1.55  2007/08/06 12:26:40  acolla
  70 Function Bool_t GetHLTStatus added to preprocessor. It returns the status of HLT
  71 read from the run logbook.
  72
  73 Revision 1.54  2007/07/12 09:51:25  jgrosseo
  74 removed duplicated log message in GetFile
  75
  76 Revision 1.53  2007/07/12 09:26:28  jgrosseo
  77 updating hlt fxs base path
  78
  79 Revision 1.52  2007/07/12 08:06:45  jgrosseo
  80 adding log messages in getfile... functions
  81 adding not implemented copy constructor in alishuttleconfigholder
  82
  83 Revision 1.51  2007/07/03 17:24:52  acolla
  84 root moved to v5-16-00. TFileMerger->Cp moved to TFile::Cp.
  85
  86 Revision 1.50  2007/07/02 17:19:32  acolla
  87 preprocessor is run in a temp directory that is removed when process is finished.
  88
  89 Revision 1.49  2007/06/29 10:45:06  acolla
  90 Number of columns in MySql Shuttle logbook increased by one (HLT added)
  91
  92 Revision 1.48  2007/06/21 13:06:19  acolla
  93 GetFileSources returns dummy list with 1 source if system=DCS (better than
  94 returning error as it was)
  95
  96 Revision 1.47  2007/06/19 17:28:56  acolla
  97 HLT updated; missing map bug removed.
  98
  99 Revision 1.46  2007/06/09 13:01:09  jgrosseo
 100 Switching to retrieval of several DCS DPs at a time (multiDPrequest)
 101
 102 Revision 1.45  2007/05/30 06:35:20  jgrosseo
 103 Adding functionality to the Shuttle/TestShuttle:
 104 o) Function to retrieve list of sources from a given system (GetFileSources with id=0)
 105 o) Function to retrieve list of IDs for a given source      (GetFileIDs)
 106 These functions are needed for dealing with the tag files that are saved for the GRP preprocessor
 107 Example code has been added to the TestProcessor in TestShuttle
 108
 109 Revision 1.44  2007/05/11 16:09:32  acolla
 110 Reference files for ITS, MUON and PHOS are now stored in OfflineDetName/OnlineDetName/run_...
 111 example: ITS/SPD/100_filename.root
 112
 113 Revision 1.43  2007/05/10 09:59:51  acolla
 114 Various bug fixes in StoreRefFilesToGrid; Cleaning of reference storage before processing detector (CleanReferenceStorage)
 115
 116 Revision 1.42  2007/05/03 08:01:39  jgrosseo
 117 typo in last commit :-(
 118
 119 Revision 1.41  2007/05/03 08:00:48  jgrosseo
 120 fixing log message when pp want to skip dcs value retrieval
 121
 122 Revision 1.40  2007/04/27 07:06:48  jgrosseo
 123 GetFileSources returns empty list in case of no files, but successful query
 124 No mails sent in testmode
 125
 126 Revision 1.39  2007/04/17 12:43:57  acolla
 127 Correction in StoreOCDB; change of text in mail to detector expert
 128
 129 Revision 1.38  2007/04/12 08:26:18  jgrosseo
 130 updated comment
 131
 132 Revision 1.37  2007/04/10 16:53:14  jgrosseo
 133 redirecting sub detector stdout, stderr to sub detector log file
 134
 135 Revision 1.35  2007/04/04 16:26:38  acolla
 136 1. Re-organization of function calls in TestPreprocessor to make it more meaningful.
 137 2. Added missing dependency in test preprocessors.
 138 3. in AliShuttle.cxx: processing time and memory consumption info on a single line.
 139
 140 Revision 1.34  2007/04/04 10:33:36  jgrosseo
 141 1) Storing of files to the Grid is now done _after_ your preprocessors succeeded. This is transparent, which means that you can still use the same functions (Store, StoreReferenceData) to store files to the Grid. However, the Shuttle first stores them locally and transfers them after the preprocessor finished. The return code of these two functions has changed from UInt_t to Bool_t which gives you the success of the storing.
 142 In case of an error with the Grid, the Shuttle will retry the storing later, the preprocessor does not need to be run again.
 143
 144 2) The meaning of the return code of the preprocessor has changed. 0 is now success and any other value means failure. This value is stored in the log and you can use it to keep details about the error condition.
 145
 146 3) New function StoreReferenceFile to _directly_ store a file (without opening it) to the reference storage.
 147
 148 4) The memory usage of the preprocessor is monitored. If it exceeds 2 GB it is terminated.
 149
 150 5) New function AliPreprocessor::ProcessDCS(). If you do not need to have DCS data in all cases, you can skip the processing by implemting this function and returning kFALSE under certain conditions. E.g. if there is a certain run type.
 151 If you always need DCS data (like before), you do not need to implement it.
 152
 153 6) The run type has been added to the monitoring page
 154
 155 Revision 1.33  2007/04/03 13:56:01  acolla
 156 Grid Storage at the end of preprocessing. Added virtual method to disable DCS query according to the
 157 run type.
 158
 159 Revision 1.32  2007/02/28 10:41:56  acolla
 160 Run type field added in SHUTTLE framework. Run type is read from "run type" logbook and retrieved by
 161 AliPreprocessor::GetRunType() function.
 162 Added some ldap definition files.
 163
 164 Revision 1.30  2007/02/13 11:23:21  acolla
 165 Moved getters and setters of Shuttle's main OCDB/Reference, local
 166 OCDB/Reference, temp and log folders to AliShuttleInterface
 167
 168 Revision 1.27  2007/01/30 17:52:42  jgrosseo
 169 adding monalisa monitoring
 170
 171 Revision 1.26  2007/01/23 19:20:03  acolla
 172 Removed old ldif files, added TOF, MCH ldif files. Added some options in
 173 AliShuttleConfig::Print. Added in Ali Shuttle: SetShuttleTempDir and
 174 SetShuttleLogDir
 175
 176 Revision 1.25  2007/01/15 19:13:52  acolla
 177 Moved some AliInfo to AliDebug in SendMail function
 178
 179 Revision 1.21  2006/12/07 08:51:26  jgrosseo
 180 update (alberto):
 181 table, db names in ldap configuration
 182 added GRP preprocessor
 183 DCS data can also be retrieved by data point
 184
 185 Revision 1.20  2006/11/16 16:16:48  jgrosseo
 186 introducing strict run ordering flag
 187 removed giving preprocessor name to preprocessor, they have to know their name themselves ;-)
 188
 189 Revision 1.19  2006/11/06 14:23:04  jgrosseo
 190 major update (Alberto)
 191 o) reading of run parameters from the logbook
 192 o) online offline naming conversion
 193 o) standalone DCSclient package
 194
 195 Revision 1.18  2006/10/20 15:22:59  jgrosseo
 196 o) Adding time out to the execution of the preprocessors: The Shuttle forks and the parent process monitors the child
 197 o) Merging Collect, CollectAll, CollectNew function
 198 o) Removing implementation of empty copy constructors (declaration still there!)
 199
 200 Revision 1.17  2006/10/05 16:20:55  jgrosseo
 201 adapting to new CDB classes
 202
 203 Revision 1.16  2006/10/05 15:46:26  jgrosseo
 204 applying to the new interface
 205
 206 Revision 1.15  2006/10/02 16:38:39  jgrosseo
 207 update (alberto):
 208 fixed memory leaks
 209 storing of objects that failed to be stored to the grid before
 210 interfacing of shuttle status table in daq system
 211
 212 Revision 1.14  2006/08/29 09:16:05  jgrosseo
 213 small update
 214
 215 Revision 1.13  2006/08/15 10:50:00  jgrosseo
 216 effc++ corrections (alberto)
 217
 218 Revision 1.12  2006/08/08 14:19:29  jgrosseo
 219 Update to shuttle classes (Alberto)
 220
 221 - Possibility to set the full object's path in the Preprocessor's and
 222 Shuttle's  Store functions
 223 - Possibility to extend the object's run validity in the same classes
 224 ("startValidity" and "validityInfinite" parameters)
 225 - Implementation of the StoreReferenceData function to store reference
 226 data in a dedicated CDB storage.
 227
 228 Revision 1.11  2006/07/21 07:37:20  jgrosseo
 229 last run is stored after each run
 230
 231 Revision 1.10  2006/07/20 09:54:40  jgrosseo
 232 introducing status management: The processing per subdetector is divided into several steps,
 233 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
 234 can keep track of the number of failures and skips further processing after a certain threshold is
 235 exceeded. These thresholds can be configured in LDAP.
 236
 237 Revision 1.9  2006/07/19 10:09:55  jgrosseo
 238 new configuration, accesst to DAQ FES (Alberto)
 239
 240 Revision 1.8  2006/07/11 12:44:36  jgrosseo
 241 adding parameters for extended validity range of data produced by preprocessor
 242
 243 Revision 1.7  2006/07/10 14:37:09  jgrosseo
 244 small fix + todo comment
 245
 246 Revision 1.6  2006/07/10 13:01:41  jgrosseo
 247 enhanced storing of last sucessfully processed run (alberto)
 248
 249 Revision 1.5  2006/07/04 14:59:57  jgrosseo
 250 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
 251
 252 Revision 1.4  2006/06/12 09:11:16  jgrosseo
 253 coding conventions (Alberto)
 254
 255 Revision 1.3  2006/06/06 14:26:40  jgrosseo
 256 o) removed files that were moved to STEER
 257 o) shuttle updated to follow the new interface (Alberto)
 258
 259 Revision 1.2  2006/03/07 07:52:34  hristov
 260 New version (B.Yordanov)
 261
 262 Revision 1.6  2005/11/19 17:19:14  byordano
 263 RetrieveDATEEntries and RetrieveConditionsData added
 264
 265 Revision 1.5  2005/11/19 11:09:27  byordano
 266 AliShuttle declaration added
 267
 268 Revision 1.4  2005/11/17 17:47:34  byordano
 269 TList changed to TObjArray
 270
 271 Revision 1.3  2005/11/17 14:43:23  byordano
 272 import to local CVS
 273
 274 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
 275 Initial import as subdirectory in AliRoot
 276
 277 Revision 1.2  2005/09/13 08:41:15  byordano
 278 default startTime endTime added
 279
 280 Revision 1.4  2005/08/30 09:13:02  byordano
 281 some docs added
 282
 283 Revision 1.3  2005/08/29 21:15:47  byordano
 284 some docs added
 285
 286 */
 287
 288 //
 289 // This class is the main manager for AliShuttle.
 290 // It organizes the data retrieval from DCS and call the
 291 // interface methods of AliPreprocessor.
 292 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
 293 // data for its set of aliases is retrieved. If there is registered
 294 // AliPreprocessor for this detector then it will be used
 295 // accroding to the schema (see AliPreprocessor).
 296 // If there isn't registered AliPreprocessor than the retrieved
 297 // data is stored automatically to the undelying AliCDBStorage.
 298 // For detSpec is used the alias name.
 299 //
 300
 301 #include "AliShuttle.h"
 302
 303 #include "AliCDBManager.h"
 304 #include "AliCDBStorage.h"
 305 #include "AliCDBId.h"
 306 #include "AliCDBRunRange.h"
 307 #include "AliCDBPath.h"
 308 #include "AliCDBEntry.h"
 309 #include "AliShuttleConfig.h"
 310 #include "DCSClient/AliDCSClient.h"
 311 #include "AliLog.h"
 312 #include "AliPreprocessor.h"
 313 #include "AliShuttleStatus.h"
 314 #include "AliShuttleLogbookEntry.h"
 315
 316 #include <TSystem.h>
 317 #include <TObject.h>
 318 #include <TString.h>
 319 #include <TTimeStamp.h>
 320 #include <TObjString.h>
 321 #include <TSQLServer.h>
 322 #include <TSQLResult.h>
 323 #include <TSQLRow.h>
 324 #include <TMutex.h>
 325 #include <TSystemDirectory.h>
 326 #include <TSystemFile.h>
 327 #include <TFile.h>
 328 #include <TGrid.h>
 329 #include <TGridResult.h>
 330
 331 #include <TMonaLisaWriter.h>
 332
 333 #include <fstream>
 334
 335 #include <sys/types.h>
 336 #include <sys/wait.h>
 337
 338 ClassImp(AliShuttle)
 339
 340 //______________________________________________________________________________________________
 341 AliShuttle::AliShuttle(const AliShuttleConfig* config,
 342                 UInt_t timeout, Int_t retries):
 343 fConfig(config),
 344 fTimeout(timeout), fRetries(retries),
 345 fPreprocessorMap(),
 346 fLogbookEntry(0),
 347 fCurrentDetector(),
 348 fStatusEntry(0),
 349 fMonitoringMutex(0),
 350 fLastActionTime(0),
 351 fLastAction(),
 352 fMonaLisa(0),
 353 fTestMode(kNone),
 354 fReadTestMode(kFALSE),
 355 fOutputRedirected(kFALSE)
 356 {
 357         //
 358         // config: AliShuttleConfig used
 359         // timeout: timeout used for AliDCSClient connection
 360         // retries: the number of retries in case of connection error.
 361         //
 362
 363         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
 364         for(int iSys=0;iSys<4;iSys++) {
 365                 fServer[iSys]=0;
 366                 if (iSys < 3)
 367                         fFXSlist[iSys].SetOwner(kTRUE);
 368         }
 369         fPreprocessorMap.SetOwner(kTRUE);
 370
 371         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
 372                 fFirstUnprocessed[iDet] = kFALSE;
 373
 374         fMonitoringMutex = new TMutex();
 375 }
 376
 377 //______________________________________________________________________________________________
 378 AliShuttle::~AliShuttle()
 379 {
 380         //
 381         // destructor
 382         //
 383
 384         fPreprocessorMap.DeleteAll();
 385         for(int iSys=0;iSys<4;iSys++)
 386                 if(fServer[iSys]) {
 387                         fServer[iSys]->Close();
 388                         delete fServer[iSys];
 389                         fServer[iSys] = 0;
 390                 }
 391
 392         if (fStatusEntry){
 393                 delete fStatusEntry;
 394                 fStatusEntry = 0;
 395         }
 396
 397         if (fMonitoringMutex)
 398         {
 399                 delete fMonitoringMutex;
 400                 fMonitoringMutex = 0;
 401         }
 402 }
 403
 404 //______________________________________________________________________________________________
 405 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
 406 {
 407         //
 408         // Registers new AliPreprocessor.
 409         // It uses GetName() for indentificator of the pre processor.
 410         // The pre processor is registered it there isn't any other
 411         // with the same identificator (GetName()).
 412         //
 413
 414         const char* detName = preprocessor->GetName();
 415         if(GetDetPos(detName) < 0)
 416                 AliFatal(Form("********** !!!!! Invalid detector name: %s !!!!! **********", detName));
 417
 418         if (fPreprocessorMap.GetValue(detName)) {
 419                 AliWarning(Form("AliPreprocessor %s is already registered!", detName));
 420                 return;
 421         }
 422
 423         fPreprocessorMap.Add(new TObjString(detName), preprocessor);
 424 }
 425 //______________________________________________________________________________________________
 426 Bool_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
 427                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
 428 {
 429         // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
 430         // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
 431         // using this function. Use StoreReferenceData instead!
 432         // It calls StoreLocally function which temporarily stores the data locally; when the preprocessor
 433         // finishes the data are transferred to the main storage (Grid).
 434
 435         return StoreLocally(fgkLocalCDB, path, object, metaData, validityStart, validityInfinite);
 436 }
 437
 438 //______________________________________________________________________________________________
 439 Bool_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData)
 440 {
 441         // Stores a CDB object in the storage for reference data. This objects will not be available during
 442         // offline reconstrunction. Use this function for reference data only!
 443         // It calls StoreLocally function which temporarily stores the data locally; when the preprocessor
 444         // finishes the data are transferred to the main storage (Grid).
 445
 446         return StoreLocally(fgkLocalRefStorage, path, object, metaData);
 447 }
 448
 449 //______________________________________________________________________________________________
 450 Bool_t AliShuttle::StoreLocally(const TString& localUri,
 451                         const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
 452                         Int_t validityStart, Bool_t validityInfinite)
 453 {
 454         // Store object temporarily in local storage. Parameters are passed by Store and StoreReferenceData functions.
 455         // when the preprocessor finishes the data are transferred to the main storage (Grid).
 456         // The parameters are:
 457         //   1) Uri of the backup storage (Local)
 458         //   2) the object's path.
 459         //   3) the object to be stored
 460         //   4) the metaData to be associated with the object
 461         //   5) the validity start run number w.r.t. the current run,
 462         //      if the data is valid only for this run leave the default 0
 463         //   6) specifies if the calibration data is valid for infinity (this means until updated),
 464         //      typical for calibration runs, the default is kFALSE
 465         //
 466         // returns 0 if fail, 1 otherwise
 467
 468         if (fTestMode & kErrorStorage)
 469         {
 470                 Log(fCurrentDetector, "StoreLocally - In TESTMODE - Simulating error while storing locally");
 471                 return kFALSE;
 472         }
 473
 474         const char* cdbType = (localUri == fgkLocalCDB) ? "CDB" : "Reference";
 475
 476         Int_t firstRun = GetCurrentRun() - validityStart;
 477         if(firstRun < 0) {
 478                 AliWarning("First valid run happens to be less than 0! Setting it to 0.");
 479                 firstRun=0;
 480         }
 481
 482         Int_t lastRun = -1;
 483         if(validityInfinite) {
 484                 lastRun = AliCDBRunRange::Infinity();
 485         } else {
 486                 lastRun = GetCurrentRun();
 487         }
 488
 489         // Version is set to current run, it will be used later to transfer data to Grid
 490         AliCDBId id(path, firstRun, lastRun, GetCurrentRun(), -1);
 491
 492         if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
 493                 TObjString runUsed = Form("%d", GetCurrentRun());
 494                 metaData->SetProperty("RunUsed(TObjString)", runUsed.Clone());
 495         }
 496
 497         Bool_t result = kFALSE;
 498
 499         if (!(AliCDBManager::Instance()->GetStorage(localUri))) {
 500                 Log("SHUTTLE", Form("StoreLocally - Cannot activate local %s storage", cdbType));
 501         } else {
 502                 result = AliCDBManager::Instance()->GetStorage(localUri)
 503                                         ->Put(object, id, metaData);
 504         }
 505
 506         if(!result) {
 507
 508                 Log(fCurrentDetector, Form("StoreLocally - Can't store object <%s>!", id.ToString().Data()));
 509         }
 510
 511         return result;
 512 }
 513
 514 //______________________________________________________________________________________________
 515 Bool_t AliShuttle::StoreOCDB()
 516 {
 517         //
 518         // Called when preprocessor ends successfully or when previous storage attempt failed (kStoreError status)
 519         // Calls underlying StoreOCDB(const char*) function twice, for OCDB and Reference storage.
 520         // Then calls StoreRefFilesToGrid to store reference files.
 521         //
 522
 523         if (fTestMode & kErrorGrid)
 524         {
 525                 Log("SHUTTLE", "StoreOCDB - In TESTMODE - Simulating error while storing in the Grid");
 526                 Log(fCurrentDetector, "StoreOCDB - In TESTMODE - Simulating error while storing in the Grid");
 527                 return kFALSE;
 528         }
 529
 530         Log("SHUTTLE","StoreOCDB - Storing OCDB data ...");
 531         Bool_t resultCDB = StoreOCDB(fgkMainCDB);
 532
 533         Log("SHUTTLE","StoreOCDB - Storing reference data ...");
 534         Bool_t resultRef = StoreOCDB(fgkMainRefStorage);
 535
 536         Log("SHUTTLE","StoreOCDB - Storing reference files ...");
 537         Bool_t resultRefFiles = CopyFilesToGrid("reference");
 538
 539         Bool_t resultMetadata = kTRUE;
 540         if(fCurrentDetector == "GRP")
 541         {
 542                 Log("StoreOCDB - SHUTTLE","Storing Run Metadata file ...");
 543                 resultMetadata = CopyFilesToGrid("metadata");
 544         }
 545
 546         return resultCDB && resultRef && resultRefFiles && resultMetadata;
 547 }
 548
 549 //______________________________________________________________________________________________
 550 Bool_t AliShuttle::StoreOCDB(const TString& gridURI)
 551 {
 552         //
 553         // Called by StoreOCDB(), performs actual storage to the main OCDB and reference storages (Grid)
 554         //
 555
 556         TObjArray* gridIds=0;
 557
 558         Bool_t result = kTRUE;
 559
 560         const char* type = 0;
 561         TString localURI;
 562         if(gridURI == fgkMainCDB) {
 563                 type = "OCDB";
 564                 localURI = fgkLocalCDB;
 565         } else if(gridURI == fgkMainRefStorage) {
 566                 type = "reference";
 567                 localURI = fgkLocalRefStorage;
 568         } else {
 569                 AliError(Form("Invalid storage URI: %s", gridURI.Data()));
 570                 return kFALSE;
 571         }
 572
 573         AliCDBManager* man = AliCDBManager::Instance();
 574
 575         AliCDBStorage *gridSto = man->GetStorage(gridURI);
 576         if(!gridSto) {
 577                 Log("SHUTTLE",
 578                         Form("StoreOCDB - cannot activate main %s storage", type));
 579                 return kFALSE;
 580         }
 581
 582         gridIds = gridSto->GetQueryCDBList();
 583
 584         // get objects previously stored in local CDB
 585         AliCDBStorage *localSto = man->GetStorage(localURI);
 586         if(!localSto) {
 587                 Log("SHUTTLE",
 588                         Form("StoreOCDB - cannot activate local %s storage", type));
 589                 return kFALSE;
 590         }
 591         AliCDBPath aPath(GetOfflineDetName(fCurrentDetector.Data()),"*","*");
 592         // Local objects were stored with current run as Grid version!
 593         TList* localEntries = localSto->GetAll(aPath.GetPath(), GetCurrentRun(), GetCurrentRun());
 594         localEntries->SetOwner(1);
 595
 596         // loop on local stored objects
 597         TIter localIter(localEntries);
 598         AliCDBEntry *aLocEntry = 0;
 599         while((aLocEntry = dynamic_cast<AliCDBEntry*> (localIter.Next()))){
 600                 aLocEntry->SetOwner(1);
 601                 AliCDBId aLocId = aLocEntry->GetId();
 602                 aLocEntry->SetVersion(-1);
 603                 aLocEntry->SetSubVersion(-1);
 604
 605                 // If local object is valid up to infinity we store it only if it is
 606                 // the first unprocessed run!
 607                 if (aLocId.GetLastRun() == AliCDBRunRange::Infinity() &&
 608                         !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
 609                 {
 610                         Log("SHUTTLE", Form("StoreOCDB - %s: object %s has validity infinite but "
 611                                                 "there are previous unprocessed runs!",
 612                                                 fCurrentDetector.Data(), aLocId.GetPath().Data()));
 613                         continue;
 614                 }
 615
 616                 // loop on Grid valid Id's
 617                 Bool_t store = kTRUE;
 618                 TIter gridIter(gridIds);
 619                 AliCDBId* aGridId = 0;
 620                 while((aGridId = dynamic_cast<AliCDBId*> (gridIter.Next()))){
 621                         if(aGridId->GetPath() != aLocId.GetPath()) continue;
 622                         // skip all objects valid up to infinity
 623                         if(aGridId->GetLastRun() == AliCDBRunRange::Infinity()) continue;
 624                         // if we get here, it means there's already some more recent object stored on Grid!
 625                         store = kFALSE;
 626                         break;
 627                 }
 628
 629                 // If we get here, the file can be stored!
 630                 Bool_t storeOk = gridSto->Put(aLocEntry);
 631                 if(!store || storeOk){
 632
 633                         if (!store)
 634                         {
 635                                 Log(fCurrentDetector.Data(),
 636                                         Form("StoreOCDB - A more recent object already exists in %s storage: <%s>",
 637                                                 type, aGridId->ToString().Data()));
 638                         } else {
 639                                 Log("SHUTTLE",
 640                                         Form("StoreOCDB - Object <%s> successfully put into %s storage",
 641                                                 aLocId.ToString().Data(), type));
 642                                 Log(fCurrentDetector.Data(),
 643                                         Form("StoreOCDB - Object <%s> successfully put into %s storage",
 644                                                 aLocId.ToString().Data(), type));
 645                         }
 646
 647                         // removing local filename...
 648                         TString filename;
 649                         localSto->IdToFilename(aLocId, filename);
 650                         Log("SHUTTLE", Form("StoreOCDB - Removing local file %s", filename.Data()));
 651                         RemoveFile(filename.Data());
 652                         continue;
 653                 } else  {
 654                         Log("SHUTTLE",
 655                                 Form("StoreOCDB - Grid %s storage of object <%s> failed",
 656                                         type, aLocId.ToString().Data()));
 657                         Log(fCurrentDetector.Data(),
 658                                 Form("StoreOCDB - Grid %s storage of object <%s> failed",
 659                                         type, aLocId.ToString().Data()));
 660                         result = kFALSE;
 661                 }
 662         }
 663         localEntries->Clear();
 664
 665         return result;
 666 }
 667
 668 //______________________________________________________________________________________________
 669 Bool_t AliShuttle::CleanReferenceStorage(const char* detector)
 670 {
 671         // clears the directory used to store reference files of a given subdetector
 672
 673         AliCDBManager* man = AliCDBManager::Instance();
 674         AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
 675         TString localBaseFolder = sto->GetBaseFolder();
 676
 677         TString targetDir = GetRefFilePrefix(localBaseFolder.Data(), detector);
 678
 679         Log("SHUTTLE", Form("CleanReferenceStorage - Cleaning %s", targetDir.Data()));
 680
 681         TString begin;
 682         begin.Form("%d_", GetCurrentRun());
 683
 684         TSystemDirectory* baseDir = new TSystemDirectory("/", targetDir);
 685         if (!baseDir)
 686                 return kTRUE;
 687
 688         TList* dirList = baseDir->GetListOfFiles();
 689         delete baseDir;
 690
 691         if (!dirList) return kTRUE;
 692
 693         if (dirList->GetEntries() < 3)
 694         {
 695                 delete dirList;
 696                 return kTRUE;
 697         }
 698
 699         Int_t nDirs = 0, nDel = 0;
 700         TIter dirIter(dirList);
 701         TSystemFile* entry = 0;
 702
 703         Bool_t success = kTRUE;
 704
 705         while ((entry = dynamic_cast<TSystemFile*> (dirIter.Next())))
 706         {
 707                 if (entry->IsDirectory())
 708                         continue;
 709
 710                 TString fileName(entry->GetName());
 711                 if (!fileName.BeginsWith(begin))
 712                         continue;
 713
 714                 nDirs++;
 715
 716                 // delete file
 717                 Int_t result = gSystem->Unlink(fileName.Data());
 718
 719                 if (result)
 720                 {
 721                         Log("SHUTTLE", Form("CleanReferenceStorage - Could not delete file %s!", fileName.Data()));
 722                         success = kFALSE;
 723                 } else {
 724                         nDel++;
 725                 }
 726         }
 727
 728         if(nDirs > 0)
 729                 Log("SHUTTLE", Form("CleanReferenceStorage - %d (over %d) reference files in folder %s were deleted.",
 730                         nDel, nDirs, targetDir.Data()));
 731
 732
 733         delete dirList;
 734         return success;
 735
 736
 737
 738
 739
 740
 741   Int_t result = gSystem->GetPathInfo(targetDir, 0, (Long64_t*) 0, 0, 0);
 742   if (result == 0)
 743   {
 744     // delete directory
 745     result = gSystem->Exec(Form("rm -rf %s", targetDir.Data()));
 746     if (result != 0)
 747     {
 748       Log("SHUTTLE", Form("CleanReferenceStorage - Could not clean directory %s", targetDir.Data()));
 749       return kFALSE;
 750     }
 751   }
 752
 753   result = gSystem->mkdir(targetDir, kTRUE);
 754   if (result != 0)
 755   {
 756     Log("SHUTTLE", Form("CleanReferenceStorage - Error creating base directory %s", targetDir.Data()));
 757     return kFALSE;
 758   }
 759
 760   return kTRUE;
 761 }
 762
 763 //______________________________________________________________________________________________
 764 Bool_t AliShuttle::StoreReferenceFile(const char* detector, const char* localFile, const char* gridFileName)
 765 {
 766         //
 767         // Stores reference file directly (without opening it). This function stores the file locally.
 768         //
 769         // The file is stored under the following location:
 770         // <base folder of local reference storage>/<DET>/<RUN#>_<gridFileName>
 771         // where <gridFileName> is the second parameter given to the function
 772         //
 773
 774         if (fTestMode & kErrorStorage)
 775         {
 776                 Log(fCurrentDetector, "StoreReferenceFile - In TESTMODE - Simulating error while storing locally");
 777                 return kFALSE;
 778         }
 779
 780         AliCDBManager* man = AliCDBManager::Instance();
 781         AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
 782
 783         TString localBaseFolder = sto->GetBaseFolder();
 784
 785         TString target = GetRefFilePrefix(localBaseFolder.Data(), detector);
 786         target.Append(Form("/%d_%s", GetCurrentRun(), gridFileName));
 787
 788         return CopyFileLocally(localFile, target);
 789 }
 790
 791 //______________________________________________________________________________________________
 792 Bool_t AliShuttle::StoreRunMetadataFile(const char* localFile, const char* gridFileName)
 793 {
 794         //
 795         // Stores Run metadata file to the Grid, in the run folder
 796         //
 797         // Only GRP can call this function.
 798
 799         if (fTestMode & kErrorStorage)
 800         {
 801                 Log(fCurrentDetector, "StoreRunMetaDataFile - In TESTMODE - Simulating error while storing locally");
 802                 return kFALSE;
 803         }
 804
 805         AliCDBManager* man = AliCDBManager::Instance();
 806         AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
 807
 808         TString localBaseFolder = sto->GetBaseFolder();
 809
 810         // Build Run level folder
 811         // folder = /alice/data/year/lhcPeriod/runNb/Raw
 812
 813
 814         TString lhcPeriod = GetLHCPeriod();
 815         if (lhcPeriod.Length() == 0)
 816         {
 817                 Log("SHUTTLE","StoreRunMetaDataFile - LHCPeriod not found in logbook!");
 818                 return 0;
 819         }
 820
 821         TString target = Form("%s/GRP/RunMetadata/alice/data/%d/%s/%09d/Raw/%s",
 822                                 localBaseFolder.Data(), GetCurrentYear(),
 823                                 lhcPeriod.Data(), GetCurrentRun(), gridFileName);
 824
 825         return CopyFileLocally(localFile, target);
 826 }
 827
 828 //______________________________________________________________________________________________
 829 Bool_t AliShuttle::CopyFileLocally(const char* localFile, const TString& target)
 830 {
 831         //
 832         // Stores file locally. Called by StoreReferenceFile and StoreRunMetadataFile
 833         // Files are temporarily stored in the local reference storage. When the preprocessor
 834         // finishes, the Shuttle calls CopyFilesToGrid to transfer the files to AliEn
 835         // (in reference or run level folders)
 836         //
 837
 838         TString targetDir(target(0, target.Last('/')));
 839
 840         //try to open base dir folder, if it does not exist
 841         void* dir = gSystem->OpenDirectory(targetDir.Data());
 842         if (dir == NULL) {
 843                 if (gSystem->mkdir(targetDir.Data(), kTRUE)) {
 844                         Log("SHUTTLE", Form("StoreFileLocally - Can't open directory <%s>", targetDir.Data()));
 845                         return kFALSE;
 846                 }
 847
 848         } else {
 849                 gSystem->FreeDirectory(dir);
 850         }
 851
 852         Int_t result = 0;
 853
 854         result = gSystem->GetPathInfo(localFile, 0, (Long64_t*) 0, 0, 0);
 855         if (result)
 856         {
 857                 Log("SHUTTLE", Form("StoreFileLocally - %s does not exist", localFile));
 858                 return kFALSE;
 859         }
 860
 861         result = gSystem->GetPathInfo(target, 0, (Long64_t*) 0, 0, 0);
 862         if (!result)
 863         {
 864                 Log("SHUTTLE", Form("StoreFileLocally - target file %s already exist, removing...", target.Data()));
 865                 if (gSystem->Unlink(target.Data()))
 866                 {
 867                         Log("SHUTTLE", Form("StoreFileLocally - Could not remove existing target file %s!", target.Data()));
 868                         return kFALSE;
 869                 }
 870         }
 871
 872         result = gSystem->CopyFile(localFile, target);
 873
 874         if (result == 0)
 875         {
 876                 Log("SHUTTLE", Form("StoreFileLocally - File %s stored locally to %s", localFile, target.Data()));
 877                 return kTRUE;
 878         }
 879         else
 880         {
 881                 Log("SHUTTLE", Form("StoreFileLocally - Could not store file %s to %s! Error code = %d",
 882                                 localFile, target.Data(), result));
 883                 return kFALSE;
 884         }
 885
 886
 887
 888 }
 889
 890 //______________________________________________________________________________________________
 891 Bool_t AliShuttle::CopyFilesToGrid(const char* type)
 892 {
 893         //
 894         // Transfers local files to the Grid. Local files can be reference files
 895         // or run metadata file (from GRP only).
 896         //
 897         // According to the type (ref, metadata) the files are stored under the following location:
 898         // ref --> <base folder of reference storage>/<DET>/<RUN#>_<gridFileName>
 899         // metadata --> <run data folder>/<MetadataFileName>
 900         //
 901
 902         AliCDBManager* man = AliCDBManager::Instance();
 903         AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
 904         if (!sto)
 905                 return kFALSE;
 906         TString localBaseFolder = sto->GetBaseFolder();
 907
 908         TString dir;
 909         TString alienDir;
 910         TString begin;
 911
 912         if (strcmp(type, "reference") == 0)
 913         {
 914                 dir = GetRefFilePrefix(localBaseFolder.Data(), fCurrentDetector.Data());
 915                 AliCDBStorage* gridSto = man->GetStorage(fgkMainRefStorage);
 916                 if (!gridSto)
 917                         return kFALSE;
 918                 TString gridBaseFolder = gridSto->GetBaseFolder();
 919                 alienDir = GetRefFilePrefix(gridBaseFolder.Data(), fCurrentDetector.Data());
 920                 begin = Form("%d_", GetCurrentRun());
 921         }
 922         else if (strcmp(type, "metadata") == 0)
 923         {
 924
 925                 TString lhcPeriod = GetLHCPeriod();
 926
 927                 if (lhcPeriod.Length() == 0)
 928                 {
 929                         Log("SHUTTLE","CopyFilesToGrid - LHCPeriod not found in logbook!");
 930                         return 0;
 931                 }
 932
 933                 dir = Form("%s/GRP/RunMetadata/alice/data/%d/%s/%09d/Raw",
 934                                 localBaseFolder.Data(), GetCurrentYear(),
 935                                 lhcPeriod.Data(), GetCurrentRun());
 936                 alienDir = dir(dir.Index("/alice/data/"), dir.Length());
 937
 938                 begin = "";
 939         }
 940         else
 941         {
 942                 Log("SHUTTLE", "CopyFilesToGrid - Unexpected: type label must be reference or metadata!");
 943                 return kFALSE;
 944         }
 945
 946         TSystemDirectory* baseDir = new TSystemDirectory("/", dir);
 947         if (!baseDir)
 948                 return kTRUE;
 949
 950         TList* dirList = baseDir->GetListOfFiles();
 951         delete baseDir;
 952
 953         if (!dirList) return kTRUE;
 954
 955         if (dirList->GetEntries() < 3)
 956         {
 957                 delete dirList;
 958                 return kTRUE;
 959         }
 960
 961         if (!gGrid)
 962         {
 963                 Log("SHUTTLE", "CopyFilesToGrid - Connection to Grid failed: Cannot continue!");
 964                 delete dirList;
 965                 return kFALSE;
 966         }
 967
 968         Int_t nDirs = 0, nTransfer = 0;
 969         TIter dirIter(dirList);
 970         TSystemFile* entry = 0;
 971
 972         Bool_t success = kTRUE;
 973         Bool_t first = kTRUE;
 974
 975         while ((entry = dynamic_cast<TSystemFile*> (dirIter.Next())))
 976         {
 977                 if (entry->IsDirectory())
 978                         continue;
 979
 980                 TString fileName(entry->GetName());
 981                 if (!fileName.BeginsWith(begin))
 982                         continue;
 983
 984                 nDirs++;
 985
 986                 if (first)
 987                 {
 988                         first = kFALSE;
 989                         // check that folder exists, otherwise create it
 990                         TGridResult* result = gGrid->Ls(alienDir.Data(), "a");
 991
 992                         if (!result)
 993                         {
 994                                 delete dirList;
 995                                 return kFALSE;
 996                         }
 997
 998                         if (!result->GetFileName(1)) // TODO: It looks like element 0 is always 0!!
 999                         {
1000                                 // TODO It does not work currently! Bug in TAliEn::Mkdir
1001                                 // TODO Manually fixed in local root v5-16-00
1002                                 if (!gGrid->Mkdir(alienDir.Data(),"-p",0))
1003                                 {
1004                                         Log("SHUTTLE", Form("CopyFilesToGrid - Cannot create directory %s",
1005                                                         alienDir.Data()));
1006                                         delete dirList;
1007                                         return kFALSE;
1008                                 } else {
1009                                         Log("SHUTTLE",Form("CopyFilesToGrid - Folder %s created", alienDir.Data()));
1010                                 }
1011
1012                         } else {
1013                                         Log("SHUTTLE",Form("CopyFilesToGrid - Folder %s found", alienDir.Data()));
1014                         }
1015                 }
1016
1017                 TString fullLocalPath;
1018                 fullLocalPath.Form("%s/%s", dir.Data(), fileName.Data());
1019
1020                 TString fullGridPath;
1021                 fullGridPath.Form("alien://%s/%s", alienDir.Data(), fileName.Data());
1022
1023                 Bool_t result = TFile::Cp(fullLocalPath, fullGridPath);
1024
1025                 if (result)
1026                 {
1027                         Log("SHUTTLE", Form("CopyFilesToGrid - Copying local file %s to %s succeeded!",
1028                                                 fullLocalPath.Data(), fullGridPath.Data()));
1029                         RemoveFile(fullLocalPath);
1030                         nTransfer++;
1031                 }
1032                 else
1033                 {
1034                         Log("SHUTTLE", Form("CopyFilesToGrid - Copying local file %s to %s FAILED!",
1035                                                 fullLocalPath.Data(), fullGridPath.Data()));
1036                         success = kFALSE;
1037                 }
1038         }
1039
1040         Log("SHUTTLE", Form("CopyFilesToGrid - %d (over %d) files in folder %s copied to Grid.",
1041                                                 nTransfer, nDirs, dir.Data()));
1042
1043
1044         delete dirList;
1045         return success;
1046 }
1047
1048 //______________________________________________________________________________________________
1049 const char* AliShuttle::GetRefFilePrefix(const char* base, const char* detector)
1050 {
1051         //
1052         // Get folder name of reference files
1053         //
1054
1055         TString offDetStr(GetOfflineDetName(detector));
1056         TString dir;
1057         if (offDetStr == "ITS" || offDetStr == "MUON" || offDetStr == "PHOS")
1058         {
1059                 dir.Form("%s/%s/%s", base, offDetStr.Data(), detector);
1060         } else {
1061                 dir.Form("%s/%s", base, offDetStr.Data());
1062         }
1063
1064         return dir.Data();
1065
1066
1067 }
1068
1069 //______________________________________________________________________________________________
1070 void AliShuttle::CleanLocalStorage(const TString& uri)
1071 {
1072         //
1073         // Called in case the preprocessor is declared failed. Remove remaining objects from the local storages.
1074         //
1075
1076         const char* type = 0;
1077         if(uri == fgkLocalCDB) {
1078                 type = "OCDB";
1079         } else if(uri == fgkLocalRefStorage) {
1080                 type = "Reference";
1081         } else {
1082                 AliError(Form("Invalid storage URI: %s", uri.Data()));
1083                 return;
1084         }
1085
1086         AliCDBManager* man = AliCDBManager::Instance();
1087
1088         // open local storage
1089         AliCDBStorage *localSto = man->GetStorage(uri);
1090         if(!localSto) {
1091                 Log("SHUTTLE",
1092                         Form("CleanLocalStorage - cannot activate local %s storage", type));
1093                 return;
1094         }
1095
1096         TString filename(Form("%s/%s/*/Run*_v%d_s*.root",
1097                 localSto->GetBaseFolder().Data(), GetOfflineDetName(fCurrentDetector.Data()), GetCurrentRun()));
1098
1099         AliDebug(2, Form("filename = %s", filename.Data()));
1100
1101         Log("SHUTTLE", Form("Removing remaining local files for run %d and detector %s ...",
1102                 GetCurrentRun(), fCurrentDetector.Data()));
1103
1104         RemoveFile(filename.Data());
1105
1106 }
1107
1108 //______________________________________________________________________________________________
1109 void AliShuttle::RemoveFile(const char* filename)
1110 {
1111         //
1112         // removes local file
1113         //
1114
1115         TString command(Form("rm -f %s", filename));
1116
1117         Int_t result = gSystem->Exec(command.Data());
1118         if(result != 0)
1119         {
1120                 Log("SHUTTLE", Form("RemoveFile - %s: Cannot remove file %s!",
1121                         fCurrentDetector.Data(), filename));
1122         }
1123 }
1124
1125 //______________________________________________________________________________________________
1126 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
1127 {
1128         //
1129         // Reads the AliShuttleStatus from the CDB
1130         //
1131
1132         if (fStatusEntry){
1133                 delete fStatusEntry;
1134                 fStatusEntry = 0;
1135         }
1136
1137         fStatusEntry = AliCDBManager::Instance()->GetStorage(GetLocalCDB())
1138                 ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), GetCurrentRun());
1139
1140         if (!fStatusEntry) return 0;
1141         fStatusEntry->SetOwner(1);
1142
1143         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
1144         if (!status) {
1145                 AliError("Invalid object stored to CDB!");
1146                 return 0;
1147         }
1148
1149         return status;
1150 }
1151
1152 //______________________________________________________________________________________________
1153 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
1154 {
1155         //
1156         // writes the status for one subdetector
1157         //
1158
1159         if (fStatusEntry){
1160                 delete fStatusEntry;
1161                 fStatusEntry = 0;
1162         }
1163
1164         Int_t run = GetCurrentRun();
1165
1166         AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), run, run);
1167
1168         fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
1169         fStatusEntry->SetOwner(1);
1170
1171         UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
1172
1173         if (!result) {
1174                 Log("SHUTTLE", Form("WriteShuttleStatus - Failed for %s, run %d",
1175                                                 fCurrentDetector.Data(), run));
1176                 return kFALSE;
1177         }
1178
1179         SendMLInfo();
1180
1181         return kTRUE;
1182 }
1183
1184 //______________________________________________________________________________________________
1185 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
1186 {
1187         //
1188         // changes the AliShuttleStatus for the given detector and run to the given status
1189         //
1190
1191         if (!fStatusEntry){
1192                 AliError("UNEXPECTED: fStatusEntry empty");
1193                 return;
1194         }
1195
1196         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
1197
1198         if (!status){
1199                 Log("SHUTTLE", "UpdateShuttleStatus - UNEXPECTED: status could not be read from current CDB entry");
1200                 return;
1201         }
1202
1203         TString actionStr = Form("UpdateShuttleStatus - %s: Changing state from %s to %s",
1204                                 fCurrentDetector.Data(),
1205                                 status->GetStatusName(),
1206                                 status->GetStatusName(newStatus));
1207         Log("SHUTTLE", actionStr);
1208         SetLastAction(actionStr);
1209
1210         status->SetStatus(newStatus);
1211         if (increaseCount) status->IncreaseCount();
1212
1213         AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
1214
1215         SendMLInfo();
1216 }
1217
1218 //______________________________________________________________________________________________
1219 void AliShuttle::SendMLInfo()
1220 {
1221         //
1222         // sends ML information about the current status of the current detector being processed
1223         //
1224
1225         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
1226
1227         if (!status){
1228                 Log("SHUTTLE", "SendMLInfo - UNEXPECTED: status could not be read from current CDB entry");
1229                 return;
1230         }
1231
1232         TMonaLisaText  mlStatus(Form("%s_status", fCurrentDetector.Data()), status->GetStatusName());
1233         TMonaLisaValue mlRetryCount(Form("%s_count", fCurrentDetector.Data()), status->GetCount());
1234
1235         TList mlList;
1236         mlList.Add(&mlStatus);
1237         mlList.Add(&mlRetryCount);
1238
1239         fMonaLisa->SendParameters(&mlList);
1240 }
1241
1242 //______________________________________________________________________________________________
1243 Bool_t AliShuttle::ContinueProcessing()
1244 {
1245         // this function reads the AliShuttleStatus information from CDB and
1246         // checks if the processing should be continued
1247         // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
1248
1249         if (!fConfig->HostProcessDetector(fCurrentDetector)) return kFALSE;
1250
1251         AliPreprocessor* aPreprocessor =
1252                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
1253         if (!aPreprocessor)
1254         {
1255                 Log("SHUTTLE", Form("ContinueProcessing - %s: no preprocessor registered", fCurrentDetector.Data()));
1256                 return kFALSE;
1257         }
1258
1259         AliShuttleLogbookEntry::Status entryStatus =
1260                 fLogbookEntry->GetDetectorStatus(fCurrentDetector);
1261
1262         if(entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
1263                 Log("SHUTTLE", Form("ContinueProcessing - %s is %s",
1264                                 fCurrentDetector.Data(),
1265                                 fLogbookEntry->GetDetectorStatusName(entryStatus)));
1266                 return kFALSE;
1267         }
1268
1269         // if we get here, according to Shuttle logbook subdetector is in UNPROCESSED state
1270
1271         // check if current run is first unprocessed run for current detector
1272         if (fConfig->StrictRunOrder(fCurrentDetector) &&
1273                 !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
1274         {
1275                 if (fTestMode == kNone)
1276                 {
1277                         Log("SHUTTLE", Form("ContinueProcessing - %s requires strict run ordering"
1278                                         " but this is not the first unprocessed run!"));
1279                         return kFALSE;
1280                 }
1281                 else
1282                 {
1283                         Log("SHUTTLE", Form("ContinueProcessing - In TESTMODE - "
1284                                         "Although %s requires strict run ordering "
1285                                         "and this is not the first unprocessed run, "
1286                                         "the SHUTTLE continues"));
1287                 }
1288         }
1289
1290         AliShuttleStatus* status = ReadShuttleStatus();
1291         if (!status) {
1292                 // first time
1293                 Log("SHUTTLE", Form("ContinueProcessing - %s: Processing first time",
1294                                 fCurrentDetector.Data()));
1295                 status = new AliShuttleStatus(AliShuttleStatus::kStarted);
1296                 return WriteShuttleStatus(status);
1297         }
1298
1299         // The following two cases shouldn't happen if Shuttle Logbook was correctly updated.
1300         // If it happens it may mean Logbook updating failed... let's do it now!
1301         if (status->GetStatus() == AliShuttleStatus::kDone ||
1302             status->GetStatus() == AliShuttleStatus::kFailed){
1303                 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s. Updating Shuttle Logbook",
1304                                         fCurrentDetector.Data(),
1305                                         status->GetStatusName(status->GetStatus())));
1306                 UpdateShuttleLogbook(fCurrentDetector.Data(),
1307                                         status->GetStatusName(status->GetStatus()));
1308                 return kFALSE;
1309         }
1310
1311         if (status->GetStatus() == AliShuttleStatus::kStoreError) {
1312                 Log("SHUTTLE",
1313                         Form("ContinueProcessing - %s: Grid storage of one or more "
1314                                 "objects failed. Trying again now",
1315                                 fCurrentDetector.Data()));
1316                 UpdateShuttleStatus(AliShuttleStatus::kStoreStarted);
1317                 if (StoreOCDB()){
1318                         Log("SHUTTLE", Form("ContinueProcessing - %s: all objects "
1319                                 "successfully stored into main storage",
1320                                 fCurrentDetector.Data()));
1321                         UpdateShuttleStatus(AliShuttleStatus::kDone);
1322                         UpdateShuttleLogbook(fCurrentDetector.Data(), "DONE");
1323                 } else {
1324                         Log("SHUTTLE",
1325                                 Form("ContinueProcessing - %s: Grid storage failed again",
1326                                         fCurrentDetector.Data()));
1327                         UpdateShuttleStatus(AliShuttleStatus::kStoreError);
1328                 }
1329                 return kFALSE;
1330         }
1331
1332         // if we get here, there is a restart
1333         Bool_t cont = kFALSE;
1334
1335         // abort conditions
1336         if (status->GetCount() >= fConfig->GetMaxRetries()) {
1337                 Log("SHUTTLE", Form("ContinueProcessing - %s failed %d times in status %s - "
1338                                 "Updating Shuttle Logbook", fCurrentDetector.Data(),
1339                                 status->GetCount(), status->GetStatusName()));
1340                 UpdateShuttleLogbook(fCurrentDetector.Data(), "FAILED");
1341                 UpdateShuttleStatus(AliShuttleStatus::kFailed);
1342
1343                 // there may still be objects in local OCDB and reference storage
1344                 // and FXS databases may be not updated: do it now!
1345
1346                 // TODO Currently disabled, we want to keep files in case of failure!
1347                 // CleanLocalStorage(fgkLocalCDB);
1348                 // CleanLocalStorage(fgkLocalRefStorage);
1349                 // UpdateTableFailCase();
1350
1351                 // Send mail to detector expert!
1352                 Log("SHUTTLE", Form("ContinueProcessing - Sending mail to %s expert...",
1353                                         fCurrentDetector.Data()));
1354                 if (!SendMail())
1355                         Log("SHUTTLE", Form("ContinueProcessing - Could not send mail to %s expert",
1356                                         fCurrentDetector.Data()));
1357
1358         } else {
1359                 Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. "
1360                                 "Aborted before with %s. Retry number %d.", fCurrentDetector.Data(),
1361                                 status->GetStatusName(), status->GetCount()));
1362                 Bool_t increaseCount = kTRUE;
1363                 if (status->GetStatus() == AliShuttleStatus::kDCSError ||
1364                         status->GetStatus() == AliShuttleStatus::kDCSStarted)
1365                                 increaseCount = kFALSE;
1366
1367                 UpdateShuttleStatus(AliShuttleStatus::kStarted, increaseCount);
1368                 cont = kTRUE;
1369         }
1370
1371         return cont;
1372 }
1373
1374 //______________________________________________________________________________________________
1375 Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
1376 {
1377         //
1378         // Makes data retrieval for all detectors in the configuration.
1379         // entry: Shuttle logbook entry, contains run paramenters and status of detectors
1380         // (Unprocessed, Inactive, Failed or Done).
1381         // Returns kFALSE in case of error occured and kTRUE otherwise
1382         //
1383
1384         if (!entry) return kFALSE;
1385
1386         fLogbookEntry = entry;
1387
1388         Log("SHUTTLE", Form("\t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^*",
1389                                         GetCurrentRun()));
1390
1391         // create ML instance that monitors this run
1392         fMonaLisa = new TMonaLisaWriter(Form("%d", GetCurrentRun()), "SHUTTLE", "aliendb1.cern.ch");
1393         // disable monitoring of other parameters that come e.g. from TFile
1394         gMonitoringWriter = 0;
1395
1396         // Send the information to ML
1397         TMonaLisaText  mlStatus("SHUTTLE_status", "Processing");
1398         TMonaLisaText  mlRunType("SHUTTLE_runtype", Form("%s (%s)", entry->GetRunType(), entry->GetRunParameter("log")));
1399
1400         TList mlList;
1401         mlList.Add(&mlStatus);
1402         mlList.Add(&mlRunType);
1403
1404         fMonaLisa->SendParameters(&mlList);
1405
1406         if (fLogbookEntry->IsDone())
1407         {
1408                 Log("SHUTTLE","Process - Shuttle is already DONE. Updating logbook");
1409                 UpdateShuttleLogbook("shuttle_done");
1410                 fLogbookEntry = 0;
1411                 return kTRUE;
1412         }
1413
1414         // read test mode if flag is set
1415         if (fReadTestMode)
1416         {
1417                 fTestMode = kNone;
1418                 TString logEntry(entry->GetRunParameter("log"));
1419                 //printf("log entry = %s\n", logEntry.Data());
1420                 TString searchStr("Testmode: ");
1421                 Int_t pos = logEntry.Index(searchStr.Data());
1422                 //printf("%d\n", pos);
1423                 if (pos >= 0)
1424                 {
1425                         TSubString subStr = logEntry(pos + searchStr.Length(), logEntry.Length());
1426                         //printf("%s\n", subStr.String().Data());
1427                         TString newStr(subStr.Data());
1428                         TObjArray* token = newStr.Tokenize(' ');
1429                         if (token)
1430                         {
1431                                 //token->Print();
1432                                 TObjString* tmpStr = dynamic_cast<TObjString*> (token->First());
1433                                 if (tmpStr)
1434                                 {
1435                                         Int_t testMode = tmpStr->String().Atoi();
1436                                         if (testMode > 0)
1437                                         {
1438                                                 Log("SHUTTLE", Form("Process - Enabling test mode %d", testMode));
1439                                                 SetTestMode((TestMode) testMode);
1440                                         }
1441                                 }
1442                                 delete token;
1443                         }
1444                 }
1445         }
1446
1447         fLogbookEntry->Print("all");
1448
1449         // Initialization
1450         Bool_t hasError = kFALSE;
1451
1452         // Set the CDB and Reference folders according to the year and LHC period
1453         TString lhcPeriod(GetLHCPeriod());
1454         if (lhcPeriod.Length() == 0)
1455         {
1456                 Log("SHUTTLE","StoreRunMetaDataFile - LHCPeriod not found in logbook!");
1457                 return 0;
1458         }
1459
1460         if (fgkMainCDB.Length() == 0)
1461                 fgkMainCDB = Form("alien://folder=/alice/data/%d/%s/OCDB?user=alidaq?cacheFold=/tmp/OCDBCache",
1462                                         GetCurrentYear(), lhcPeriod.Data());
1463
1464         if (fgkMainRefStorage.Length() == 0)
1465                 fgkMainRefStorage = Form("alien://folder=/alice/data/%d/%s/Reference?user=alidaq?cacheFold=/tmp/OCDBCache",
1466                                         GetCurrentYear(), lhcPeriod.Data());
1467
1468         AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
1469         if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
1470         AliCDBStorage *mainRefSto = AliCDBManager::Instance()->GetStorage(fgkMainRefStorage);
1471         if(mainRefSto) mainRefSto->QueryCDB(GetCurrentRun());
1472
1473         // Loop on detectors in the configuration
1474         TIter iter(fConfig->GetDetectors());
1475         TObjString* aDetector = 0;
1476
1477         while ((aDetector = (TObjString*) iter.Next()))
1478         {
1479                 fCurrentDetector = aDetector->String();
1480
1481                 if (ContinueProcessing() == kFALSE) continue;
1482
1483                 Log("SHUTTLE", Form("\t\t\t****** run %d - %s: START  ******",
1484                                                 GetCurrentRun(), aDetector->GetName()));
1485
1486                 for(Int_t iSys=0;iSys<3;iSys++) fFXSCalled[iSys]=kFALSE;
1487
1488                 Log(fCurrentDetector.Data(), "Process - Starting processing");
1489
1490                 Int_t pid = fork();
1491
1492                 if (pid < 0)
1493                 {
1494                         Log("SHUTTLE", "Process - ERROR: Forking failed");
1495                 }
1496                 else if (pid > 0)
1497                 {
1498                         // parent
1499                         Log("SHUTTLE", Form("Process - In parent process of %d - %s: Starting monitoring",
1500                                                         GetCurrentRun(), aDetector->GetName()));
1501
1502                         Long_t begin = time(0);
1503
1504                         int status; // to be used with waitpid, on purpose an int (not Int_t)!
1505                         while (waitpid(pid, &status, WNOHANG) == 0)
1506                         {
1507                                 Long_t expiredTime = time(0) - begin;
1508
1509                                 if (expiredTime > fConfig->GetPPTimeOut())
1510                                 {
1511                                         TString tmp;
1512                                         tmp.Form("Process - Process of %s time out. "
1513                                                         "Run time: %d seconds. Killing...",
1514                                                         fCurrentDetector.Data(), expiredTime);
1515                                         Log("SHUTTLE", tmp);
1516                                         Log(fCurrentDetector, tmp);
1517
1518                                         kill(pid, 9);
1519
1520                                         UpdateShuttleStatus(AliShuttleStatus::kPPTimeOut);
1521                                         hasError = kTRUE;
1522
1523                                         gSystem->Sleep(1000);
1524                                 }
1525                                 else
1526                                 {
1527                                         gSystem->Sleep(1000);
1528
1529                                         TString checkStr;
1530                                         checkStr.Form("ps -o vsize --pid %d | tail -n 1", pid);
1531                                         FILE* pipe = gSystem->OpenPipe(checkStr, "r");
1532                                         if (!pipe)
1533                                         {
1534                                                 Log("SHUTTLE", Form("Process - Error: "
1535                                                         "Could not open pipe to %s", checkStr.Data()));
1536                                                 continue;
1537                                         }
1538
1539                                         char buffer[100];
1540                                         if (!fgets(buffer, 100, pipe))
1541                                         {
1542                                                 Log("SHUTTLE", "Process - Error: ps did not return anything");
1543                                                 gSystem->ClosePipe(pipe);
1544                                                 continue;
1545                                         }
1546                                         gSystem->ClosePipe(pipe);
1547
1548                                         //Log("SHUTTLE", Form("ps returned %s", buffer));
1549
1550                                         Int_t mem = 0;
1551                                         if ((sscanf(buffer, "%d\n", &mem) != 1) || !mem)
1552                                         {
1553                                                 Log("SHUTTLE", "Process - Error: Could not parse output of ps");
1554                                                 continue;
1555                                         }
1556
1557                                         if (expiredTime % 60 == 0)
1558                                                 Log("SHUTTLE", Form("Process - %s: Checking process. "
1559                                                         "Run time: %d seconds - Memory consumption: %d KB",
1560                                                         fCurrentDetector.Data(), expiredTime, mem));
1561
1562                                         if (mem > fConfig->GetPPMaxMem())
1563                                         {
1564                                                 TString tmp;
1565                                                 tmp.Form("Process - Process exceeds maximum allowed memory "
1566                                                         "(%d KB > %d KB). Killing...",
1567                                                         mem, fConfig->GetPPMaxMem());
1568                                                 Log("SHUTTLE", tmp);
1569                                                 Log(fCurrentDetector, tmp);
1570
1571                                                 kill(pid, 9);
1572
1573                                                 UpdateShuttleStatus(AliShuttleStatus::kPPOutOfMemory);
1574                                                 hasError = kTRUE;
1575
1576                                                 gSystem->Sleep(1000);
1577                                         }
1578                                 }
1579                         }
1580
1581                         Log("SHUTTLE", Form("Process - In parent process of %d - %s: Client has terminated.",
1582                                                                 GetCurrentRun(), aDetector->GetName()));
1583
1584                         if (WIFEXITED(status))
1585                         {
1586                                 Int_t returnCode = WEXITSTATUS(status);
1587
1588                                 Log("SHUTTLE", Form("Process - %s: the return code is %d", fCurrentDetector.Data(),
1589                                                                                 returnCode));
1590
1591                                 if (returnCode == 0) hasError = kTRUE;
1592                         }
1593                 }
1594                 else if (pid == 0)
1595                 {
1596                         // client
1597                         Log("SHUTTLE", Form("Process - In client process of %d - %s", GetCurrentRun(),
1598                                 aDetector->GetName()));
1599
1600                         Log("SHUTTLE", Form("Process - Redirecting output to %s log",fCurrentDetector.Data()));
1601
1602                         if ((freopen(GetLogFileName(fCurrentDetector), "a", stdout)) == 0)
1603                         {
1604                                 Log("SHUTTLE", "Process - Could not freopen stdout");
1605                         }
1606                         else
1607                         {
1608                                 fOutputRedirected = kTRUE;
1609                                 if ((dup2(fileno(stdout), fileno(stderr))) < 0)
1610                                         Log("SHUTTLE", "Process - Could not redirect stderr");
1611
1612                         }
1613
1614                         TString wd = gSystem->WorkingDirectory();
1615                         TString tmpDir = Form("%s/%s_%d_process", GetShuttleTempDir(),
1616                                 fCurrentDetector.Data(), GetCurrentRun());
1617
1618                         Int_t result = gSystem->GetPathInfo(tmpDir.Data(), 0, (Long64_t*) 0, 0, 0);
1619                         if (!result) // temp dir already exists!
1620                         {
1621                                 Log(fCurrentDetector.Data(),
1622                                         Form("Process - %s dir already exists! Removing...", tmpDir.Data()));
1623                                 gSystem->Exec(Form("rm -rf %s",tmpDir.Data()));
1624                         }
1625
1626                         if (gSystem->mkdir(tmpDir.Data(), 1))
1627                         {
1628                                 Log(fCurrentDetector.Data(), "Process - could not make temp directory!!");
1629                                 gSystem->Exit(1);
1630                         }
1631
1632                         if (!gSystem->ChangeDirectory(tmpDir.Data()))
1633                         {
1634                                 Log(fCurrentDetector.Data(), "Process - could not change directory!!");
1635                                 gSystem->Exit(1);
1636                         }
1637
1638                         Bool_t success = ProcessCurrentDetector();
1639
1640                         gSystem->ChangeDirectory(wd.Data());
1641
1642                         if (success) // Preprocessor finished successfully!
1643                         {
1644                                 // remove temporary folder
1645                                 gSystem->Exec(Form("rm -rf %s",tmpDir.Data()));
1646
1647                                 // Update time_processed field in FXS DB
1648                                 if (UpdateTable() == kFALSE)
1649                                         Log("SHUTTLE", Form("Process - %s: Could not update FXS databases!",
1650                                                         fCurrentDetector.Data()));
1651
1652                                 // Transfer the data from local storage to main storage (Grid)
1653                                 UpdateShuttleStatus(AliShuttleStatus::kStoreStarted);
1654                                 if (StoreOCDB() == kFALSE)
1655                                 {
1656                                         Log("SHUTTLE",
1657                                                 Form("\t\t\t****** run %d - %s: STORAGE ERROR ******",
1658                                                         GetCurrentRun(), aDetector->GetName()));
1659                                         UpdateShuttleStatus(AliShuttleStatus::kStoreError);
1660                                         success = kFALSE;
1661                                 } else {
1662                                         Log("SHUTTLE",
1663                                                 Form("\t\t\t****** run %d - %s: DONE ******",
1664                                                         GetCurrentRun(), aDetector->GetName()));
1665                                         UpdateShuttleStatus(AliShuttleStatus::kDone);
1666                                         UpdateShuttleLogbook(fCurrentDetector, "DONE");
1667                                 }
1668                         } else
1669                         {
1670                                 Log("SHUTTLE",
1671                                         Form("\t\t\t****** run %d - %s: PP ERROR ******",
1672                                                 GetCurrentRun(), aDetector->GetName()));
1673                         }
1674
1675                         for (UInt_t iSys=0; iSys<3; iSys++)
1676                         {
1677                                 if (fFXSCalled[iSys]) fFXSlist[iSys].Clear();
1678                         }
1679
1680                         Log("SHUTTLE", Form("Process - Client process of %d - %s is exiting now with %d.",
1681                                                         GetCurrentRun(), aDetector->GetName(), success));
1682
1683                         // the client exits here
1684                         gSystem->Exit(success);
1685
1686                         AliError("We should never get here!!!");
1687                 }
1688         }
1689
1690         Log("SHUTTLE", Form("\t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^*",
1691                                                         GetCurrentRun()));
1692
1693         //check if shuttle is done for this run, if so update logbook
1694         TObjArray checkEntryArray;
1695         checkEntryArray.SetOwner(1);
1696         TString whereClause = Form("where run=%d", GetCurrentRun());
1697         if (!QueryShuttleLogbook(whereClause.Data(), checkEntryArray) || checkEntryArray.GetEntries() == 0) {
1698                 Log("SHUTTLE", Form("Process - Warning: Cannot check status of run %d on Shuttle logbook!",
1699                                                 GetCurrentRun()));
1700                 return hasError == kFALSE;
1701         }
1702
1703         AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
1704                                                 (checkEntryArray.At(0));
1705
1706         if (checkEntry)
1707         {
1708                 if (checkEntry->IsDone())
1709                 {
1710                         Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
1711                         UpdateShuttleLogbook("shuttle_done");
1712                 }
1713                 else
1714                 {
1715                         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
1716                         {
1717                                 if (checkEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
1718                                 {
1719                                         AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
1720                                                         checkEntry->GetRun(), GetDetName(iDet)));
1721                                         fFirstUnprocessed[iDet] = kFALSE;
1722                                 }
1723                         }
1724                 }
1725         }
1726
1727         // remove ML instance
1728         delete fMonaLisa;
1729         fMonaLisa = 0;
1730
1731         fLogbookEntry = 0;
1732
1733         return hasError == kFALSE;
1734 }
1735
1736 //______________________________________________________________________________________________
1737 Bool_t AliShuttle::ProcessCurrentDetector()
1738 {
1739         //
1740         // Makes data retrieval just for a specific detector (fCurrentDetector).
1741         // Threre should be a configuration for this detector.
1742
1743         Log("SHUTTLE", Form("ProcessCurrentDetector - Retrieving values for %s, run %d",
1744                                                 fCurrentDetector.Data(), GetCurrentRun()));
1745
1746         TString wd = gSystem->WorkingDirectory();
1747
1748         if (!CleanReferenceStorage(fCurrentDetector.Data()))
1749                 return kFALSE;
1750
1751         gSystem->ChangeDirectory(wd.Data());
1752
1753         TMap* dcsMap = new TMap();
1754
1755         // call preprocessor
1756         AliPreprocessor* aPreprocessor =
1757                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
1758
1759         aPreprocessor->Initialize(GetCurrentRun(), GetCurrentStartTime(), GetCurrentEndTime());
1760
1761         Bool_t processDCS = aPreprocessor->ProcessDCS();
1762
1763         if (!processDCS)
1764         {
1765                 Log(fCurrentDetector, "ProcessCurrentDetector -"
1766                         " The preprocessor requested to skip the retrieval of DCS values");
1767         }
1768         else if (fTestMode & kSkipDCS)
1769         {
1770                 Log(fCurrentDetector, "ProcessCurrentDetector - In TESTMODE: Skipping DCS processing");
1771         }
1772         else if (fTestMode & kErrorDCS)
1773         {
1774                 Log(fCurrentDetector, "ProcessCurrentDetector - In TESTMODE: Simulating DCS error");
1775                 UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
1776                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
1777                 delete dcsMap;
1778                 return kFALSE;
1779         } else {
1780
1781                 UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
1782
1783                 // Query DCS archive
1784                 Int_t nServers = fConfig->GetNServers(fCurrentDetector);
1785
1786                 for (int iServ=0; iServ<nServers; iServ++)
1787                 {
1788
1789                         TString host(fConfig->GetDCSHost(fCurrentDetector, iServ));
1790                         Int_t port = fConfig->GetDCSPort(fCurrentDetector, iServ);
1791                         Int_t multiSplit = fConfig->GetMultiSplit(fCurrentDetector, iServ);
1792
1793                         Log(fCurrentDetector, Form("ProcessCurrentDetector -"
1794                                         " Querying DCS Amanda server %s:%d (%d of %d)",
1795                                         host.Data(), port, iServ+1, nServers));
1796
1797                         TMap* aliasMap = 0;
1798                         TMap* dpMap = 0;
1799
1800                         if (fConfig->GetDCSAliases(fCurrentDetector, iServ)->GetEntries() > 0)
1801                         {
1802                                 aliasMap = GetValueSet(host, port,
1803                                                 fConfig->GetDCSAliases(fCurrentDetector, iServ),
1804                                                 kAlias, multiSplit);
1805                                 if (!aliasMap)
1806                                 {
1807                                         Log(fCurrentDetector,
1808                                                 Form("ProcessCurrentDetector -"
1809                                                         " Error retrieving DCS aliases from server %s."
1810                                                         " Sending mail to DCS experts!", host.Data()));
1811                                         UpdateShuttleStatus(AliShuttleStatus::kDCSError);
1812
1813                                         if (!SendMailToDCS())
1814                                                 Log("SHUTTLE", Form("ProcessCurrentDetector - Could not send mail to DCS experts!"));
1815
1816                                         delete dcsMap;
1817                                         return kFALSE;
1818                                 }
1819                         }
1820
1821                         if (fConfig->GetDCSDataPoints(fCurrentDetector, iServ)->GetEntries() > 0)
1822                         {
1823                                 dpMap = GetValueSet(host, port,
1824                                                 fConfig->GetDCSDataPoints(fCurrentDetector, iServ),
1825                                                 kDP, multiSplit);
1826                                 if (!dpMap)
1827                                 {
1828                                         Log(fCurrentDetector,
1829                                                 Form("ProcessCurrentDetector -"
1830                                                         " Error retrieving DCS data points from server %s."
1831                                                         " Sending mail to DCS experts!", host.Data()));
1832                                         UpdateShuttleStatus(AliShuttleStatus::kDCSError);
1833
1834                                         if (!SendMailToDCS())
1835                                                 Log("SHUTTLE", Form("ProcessCurrentDetector - Could not send mail to DCS experts!"));
1836
1837                                         if (aliasMap) delete aliasMap;
1838                                         delete dcsMap;
1839                                         return kFALSE;
1840                                 }
1841                         }
1842
1843                         // merge aliasMap and dpMap into dcsMap
1844                         if(aliasMap) {
1845                                 TIter iter(aliasMap);
1846                                 TObjString* key = 0;
1847                                 while ((key = (TObjString*) iter.Next()))
1848                                         dcsMap->Add(key, aliasMap->GetValue(key->String()));
1849
1850                                 aliasMap->SetOwner(kFALSE);
1851                                 delete aliasMap;
1852                         }
1853
1854                         if(dpMap) {
1855                                 TIter iter(dpMap);
1856                                 TObjString* key = 0;
1857                                 while ((key = (TObjString*) iter.Next()))
1858                                         dcsMap->Add(key, dpMap->GetValue(key->String()));
1859
1860                                 dpMap->SetOwner(kFALSE);
1861                                 delete dpMap;
1862                         }
1863                 }
1864         }
1865
1866         // save map into file, to help debugging in case of preprocessor error
1867         TFile* f = TFile::Open("DCSMap.root","recreate");
1868         f->cd();
1869         dcsMap->Write("DCSMap", TObject::kSingleKey);
1870         f->Close();
1871         delete f;
1872
1873         // DCS Archive DB processing successful. Call Preprocessor!
1874         UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
1875
1876         UInt_t returnValue = aPreprocessor->Process(dcsMap);
1877
1878         if (returnValue > 0) // Preprocessor error!
1879         {
1880                 Log(fCurrentDetector, Form("ProcessCurrentDetector - "
1881                                 "Preprocessor failed. Process returned %d.", returnValue));
1882                 UpdateShuttleStatus(AliShuttleStatus::kPPError);
1883                 dcsMap->DeleteAll();
1884                 delete dcsMap;
1885                 return kFALSE;
1886         }
1887
1888         // preprocessor ok!
1889         UpdateShuttleStatus(AliShuttleStatus::kPPDone);
1890         Log(fCurrentDetector, Form("ProcessCurrentDetector - %s preprocessor returned success",
1891                                 fCurrentDetector.Data()));
1892
1893         dcsMap->DeleteAll();
1894         delete dcsMap;
1895
1896         return kTRUE;
1897 }
1898
1899 //______________________________________________________________________________________________
1900 Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
1901                 TObjArray& entries)
1902 {
1903         // Query DAQ's Shuttle logbook and fills detector status object.
1904         // Call QueryRunParameters to query DAQ logbook for run parameters.
1905         //
1906
1907         entries.SetOwner(1);
1908
1909         // check connection, in case connect
1910         if(!Connect(3)) return kFALSE;
1911
1912         TString sqlQuery;
1913         sqlQuery = Form("select * from %s %s order by run", fConfig->GetShuttlelbTable(), whereClause);
1914
1915         TSQLResult* aResult = fServer[3]->Query(sqlQuery);
1916         if (!aResult) {
1917                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
1918                 return kFALSE;
1919         }
1920
1921         AliDebug(2,Form("Query = %s", sqlQuery.Data()));
1922
1923         if(aResult->GetRowCount() == 0) {
1924                 Log("SHUTTLE", "No entries in Shuttle Logbook match request");
1925                 delete aResult;
1926                 return kTRUE;
1927         }
1928
1929         // TODO Check field count!
1930         const UInt_t nCols = 23;
1931         if (aResult->GetFieldCount() != (Int_t) nCols) {
1932                 Log("SHUTTLE", "Invalid SQL result field number!");
1933                 delete aResult;
1934                 return kFALSE;
1935         }
1936
1937         TSQLRow* aRow;
1938         while ((aRow = aResult->Next())) {
1939                 TString runString(aRow->GetField(0), aRow->GetFieldLength(0));
1940                 Int_t run = runString.Atoi();
1941
1942                 AliShuttleLogbookEntry *entry = QueryRunParameters(run);
1943                 if (!entry)
1944                         continue;
1945
1946                 // loop on detectors
1947                 for(UInt_t ii = 0; ii < nCols; ii++)
1948                         entry->SetDetectorStatus(aResult->GetFieldName(ii), aRow->GetField(ii));
1949
1950                 entries.AddLast(entry);
1951                 delete aRow;
1952         }
1953
1954         delete aResult;
1955         return kTRUE;
1956 }
1957
1958 //______________________________________________________________________________________________
1959 AliShuttleLogbookEntry* AliShuttle::QueryRunParameters(Int_t run)
1960 {
1961         //
1962         // Retrieve run parameters written in the DAQ logbook and sets them into AliShuttleLogbookEntry object
1963         //
1964
1965         // check connection, in case connect
1966         if (!Connect(3))
1967                 return 0;
1968
1969         TString sqlQuery;
1970         sqlQuery.Form("select * from %s where run=%d", fConfig->GetDAQlbTable(), run);
1971
1972         TSQLResult* aResult = fServer[3]->Query(sqlQuery);
1973         if (!aResult) {
1974                 Log("SHUTTLE", Form("Can't execute query <%s>!", sqlQuery.Data()));
1975                 return 0;
1976         }
1977
1978         if (aResult->GetRowCount() == 0) {
1979                 Log("SHUTTLE", Form("QueryRunParameters - No entry in DAQ Logbook for run %d. Skipping", run));
1980                 delete aResult;
1981                 return 0;
1982         }
1983
1984         if (aResult->GetRowCount() > 1) {
1985                 Log("SHUTTLE", Form("QueryRunParameters - UNEXPECTED: "
1986                                 "more than one entry in DAQ Logbook for run %d!", run));
1987                 delete aResult;
1988                 return 0;
1989         }
1990
1991         TSQLRow* aRow = aResult->Next();
1992         if (!aRow)
1993         {
1994                 Log("SHUTTLE", Form("QueryRunParameters - Could not retrieve row for run %d. Skipping", run));
1995                 delete aResult;
1996                 return 0;
1997         }
1998
1999         AliShuttleLogbookEntry* entry = new AliShuttleLogbookEntry(run);
2000
2001         for (Int_t ii = 0; ii < aResult->GetFieldCount(); ii++)
2002                 entry->SetRunParameter(aResult->GetFieldName(ii), aRow->GetField(ii));
2003
2004         UInt_t startTime = entry->GetStartTime();
2005         UInt_t endTime = entry->GetEndTime();
2006
2007         if (!startTime || !endTime || startTime > endTime) {
2008                 Log("SHUTTLE",
2009                         Form("QueryRunParameters - Invalid parameters for Run %d: startTime = %d, endTime = %d",
2010                                 run, startTime, endTime));
2011                 delete entry;
2012                 delete aRow;
2013                 delete aResult;
2014                 return 0;
2015         }
2016
2017         delete aRow;
2018         delete aResult;
2019
2020         return entry;
2021 }
2022
2023 //______________________________________________________________________________________________
2024 TMap* AliShuttle::GetValueSet(const char* host, Int_t port, const TSeqCollection* entries,
2025                               DCSType type, Int_t multiSplit)
2026 {
2027         // Retrieve all "entry" data points from the DCS server
2028         // host, port: TSocket connection parameters
2029         // entries: list of name of the alias or data point
2030         // type: kAlias or kDP
2031         // returns TMap of values, 0 when failure
2032
2033         AliDCSClient client(host, port, fTimeout, fRetries, multiSplit);
2034
2035         TMap* result = 0;
2036         if (type == kAlias)
2037         {
2038                 result = client.GetAliasValues(entries, GetCurrentStartTime(),
2039                         GetCurrentEndTime());
2040         }
2041         else if (type == kDP)
2042         {
2043                 result = client.GetDPValues(entries, GetCurrentStartTime(),
2044                         GetCurrentEndTime());
2045         }
2046
2047         if (result == 0)
2048         {
2049                 Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get entries! Reason: %s",
2050                         client.GetErrorString(client.GetResultErrorCode())));
2051                 if (client.GetResultErrorCode() == AliDCSClient::fgkServerError)
2052                         Log(fCurrentDetector.Data(), Form("GetValueSet - Server error code: %s",
2053                                 client.GetServerError().Data()));
2054
2055                 return 0;
2056         }
2057
2058         return result;
2059 }
2060
2061 //______________________________________________________________________________________________
2062 const char* AliShuttle::GetFile(Int_t system, const char* detector,
2063                 const char* id, const char* source)
2064 {
2065         // Get calibration file from file exchange servers
2066         // First queris the FXS database for the file name, using the run, detector, id and source info
2067         // then calls RetrieveFile(filename) for actual copy to local disk
2068         // run: current run being processed (given by Logbook entry fLogbookEntry)
2069         // detector: the Preprocessor name
2070         // id: provided as a parameter by the Preprocessor
2071         // source: provided by the Preprocessor through GetFileSources function
2072
2073         // check if test mode should simulate a FXS error
2074         if (fTestMode & kErrorFXSFiles)
2075         {
2076                 Log(detector, Form("GetFile - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
2077                 return 0;
2078         }
2079
2080         // check connection, in case connect
2081         if (!Connect(system))
2082         {
2083                 Log(detector, Form("GetFile - Couldn't connect to %s FXS database", GetSystemName(system)));
2084                 return 0;
2085         }
2086
2087         // Query preparation
2088         TString sourceName(source);
2089         Int_t nFields = 3;
2090         TString sqlQueryStart = Form("select filePath,size,fileChecksum from %s where",
2091                                                                 fConfig->GetFXSdbTable(system));
2092         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
2093                                                                 GetCurrentRun(), detector, id);
2094
2095         if (system == kDAQ)
2096         {
2097                 whereClause += Form(" and DAQsource=\"%s\"", source);
2098         }
2099         else if (system == kDCS)
2100         {
2101                 sourceName="none";
2102         }
2103         else if (system == kHLT)
2104         {
2105                 whereClause += Form(" and DDLnumbers=\"%s\"", source);
2106                 nFields = 3;
2107         }
2108
2109         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
2110
2111         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2112
2113         // Query execution
2114         TSQLResult* aResult = 0;
2115         aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
2116         if (!aResult) {
2117                 Log(detector, Form("GetFileName - Can't execute SQL query to %s database for: id = %s, source = %s",
2118                                 GetSystemName(system), id, sourceName.Data()));
2119                 return 0;
2120         }
2121
2122         if(aResult->GetRowCount() == 0)
2123         {
2124                 Log(detector,
2125                         Form("GetFileName - No entry in %s FXS db for: id = %s, source = %s",
2126                                 GetSystemName(system), id, sourceName.Data()));
2127                 delete aResult;
2128                 return 0;
2129         }
2130
2131         if (aResult->GetRowCount() > 1) {
2132                 Log(detector,
2133                         Form("GetFileName - More than one entry in %s FXS db for: id = %s, source = %s",
2134                                 GetSystemName(system), id, sourceName.Data()));
2135                 delete aResult;
2136                 return 0;
2137         }
2138
2139         if (aResult->GetFieldCount() != nFields) {
2140                 Log(detector,
2141                         Form("GetFileName - Wrong field count in %s FXS db for: id = %s, source = %s",
2142                                 GetSystemName(system), id, sourceName.Data()));
2143                 delete aResult;
2144                 return 0;
2145         }
2146
2147         TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
2148
2149         if (!aRow){
2150                 Log(detector, Form("GetFileName - Empty set result in %s FXS db from query: id = %s, source = %s",
2151                                 GetSystemName(system), id, sourceName.Data()));
2152                 delete aResult;
2153                 return 0;
2154         }
2155
2156         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
2157         TString fileSize(aRow->GetField(1), aRow->GetFieldLength(1));
2158         TString fileChecksum(aRow->GetField(2), aRow->GetFieldLength(2));
2159
2160         delete aResult;
2161         delete aRow;
2162
2163         AliDebug(2, Form("filePath = %s; size = %s, fileChecksum = %s",
2164                                 filePath.Data(), fileSize.Data(), fileChecksum.Data()));
2165
2166         // retrieved file is renamed to make it unique
2167         TString localFileName = Form("%s/%s_%d_process/%s_%s_%d_%s_%s.shuttle",
2168                                         GetShuttleTempDir(), detector, GetCurrentRun(),
2169                                         GetSystemName(system), detector, GetCurrentRun(),
2170                                         id, sourceName.Data());
2171
2172
2173         // file retrieval from FXS
2174         UInt_t nRetries = 0;
2175         UInt_t maxRetries = 3;
2176         Bool_t result = kFALSE;
2177
2178         // copy!! if successful TSystem::Exec returns 0
2179         while(nRetries++ < maxRetries) {
2180                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
2181                 result = RetrieveFile(system, filePath.Data(), localFileName.Data());
2182                 if(!result)
2183                 {
2184                         Log(detector, Form("GetFileName - Copy of file %s from %s FXS failed",
2185                                         filePath.Data(), GetSystemName(system)));
2186                         continue;
2187                 }
2188
2189                 if (fileChecksum.Length()>0)
2190                 {
2191                         // compare md5sum of local file with the one stored in the FXS DB
2192                         Int_t md5Comp = gSystem->Exec(Form("md5sum %s |grep %s 2>&1 > /dev/null",
2193                                                 localFileName.Data(), fileChecksum.Data()));
2194
2195                         if (md5Comp != 0)
2196                         {
2197                                 Log(detector, Form("GetFileName - md5sum of file %s does not match with local copy!",
2198                                                         filePath.Data()));
2199                                 result = kFALSE;
2200                                 continue;
2201                         }
2202                 } else {
2203                         Log(fCurrentDetector, Form("GetFile - md5sum of file %s not set in %s database, skipping comparison",
2204                                                         filePath.Data(), GetSystemName(system)));
2205                 }
2206                 if (result) break;
2207         }
2208
2209         if(!result) return 0;
2210
2211         fFXSCalled[system]=kTRUE;
2212         TObjString *fileParams = new TObjString(Form("%s#!?!#%s", id, sourceName.Data()));
2213         fFXSlist[system].Add(fileParams);
2214
2215         static TString staticLocalFileName;
2216         staticLocalFileName.Form("%s", localFileName.Data());
2217
2218         Log(fCurrentDetector, Form("GetFile - Retrieved file with id %s and "
2219                         "source %s from %s to %s", id, source,
2220                         GetSystemName(system), localFileName.Data()));
2221
2222         return staticLocalFileName.Data();
2223 }
2224
2225 //______________________________________________________________________________________________
2226 Bool_t AliShuttle::RetrieveFile(UInt_t system, const char* fxsFileName, const char* localFileName)
2227 {
2228         //
2229         // Copies file from FXS to local Shuttle machine
2230         //
2231
2232         // check temp directory: trying to cd to temp; if it does not exist, create it
2233         AliDebug(2, Form("Copy file %s from %s FXS into %s",
2234                         GetSystemName(system), fxsFileName, localFileName));
2235
2236         TString tmpDir(localFileName);
2237
2238         tmpDir = tmpDir(0,tmpDir.Last('/'));
2239
2240         Int_t noDir = gSystem->GetPathInfo(tmpDir.Data(), 0, (Long64_t*) 0, 0, 0);
2241         if (noDir) // temp dir does not exists!
2242         {
2243                 if (gSystem->mkdir(tmpDir.Data(), 1))
2244                 {
2245                         Log(fCurrentDetector.Data(), "RetrieveFile - could not make temp directory!!");
2246                         return kFALSE;
2247                 }
2248         }
2249
2250         TString baseFXSFolder;
2251         if (system == kDAQ)
2252         {
2253                 baseFXSFolder = "FES/";
2254         }
2255         else if (system == kDCS)
2256         {
2257                 baseFXSFolder = "";
2258         }
2259         else if (system == kHLT)
2260         {
2261                 baseFXSFolder = "/opt/FXS/";
2262         }
2263
2264
2265         TString command = Form("scp -oPort=%d -2 %s@%s:%s%s %s",
2266                 fConfig->GetFXSPort(system),
2267                 fConfig->GetFXSUser(system),
2268                 fConfig->GetFXSHost(system),
2269                 baseFXSFolder.Data(),
2270                 fxsFileName,
2271                 localFileName);
2272
2273         AliDebug(2, Form("%s",command.Data()));
2274
2275         Bool_t result = (gSystem->Exec(command.Data()) == 0);
2276
2277         return result;
2278 }
2279
2280 //______________________________________________________________________________________________
2281 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
2282 {
2283         //
2284         // Get sources producing the condition file Id from file exchange servers
2285         // if id is NULL all sources are returned (distinct)
2286         //
2287
2288         Log(detector, Form("GetFileSources - Retrieving sources with id %s from %s", id, GetSystemName(system)));
2289
2290         // check if test mode should simulate a FXS error
2291         if (fTestMode & kErrorFXSSources)
2292         {
2293                 Log(detector, Form("GetFileSources - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
2294                 return 0;
2295         }
2296
2297         if (system == kDCS)
2298         {
2299                 Log(detector, "GetFileSources - WARNING: DCS system has only one source of data!");
2300                 TList *list = new TList();
2301                 list->SetOwner(1);
2302                 list->Add(new TObjString(" "));
2303                 return list;
2304         }
2305
2306         // check connection, in case connect
2307         if (!Connect(system))
2308         {
2309                 Log(detector, Form("GetFileSources - Couldn't connect to %s FXS database", GetSystemName(system)));
2310                 return NULL;
2311         }
2312
2313         TString sourceName = 0;
2314         if (system == kDAQ)
2315         {
2316                 sourceName = "DAQsource";
2317         } else if (system == kHLT)
2318         {
2319                 sourceName = "DDLnumbers";
2320         }
2321
2322         TString sqlQueryStart = Form("select distinct %s from %s where", sourceName.Data(), fConfig->GetFXSdbTable(system));
2323         TString whereClause = Form("run=%d and detector=\"%s\"",
2324                                 GetCurrentRun(), detector);
2325         if (id)
2326                 whereClause += Form(" and fileId=\"%s\"", id);
2327         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
2328
2329         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2330
2331         // Query execution
2332         TSQLResult* aResult;
2333         aResult = fServer[system]->Query(sqlQuery);
2334         if (!aResult) {
2335                 Log(detector, Form("GetFileSources - Can't execute SQL query to %s database for id: %s",
2336                                 GetSystemName(system), id));
2337                 return 0;
2338         }
2339
2340         TList *list = new TList();
2341         list->SetOwner(1);
2342
2343         if (aResult->GetRowCount() == 0)
2344         {
2345                 Log(detector,
2346                         Form("GetFileSources - No entry in %s FXS table for id: %s", GetSystemName(system), id));
2347                 delete aResult;
2348                 return list;
2349         }
2350
2351         Log(detector, Form("GetFileSources - Found %d sources", aResult->GetRowCount()));
2352
2353         TSQLRow* aRow;
2354         while ((aRow = aResult->Next()))
2355         {
2356
2357                 TString source(aRow->GetField(0), aRow->GetFieldLength(0));
2358                 AliDebug(2, Form("%s = %s", sourceName.Data(), source.Data()));
2359                 list->Add(new TObjString(source));
2360                 delete aRow;
2361         }
2362
2363         delete aResult;
2364
2365         return list;
2366 }
2367
2368 //______________________________________________________________________________________________
2369 TList* AliShuttle::GetFileIDs(Int_t system, const char* detector, const char* source)
2370 {
2371         //
2372         // Get all ids of condition files produced by a given source from file exchange servers
2373         //
2374
2375         Log(detector, Form("GetFileIDs - Retrieving ids with source %s with %s", source, GetSystemName(system)));
2376
2377         // check if test mode should simulate a FXS error
2378         if (fTestMode & kErrorFXSSources)
2379         {
2380                 Log(detector, Form("GetFileIDs - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
2381                 return 0;
2382         }
2383
2384         // check connection, in case connect
2385         if (!Connect(system))
2386         {
2387                 Log(detector, Form("GetFileIDs - Couldn't connect to %s FXS database", GetSystemName(system)));
2388                 return NULL;
2389         }
2390
2391         TString sourceName = 0;
2392         if (system == kDAQ)
2393         {
2394                 sourceName = "DAQsource";
2395         } else if (system == kHLT)
2396         {
2397                 sourceName = "DDLnumbers";
2398         }
2399
2400         TString sqlQueryStart = Form("select fileId from %s where", fConfig->GetFXSdbTable(system));
2401         TString whereClause = Form("run=%d and detector=\"%s\"",
2402                                 GetCurrentRun(), detector);
2403         if (sourceName.Length() > 0 && source)
2404                 whereClause += Form(" and %s=\"%s\"", sourceName.Data(), source);
2405         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
2406
2407         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2408
2409         // Query execution
2410         TSQLResult* aResult;
2411         aResult = fServer[system]->Query(sqlQuery);
2412         if (!aResult) {
2413                 Log(detector, Form("GetFileIDs - Can't execute SQL query to %s database for source: %s",
2414                                 GetSystemName(system), source));
2415                 return 0;
2416         }
2417
2418         TList *list = new TList();
2419         list->SetOwner(1);
2420
2421         if (aResult->GetRowCount() == 0)
2422         {
2423                 Log(detector,
2424                         Form("GetFileIDs - No entry in %s FXS table for source: %s", GetSystemName(system), source));
2425                 delete aResult;
2426                 return list;
2427         }
2428
2429         Log(detector, Form("GetFileIDs - Found %d ids", aResult->GetRowCount()));
2430
2431         TSQLRow* aRow;
2432
2433         while ((aRow = aResult->Next()))
2434         {
2435
2436                 TString id(aRow->GetField(0), aRow->GetFieldLength(0));
2437                 AliDebug(2, Form("fileId = %s", id.Data()));
2438                 list->Add(new TObjString(id));
2439                 delete aRow;
2440         }
2441
2442         delete aResult;
2443
2444         return list;
2445 }
2446
2447 //______________________________________________________________________________________________
2448 Bool_t AliShuttle::Connect(Int_t system)
2449 {
2450         // Connect to MySQL Server of the system's FXS MySQL databases
2451         // DAQ Logbook, Shuttle Logbook and DAQ FXS db are on the same host
2452         //
2453
2454         // check connection: if already connected return
2455         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
2456
2457         TString dbHost, dbUser, dbPass, dbName;
2458
2459         if (system < 3) // FXS db servers
2460         {
2461                 dbHost = Form("mysql://%s:%d", fConfig->GetFXSdbHost(system), fConfig->GetFXSdbPort(system));
2462                 dbUser = fConfig->GetFXSdbUser(system);
2463                 dbPass = fConfig->GetFXSdbPass(system);
2464                 dbName =   fConfig->GetFXSdbName(system);
2465         } else { // Run & Shuttle logbook servers
2466         // TODO Will the Shuttle logbook server be the same as the Run logbook server ???
2467                 dbHost = Form("mysql://%s:%d", fConfig->GetDAQlbHost(), fConfig->GetDAQlbPort());
2468                 dbUser = fConfig->GetDAQlbUser();
2469                 dbPass = fConfig->GetDAQlbPass();
2470                 dbName =   fConfig->GetDAQlbDB();
2471         }
2472
2473         fServer[system] = TSQLServer::Connect(dbHost.Data(), dbUser.Data(), dbPass.Data());
2474         if (!fServer[system] || !fServer[system]->IsConnected()) {
2475                 if(system < 3)
2476                 {
2477                 AliError(Form("Can't establish connection to FXS database for %s",
2478                                         AliShuttleInterface::GetSystemName(system)));
2479                 } else {
2480                 AliError("Can't establish connection to Run logbook.");
2481                 }
2482                 if(fServer[system]) delete fServer[system];
2483                 return kFALSE;
2484         }
2485
2486         // Get tables
2487         TSQLResult* aResult=0;
2488         switch(system){
2489                 case kDAQ:
2490                         aResult = fServer[kDAQ]->GetTables(dbName.Data());
2491                         break;
2492                 case kDCS:
2493                         aResult = fServer[kDCS]->GetTables(dbName.Data());
2494                         break;
2495                 case kHLT:
2496                         aResult = fServer[kHLT]->GetTables(dbName.Data());
2497                         break;
2498                 default:
2499                         aResult = fServer[3]->GetTables(dbName.Data());
2500                         break;
2501         }
2502
2503         delete aResult;
2504         return kTRUE;
2505 }
2506
2507 //______________________________________________________________________________________________
2508 Bool_t AliShuttle::UpdateTable()
2509 {
2510         //
2511         // Update FXS table filling time_processed field in all rows corresponding to current run and detector
2512         //
2513
2514         Bool_t result = kTRUE;
2515
2516         for (UInt_t system=0; system<3; system++)
2517         {
2518                 if(!fFXSCalled[system]) continue;
2519
2520                 // check connection, in case connect
2521                 if (!Connect(system))
2522                 {
2523                         Log(fCurrentDetector, Form("UpdateTable - Couldn't connect to %s FXS database", GetSystemName(system)));
2524                         result = kFALSE;
2525                         continue;
2526                 }
2527
2528                 TTimeStamp now; // now
2529
2530                 // Loop on FXS list entries
2531                 TIter iter(&fFXSlist[system]);
2532                 TObjString *aFXSentry=0;
2533                 while ((aFXSentry = dynamic_cast<TObjString*> (iter.Next())))
2534                 {
2535                         TString aFXSentrystr = aFXSentry->String();
2536                         TObjArray *aFXSarray = aFXSentrystr.Tokenize("#!?!#");
2537                         if (!aFXSarray || aFXSarray->GetEntries() != 2 )
2538                         {
2539                                 Log(fCurrentDetector, Form("UpdateTable - error updating %s FXS entry. Check string: <%s>",
2540                                         GetSystemName(system), aFXSentrystr.Data()));
2541                                 if(aFXSarray) delete aFXSarray;
2542                                 result = kFALSE;
2543                                 continue;
2544                         }
2545                         const char* fileId = ((TObjString*) aFXSarray->At(0))->GetName();
2546                         const char* source = ((TObjString*) aFXSarray->At(1))->GetName();
2547
2548                         TString whereClause;
2549                         if (system == kDAQ)
2550                         {
2551                                 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
2552                                                         GetCurrentRun(), fCurrentDetector.Data(), fileId, source);
2553                         }
2554                         else if (system == kDCS)
2555                         {
2556                                 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\";",
2557                                                         GetCurrentRun(), fCurrentDetector.Data(), fileId);
2558                         }
2559                         else if (system == kHLT)
2560                         {
2561                                 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DDLnumbers=\"%s\";",
2562                                                         GetCurrentRun(), fCurrentDetector.Data(), fileId, source);
2563                         }
2564
2565                         delete aFXSarray;
2566
2567                         TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(system),
2568                                                                 now.GetSec(), whereClause.Data());
2569
2570                         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2571
2572                         // Query execution
2573                         TSQLResult* aResult;
2574                         aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
2575                         if (!aResult)
2576                         {
2577                                 Log(fCurrentDetector, Form("UpdateTable - %s db: can't execute SQL query <%s>",
2578                                                                 GetSystemName(system), sqlQuery.Data()));
2579                                 result = kFALSE;
2580                                 continue;
2581                         }
2582                         delete aResult;
2583                 }
2584         }
2585
2586         return result;
2587 }
2588
2589 //______________________________________________________________________________________________
2590 Bool_t AliShuttle::UpdateTableFailCase()
2591 {
2592         // Update FXS table filling time_processed field in all rows corresponding to current run and detector
2593         // this is called in case the preprocessor is declared failed for the current run, because
2594         // the fields are updated only in case of success
2595
2596         Bool_t result = kTRUE;
2597
2598         for (UInt_t system=0; system<3; system++)
2599         {
2600                 // check connection, in case connect
2601                 if (!Connect(system))
2602                 {
2603                         Log(fCurrentDetector, Form("UpdateTableFailCase - Couldn't connect to %s FXS database",
2604                                                         GetSystemName(system)));
2605                         result = kFALSE;
2606                         continue;
2607                 }
2608
2609                 TTimeStamp now; // now
2610
2611                 // Loop on FXS list entries
2612
2613                 TString whereClause = Form("where run=%d and detector=\"%s\";",
2614                                                 GetCurrentRun(), fCurrentDetector.Data());
2615
2616
2617                 TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(system),
2618                                                         now.GetSec(), whereClause.Data());
2619
2620                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2621
2622                 // Query execution
2623                 TSQLResult* aResult;
2624                 aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
2625                 if (!aResult)
2626                 {
2627                         Log(fCurrentDetector, Form("UpdateTableFailCase - %s db: can't execute SQL query <%s>",
2628                                                         GetSystemName(system), sqlQuery.Data()));
2629                         result = kFALSE;
2630                         continue;
2631                 }
2632                 delete aResult;
2633         }
2634
2635         return result;
2636 }
2637
2638 //______________________________________________________________________________________________
2639 Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status)
2640 {
2641         //
2642         // Update Shuttle logbook filling detector or shuttle_done column
2643         // ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
2644         //
2645
2646         // check connection, in case connect
2647         if(!Connect(3)){
2648                 Log("SHUTTLE", "UpdateShuttleLogbook - Couldn't connect to DAQ Logbook.");
2649                 return kFALSE;
2650         }
2651
2652         TString detName(detector);
2653         TString setClause;
2654         if(detName == "shuttle_done")
2655         {
2656                 setClause = "set shuttle_done=1";
2657
2658                 // Send the information to ML
2659                 TMonaLisaText  mlStatus("SHUTTLE_status", "Done");
2660
2661                 TList mlList;
2662                 mlList.Add(&mlStatus);
2663
2664                 fMonaLisa->SendParameters(&mlList);
2665         } else {
2666                 TString statusStr(status);
2667                 if(statusStr.Contains("done", TString::kIgnoreCase) ||
2668                    statusStr.Contains("failed", TString::kIgnoreCase)){
2669                         setClause = Form("set %s=\"%s\"", detector, status);
2670                 } else {
2671                         Log("SHUTTLE",
2672                                 Form("UpdateShuttleLogbook - Invalid status <%s> for detector %s",
2673                                         status, detector));
2674                         return kFALSE;
2675                 }
2676         }
2677
2678         TString whereClause = Form("where run=%d", GetCurrentRun());
2679
2680         TString sqlQuery = Form("update %s %s %s",
2681                                         fConfig->GetShuttlelbTable(), setClause.Data(), whereClause.Data());
2682
2683         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2684
2685         // Query execution
2686         TSQLResult* aResult;
2687         aResult = dynamic_cast<TSQLResult*> (fServer[3]->Query(sqlQuery));
2688         if (!aResult) {
2689                 Log("SHUTTLE", Form("UpdateShuttleLogbook - Can't execute query <%s>", sqlQuery.Data()));
2690                 return kFALSE;
2691         }
2692         delete aResult;
2693
2694         return kTRUE;
2695 }
2696
2697 //______________________________________________________________________________________________
2698 Int_t AliShuttle::GetCurrentRun() const
2699 {
2700         //
2701         // Get current run from logbook entry
2702         //
2703
2704         return fLogbookEntry ? fLogbookEntry->GetRun() : -1;
2705 }
2706
2707 //______________________________________________________________________________________________
2708 UInt_t AliShuttle::GetCurrentStartTime() const
2709 {
2710         //
2711         // get current start time
2712         //
2713
2714         return fLogbookEntry ? fLogbookEntry->GetStartTime() : 0;
2715 }
2716
2717 //______________________________________________________________________________________________
2718 UInt_t AliShuttle::GetCurrentEndTime() const
2719 {
2720         //
2721         // get current end time from logbook entry
2722         //
2723
2724         return fLogbookEntry ? fLogbookEntry->GetEndTime() : 0;
2725 }
2726
2727 //______________________________________________________________________________________________
2728 UInt_t AliShuttle::GetCurrentYear() const
2729 {
2730         //
2731         // Get current year from logbook entry
2732         //
2733
2734         if (!fLogbookEntry) return 0;
2735
2736         TTimeStamp startTime(GetCurrentStartTime());
2737         TString year =  Form("%d",startTime.GetDate());
2738         year = year(0,4);
2739
2740         return year.Atoi();
2741 }
2742
2743 //______________________________________________________________________________________________
2744 const char* AliShuttle::GetLHCPeriod() const
2745 {
2746         //
2747         // Get current LHC period from logbook entry
2748         //
2749
2750         if (!fLogbookEntry) return 0;
2751
2752         return fLogbookEntry->GetRunParameter("LHCperiod");
2753 }
2754
2755 //______________________________________________________________________________________________
2756 void AliShuttle::Log(const char* detector, const char* message)
2757 {
2758         //
2759         // Fill log string with a message
2760         //
2761
2762         void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
2763         if (dir == NULL) {
2764                 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE)) {
2765                         AliError(Form("Can't open directory <%s>", GetShuttleLogDir()));
2766                         return;
2767                 }
2768
2769         } else {
2770                 gSystem->FreeDirectory(dir);
2771         }
2772
2773         TString toLog = Form("%s (%d): %s - ", TTimeStamp(time(0)).AsString("s"), getpid(), detector);
2774         if (GetCurrentRun() >= 0)
2775                 toLog += Form("run %d - ", GetCurrentRun());
2776         toLog += Form("%s", message);
2777
2778         AliInfo(toLog.Data());
2779
2780         // if we redirect the log output already to the file, leave here
2781         if (fOutputRedirected && strcmp(detector, "SHUTTLE") != 0)
2782                 return;
2783
2784         TString fileName = GetLogFileName(detector);
2785
2786         gSystem->ExpandPathName(fileName);
2787
2788         ofstream logFile;
2789         logFile.open(fileName, ofstream::out | ofstream::app);
2790
2791         if (!logFile.is_open()) {
2792                 AliError(Form("Could not open file %s", fileName.Data()));
2793                 return;
2794         }
2795
2796         logFile << toLog.Data() << "\n";
2797
2798         logFile.close();
2799 }
2800
2801 //______________________________________________________________________________________________
2802 TString AliShuttle::GetLogFileName(const char* detector) const
2803 {
2804         //
2805         // returns the name of the log file for a given sub detector
2806         //
2807
2808         TString fileName;
2809
2810         if (GetCurrentRun() >= 0)
2811                 fileName.Form("%s/%s_%d.log", GetShuttleLogDir(), detector, GetCurrentRun());
2812         else
2813                 fileName.Form("%s/%s.log", GetShuttleLogDir(), detector);
2814
2815         return fileName;
2816 }
2817
2818 //______________________________________________________________________________________________
2819 Bool_t AliShuttle::Collect(Int_t run)
2820 {
2821         //
2822         // Collects conditions data for all UNPROCESSED run written to DAQ LogBook in case of run = -1 (default)
2823         // If a dedicated run is given this run is processed
2824         //
2825         // In operational mode, this is the Shuttle function triggered by the EOR signal.
2826         //
2827
2828         if (run == -1)
2829                 Log("SHUTTLE","Collect - Shuttle called. Collecting conditions data for unprocessed runs");
2830         else
2831                 Log("SHUTTLE", Form("Collect - Shuttle called. Collecting conditions data for run %d", run));
2832
2833         SetLastAction("Starting");
2834
2835         TString whereClause("where shuttle_done=0");
2836         if (run != -1)
2837                 whereClause += Form(" and run=%d", run);
2838
2839         TObjArray shuttleLogbookEntries;
2840         if (!QueryShuttleLogbook(whereClause, shuttleLogbookEntries))
2841         {
2842                 Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
2843                 return kFALSE;
2844         }
2845
2846         if (shuttleLogbookEntries.GetEntries() == 0)
2847         {
2848                 if (run == -1)
2849                         Log("SHUTTLE","Collect - Found no UNPROCESSED runs in Shuttle logbook");
2850                 else
2851                         Log("SHUTTLE", Form("Collect - Run %d is already DONE "
2852                                                 "or it does not exist in Shuttle logbook", run));
2853                 return kTRUE;
2854         }
2855
2856         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
2857                 fFirstUnprocessed[iDet] = kTRUE;
2858
2859         if (run != -1)
2860         {
2861                 // query Shuttle logbook for earlier runs, check if some detectors are unprocessed,
2862                 // flag them into fFirstUnprocessed array
2863                 TString whereClause(Form("where shuttle_done=0 and run < %d", run));
2864                 TObjArray tmpLogbookEntries;
2865                 if (!QueryShuttleLogbook(whereClause, tmpLogbookEntries))
2866                 {
2867                         Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
2868                         return kFALSE;
2869                 }
2870
2871                 TIter iter(&tmpLogbookEntries);
2872                 AliShuttleLogbookEntry* anEntry = 0;
2873                 while ((anEntry = dynamic_cast<AliShuttleLogbookEntry*> (iter.Next())))
2874                 {
2875                         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
2876                         {
2877                                 if (anEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
2878                                 {
2879                                         AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
2880                                                         anEntry->GetRun(), GetDetName(iDet)));
2881                                         fFirstUnprocessed[iDet] = kFALSE;
2882                                 }
2883                         }
2884
2885                 }
2886
2887         }
2888
2889         if (!RetrieveConditionsData(shuttleLogbookEntries))
2890         {
2891                 Log("SHUTTLE", "Collect - Process of at least one run failed");
2892                 return kFALSE;
2893         }
2894
2895         Log("SHUTTLE", "Collect - Requested run(s) successfully processed");
2896         return kTRUE;
2897 }
2898
2899 //______________________________________________________________________________________________
2900 Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
2901 {
2902         //
2903         // Retrieve conditions data for all runs that aren't processed yet
2904         //
2905
2906         Bool_t hasError = kFALSE;
2907
2908         TIter iter(&dateEntries);
2909         AliShuttleLogbookEntry* anEntry;
2910
2911         while ((anEntry = (AliShuttleLogbookEntry*) iter.Next())){
2912                 if (!Process(anEntry)){
2913                         hasError = kTRUE;
2914                 }
2915
2916                 // clean SHUTTLE temp directory
2917                 //TString filename = Form("%s/*.shuttle", GetShuttleTempDir());
2918                 //RemoveFile(filename.Data());
2919         }
2920
2921         return hasError == kFALSE;
2922 }
2923
2924 //______________________________________________________________________________________________
2925 ULong_t AliShuttle::GetTimeOfLastAction() const
2926 {
2927         //
2928         // Gets time of last action
2929         //
2930
2931         ULong_t tmp;
2932
2933         fMonitoringMutex->Lock();
2934
2935         tmp = fLastActionTime;
2936
2937         fMonitoringMutex->UnLock();
2938
2939         return tmp;
2940 }
2941
2942 //______________________________________________________________________________________________
2943 const TString AliShuttle::GetLastAction() const
2944 {
2945         //
2946         // returns a string description of the last action
2947         //
2948
2949         TString tmp;
2950
2951         fMonitoringMutex->Lock();
2952
2953         tmp = fLastAction;
2954
2955         fMonitoringMutex->UnLock();
2956
2957         return tmp;
2958 }
2959
2960 //______________________________________________________________________________________________
2961 void AliShuttle::SetLastAction(const char* action)
2962 {
2963         //
2964         // updates the monitoring variables
2965         //
2966
2967         fMonitoringMutex->Lock();
2968
2969         fLastAction = action;
2970         fLastActionTime = time(0);
2971
2972         fMonitoringMutex->UnLock();
2973 }
2974
2975 //______________________________________________________________________________________________
2976 const char* AliShuttle::GetRunParameter(const char* param)
2977 {
2978         //
2979         // returns run parameter read from DAQ logbook
2980         //
2981
2982         if(!fLogbookEntry) {
2983                 AliError("No logbook entry!");
2984                 return 0;
2985         }
2986
2987         return fLogbookEntry->GetRunParameter(param);
2988 }
2989
2990 //______________________________________________________________________________________________
2991 AliCDBEntry* AliShuttle::GetFromOCDB(const char* detector, const AliCDBPath& path)
2992 {
2993         //
2994         // returns object from OCDB valid for current run
2995         //
2996
2997         if (fTestMode & kErrorOCDB)
2998         {
2999                 Log(detector, "GetFromOCDB - In TESTMODE - Simulating error with OCDB");
3000                 return 0;
3001         }
3002
3003         AliCDBStorage *sto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
3004         if (!sto)
3005         {
3006                 Log(detector, "GetFromOCDB - Cannot activate main OCDB for query!");
3007                 return 0;
3008         }
3009
3010         return dynamic_cast<AliCDBEntry*> (sto->Get(path, GetCurrentRun()));
3011 }
3012
3013 //______________________________________________________________________________________________
3014 Bool_t AliShuttle::SendMail()
3015 {
3016         //
3017         // sends a mail to the subdetector expert in case of preprocessor error
3018         //
3019
3020         if (fTestMode != kNone)
3021                 return kTRUE;
3022
3023         void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
3024         if (dir == NULL)
3025         {
3026                 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE))
3027                 {
3028                         Log("SHUTTLE", Form("SendMail - Can't open directory <%s>", GetShuttleLogDir()));
3029                         return kFALSE;
3030                 }
3031
3032         } else {
3033                 gSystem->FreeDirectory(dir);
3034         }
3035
3036         TString bodyFileName;
3037         bodyFileName.Form("%s/mail.body", GetShuttleLogDir());
3038         gSystem->ExpandPathName(bodyFileName);
3039
3040         ofstream mailBody;
3041         mailBody.open(bodyFileName, ofstream::out);
3042
3043         if (!mailBody.is_open())
3044         {
3045                 Log("SHUTTLE", Form("Could not open mail body file %s", bodyFileName.Data()));
3046                 return kFALSE;
3047         }
3048
3049         TString to="";
3050         TIter iterExperts(fConfig->GetResponsibles(fCurrentDetector));
3051         TObjString *anExpert=0;
3052         while ((anExpert = (TObjString*) iterExperts.Next()))
3053         {
3054                 to += Form("%s,", anExpert->GetName());
3055         }
3056         to.Remove(to.Length()-1);
3057         AliDebug(2, Form("to: %s",to.Data()));
3058
3059         if (to.IsNull()) {
3060                 Log("SHUTTLE", "List of detector responsibles not yet set!");
3061                 return kFALSE;
3062         }
3063
3064         TString cc="alberto.colla@cern.ch";
3065
3066         TString subject = Form("%s Shuttle preprocessor FAILED in run %d !",
3067                                 fCurrentDetector.Data(), GetCurrentRun());
3068         AliDebug(2, Form("subject: %s", subject.Data()));
3069
3070         TString body = Form("Dear %s expert(s), \n\n", fCurrentDetector.Data());
3071         body += Form("SHUTTLE just detected that your preprocessor "
3072                         "failed processing run %d!!\n\n", GetCurrentRun());
3073         body += Form("Please check %s status on the SHUTTLE monitoring page: \n\n", fCurrentDetector.Data());
3074         body += Form("\thttp://pcalimonitor.cern.ch:8889/shuttle.jsp?time=168 \n\n");
3075         body += Form("Find the %s log for the current run on \n\n"
3076                 "\thttp://pcalishuttle01.cern.ch:8880/logs/%s_%d.log \n\n",
3077                 fCurrentDetector.Data(), fCurrentDetector.Data(), GetCurrentRun());
3078         body += Form("The last 10 lines of %s log file are following:\n\n");
3079
3080         AliDebug(2, Form("Body begin: %s", body.Data()));
3081
3082         mailBody << body.Data();
3083         mailBody.close();
3084         mailBody.open(bodyFileName, ofstream::out | ofstream::app);
3085
3086         TString logFileName = Form("%s/%s_%d.log", GetShuttleLogDir(), fCurrentDetector.Data(), GetCurrentRun());
3087         TString tailCommand = Form("tail -n 10 %s >> %s", logFileName.Data(), bodyFileName.Data());
3088         if (gSystem->Exec(tailCommand.Data()))
3089         {
3090                 mailBody << Form("%s log file not found ...\n\n", fCurrentDetector.Data());
3091         }
3092
3093         TString endBody = Form("------------------------------------------------------\n\n");
3094         endBody += Form("In case of problems please contact the SHUTTLE core team.\n\n");
3095         endBody += "Please do not answer this message directly, it is automatically generated.\n\n";
3096         endBody += "Greetings,\n\n \t\t\tthe SHUTTLE\n";
3097
3098         AliDebug(2, Form("Body end: %s", endBody.Data()));
3099
3100         mailBody << endBody.Data();
3101
3102         mailBody.close();
3103
3104         // send mail!
3105         TString mailCommand = Form("mail -s \"%s\" -c %s %s < %s",
3106                                                 subject.Data(),
3107                                                 cc.Data(),
3108                                                 to.Data(),
3109                                                 bodyFileName.Data());
3110         AliDebug(2, Form("mail command: %s", mailCommand.Data()));
3111
3112         Bool_t result = gSystem->Exec(mailCommand.Data());
3113
3114         return result == 0;
3115 }
3116
3117 //______________________________________________________________________________________________
3118 Bool_t AliShuttle::SendMailToDCS()
3119 {
3120         //
3121         // sends a mail to the DCS experts in case of DCS error
3122         //
3123
3124         if (fTestMode != kNone)
3125                 return kTRUE;
3126
3127         void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
3128         if (dir == NULL)
3129         {
3130                 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE))
3131                 {
3132                         Log("SHUTTLE", Form("SendMailToDCS - Can't open directory <%s>", GetShuttleLogDir()));
3133                         return kFALSE;
3134                 }
3135
3136         } else {
3137                 gSystem->FreeDirectory(dir);
3138         }
3139
3140         TString bodyFileName;
3141         bodyFileName.Form("%s/mail.body", GetShuttleLogDir());
3142         gSystem->ExpandPathName(bodyFileName);
3143
3144         ofstream mailBody;
3145         mailBody.open(bodyFileName, ofstream::out);
3146
3147         if (!mailBody.is_open())
3148         {
3149                 Log("SHUTTLE", Form("SendMailToDCS - Could not open mail body file %s", bodyFileName.Data()));
3150                 return kFALSE;
3151         }
3152
3153         TString to="Vladimir.Fekete@cern.ch, Svetozar.Kapusta@cern.ch";
3154         //TString to="alberto.colla@cern.ch";
3155         AliDebug(2, Form("to: %s",to.Data()));
3156
3157         if (to.IsNull()) {
3158                 Log("SHUTTLE", "List of detector responsibles not yet set!");
3159                 return kFALSE;
3160         }
3161
3162         TString cc="alberto.colla@cern.ch";
3163
3164         TString subject = Form("Retrieval of data points for %s FAILED in run %d !",
3165                                 fCurrentDetector.Data(), GetCurrentRun());
3166         AliDebug(2, Form("subject: %s", subject.Data()));
3167
3168         TString body = Form("Dear DCS experts, \n\n");
3169         body += Form("SHUTTLE couldn\'t retrieve the data points for detector %s "
3170                         "in run %d!!\n\n", fCurrentDetector.Data(), GetCurrentRun());
3171         body += Form("Please check %s status on the SHUTTLE monitoring page: \n\n", fCurrentDetector.Data());
3172         body += Form("\thttp://pcalimonitor.cern.ch:8889/shuttle.jsp?time=168 \n\n");
3173         body += Form("Find the %s log for the current run on \n\n"
3174                 "\thttp://pcalishuttle01.cern.ch:8880/logs/%s_%d.log \n\n",
3175                 fCurrentDetector.Data(), fCurrentDetector.Data(), GetCurrentRun());
3176         body += Form("The last 10 lines of %s log file are following:\n\n");
3177
3178         AliDebug(2, Form("Body begin: %s", body.Data()));
3179
3180         mailBody << body.Data();
3181         mailBody.close();
3182         mailBody.open(bodyFileName, ofstream::out | ofstream::app);
3183
3184         TString logFileName = Form("%s/%s_%d.log", GetShuttleLogDir(), fCurrentDetector.Data(), GetCurrentRun());
3185         TString tailCommand = Form("tail -n 10 %s >> %s", logFileName.Data(), bodyFileName.Data());
3186         if (gSystem->Exec(tailCommand.Data()))
3187         {
3188                 mailBody << Form("%s log file not found ...\n\n", fCurrentDetector.Data());
3189         }
3190
3191         TString endBody = Form("------------------------------------------------------\n\n");
3192         endBody += Form("In case of problems please contact the SHUTTLE core team.\n\n");
3193         endBody += "Please do not answer this message directly, it is automatically generated.\n\n";
3194         endBody += "Greetings,\n\n \t\t\tthe SHUTTLE\n";
3195
3196         AliDebug(2, Form("Body end: %s", endBody.Data()));
3197
3198         mailBody << endBody.Data();
3199
3200         mailBody.close();
3201
3202         // send mail!
3203         TString mailCommand = Form("mail -s \"%s\" -c %s %s < %s",
3204                                                 subject.Data(),
3205                                                 cc.Data(),
3206                                                 to.Data(),
3207                                                 bodyFileName.Data());
3208         AliDebug(2, Form("mail command: %s", mailCommand.Data()));
3209
3210         Bool_t result = gSystem->Exec(mailCommand.Data());
3211
3212         return result == 0;
3213 }
3214
3215 //______________________________________________________________________________________________
3216 const char* AliShuttle::GetRunType()
3217 {
3218         //
3219         // returns run type read from "run type" logbook
3220         //
3221
3222         if(!fLogbookEntry) {
3223                 AliError("No logbook entry!");
3224                 return 0;
3225         }
3226
3227         return fLogbookEntry->GetRunType();
3228 }
3229
3230 //______________________________________________________________________________________________
3231 Bool_t AliShuttle::GetHLTStatus()
3232 {
3233         // Return HLT status (ON=1 OFF=0)
3234         // Converts the HLT status from the status string read in the run logbook (not just a bool)
3235
3236         if(!fLogbookEntry) {
3237                 AliError("No logbook entry!");
3238                 return 0;
3239         }
3240
3241         // TODO implement when HLTStatus is inserted in run logbook
3242         //TString hltStatus = fLogbookEntry->GetRunParameter("HLTStatus");
3243         //if(hltStatus == "OFF") {return kFALSE};
3244
3245         return kTRUE;
3246 }
3247
3248 //______________________________________________________________________________________________
3249 void AliShuttle::SetShuttleTempDir(const char* tmpDir)
3250 {
3251         //
3252         // sets Shuttle temp directory
3253         //
3254
3255         fgkShuttleTempDir = gSystem->ExpandPathName(tmpDir);
3256 }
3257
3258 //______________________________________________________________________________________________
3259 void AliShuttle::SetShuttleLogDir(const char* logDir)
3260 {
3261         //
3262         // sets Shuttle log directory
3263         //
3264
3265         fgkShuttleLogDir = gSystem->ExpandPathName(logDir);
3266 }