SHUTTLE/AliShuttle.cxx

   1 /**************************************************************************
   2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
   3  *                                                                        *
   4  * Author: The ALICE Off-line Project.                                    *
   5  * Contributors are mentioned in the code where appropriate.              *
   6  *                                                                        *
   7  * Permission to use, copy, modify and distribute this software and its   *
   8  * documentation strictly for non-commercial purposes is hereby granted   *
   9  * without fee, provided that the above copyright notice appears in all   *
  10  * copies and that both the copyright notice and this permission notice   *
  11  * appear in the supporting documentation. The authors make no claims     *
  12  * about the suitability of this software for any purpose. It is          *
  13  * provided "as is" without express or implied warranty.                  *
  14  **************************************************************************/
  15
  16 /*
  17 $Log$
  18 Revision 1.62  2007/10/31 18:23:13  acolla
  19 Furter developement on the Shuttle:
  20
  21 - Shuttle now connects to the Grid as alidaq. The OCDB and Reference folders
  22 are now built from /alice/data, e.g.:
  23 /alice/data/2007/LHC07a/OCDB
  24
  25 the year and LHC period are taken from the Shuttle.
  26 Raw metadata files are stored by GRP to:
  27 /alice/data/2007/LHC07a/<runNb>/Raw/RunMetadata.root
  28
  29 - Shuttle sends a mail to DCS experts each time DP retrieval fails.
  30
  31 Revision 1.61  2007/10/30 20:33:51  acolla
  32 Improved managing of temporary folders, which weren't correctly handled.
  33 Resolved bug introduced in StoreReferenceFile, which caused SPD preprocessor fail.
  34
  35 Revision 1.60  2007/10/29 18:06:16  acolla
  36
  37 New function StoreRunMetadataFile added to preprocessor and Shuttle interface
  38 This function can be used by GRP only. It stores raw data tags merged file to the
  39 raw data folder (e.g. /alice/data/2008/LHC08a/000099999/Raw).
  40
  41 KNOWN ISSUES:
  42
  43 1. Shuttle cannot write to /alice/data/ because it belongs to alidaq. Tag file is stored in /alice/simulation/... for the time being.
  44 2. Due to a bug in TAlien::Mkdir, the creation of a folder in recursive mode (-p option) does not work. The problem
  45 has been corrected in the root package on the Shuttle machine.
  46
  47 Revision 1.59  2007/10/05 12:40:55  acolla
  48
  49 Result error code added to AliDCSClient data members (it was "lost" with the new implementation of TMap* GetAliasValues and GetDPValues).
  50
  51 Revision 1.58  2007/09/28 15:27:40  acolla
  52
  53 AliDCSClient "multiSplit" option added in the DCS configuration
  54 in AliDCSMessage: variable MAX_BODY_SIZE set to 500000
  55
  56 Revision 1.57  2007/09/27 16:53:13  acolla
  57 Detectors can have more than one AMANDA server. SHUTTLE queries the servers sequentially,
  58 merges the dcs aliases/DPs in one TMap and sends it to the preprocessor.
  59
  60 Revision 1.56  2007/09/14 16:46:14  jgrosseo
  61 1) Connect and Close are called before and after each query, so one can
  62 keep the same AliDCSClient object.
  63 2) The splitting of a query is moved to GetDPValues/GetAliasValues.
  64 3) Splitting interval can be specified in constructor
  65
  66 Revision 1.55  2007/08/06 12:26:40  acolla
  67 Function Bool_t GetHLTStatus added to preprocessor. It returns the status of HLT
  68 read from the run logbook.
  69
  70 Revision 1.54  2007/07/12 09:51:25  jgrosseo
  71 removed duplicated log message in GetFile
  72
  73 Revision 1.53  2007/07/12 09:26:28  jgrosseo
  74 updating hlt fxs base path
  75
  76 Revision 1.52  2007/07/12 08:06:45  jgrosseo
  77 adding log messages in getfile... functions
  78 adding not implemented copy constructor in alishuttleconfigholder
  79
  80 Revision 1.51  2007/07/03 17:24:52  acolla
  81 root moved to v5-16-00. TFileMerger->Cp moved to TFile::Cp.
  82
  83 Revision 1.50  2007/07/02 17:19:32  acolla
  84 preprocessor is run in a temp directory that is removed when process is finished.
  85
  86 Revision 1.49  2007/06/29 10:45:06  acolla
  87 Number of columns in MySql Shuttle logbook increased by one (HLT added)
  88
  89 Revision 1.48  2007/06/21 13:06:19  acolla
  90 GetFileSources returns dummy list with 1 source if system=DCS (better than
  91 returning error as it was)
  92
  93 Revision 1.47  2007/06/19 17:28:56  acolla
  94 HLT updated; missing map bug removed.
  95
  96 Revision 1.46  2007/06/09 13:01:09  jgrosseo
  97 Switching to retrieval of several DCS DPs at a time (multiDPrequest)
  98
  99 Revision 1.45  2007/05/30 06:35:20  jgrosseo
 100 Adding functionality to the Shuttle/TestShuttle:
 101 o) Function to retrieve list of sources from a given system (GetFileSources with id=0)
 102 o) Function to retrieve list of IDs for a given source      (GetFileIDs)
 103 These functions are needed for dealing with the tag files that are saved for the GRP preprocessor
 104 Example code has been added to the TestProcessor in TestShuttle
 105
 106 Revision 1.44  2007/05/11 16:09:32  acolla
 107 Reference files for ITS, MUON and PHOS are now stored in OfflineDetName/OnlineDetName/run_...
 108 example: ITS/SPD/100_filename.root
 109
 110 Revision 1.43  2007/05/10 09:59:51  acolla
 111 Various bug fixes in StoreRefFilesToGrid; Cleaning of reference storage before processing detector (CleanReferenceStorage)
 112
 113 Revision 1.42  2007/05/03 08:01:39  jgrosseo
 114 typo in last commit :-(
 115
 116 Revision 1.41  2007/05/03 08:00:48  jgrosseo
 117 fixing log message when pp want to skip dcs value retrieval
 118
 119 Revision 1.40  2007/04/27 07:06:48  jgrosseo
 120 GetFileSources returns empty list in case of no files, but successful query
 121 No mails sent in testmode
 122
 123 Revision 1.39  2007/04/17 12:43:57  acolla
 124 Correction in StoreOCDB; change of text in mail to detector expert
 125
 126 Revision 1.38  2007/04/12 08:26:18  jgrosseo
 127 updated comment
 128
 129 Revision 1.37  2007/04/10 16:53:14  jgrosseo
 130 redirecting sub detector stdout, stderr to sub detector log file
 131
 132 Revision 1.35  2007/04/04 16:26:38  acolla
 133 1. Re-organization of function calls in TestPreprocessor to make it more meaningful.
 134 2. Added missing dependency in test preprocessors.
 135 3. in AliShuttle.cxx: processing time and memory consumption info on a single line.
 136
 137 Revision 1.34  2007/04/04 10:33:36  jgrosseo
 138 1) Storing of files to the Grid is now done _after_ your preprocessors succeeded. This is transparent, which means that you can still use the same functions (Store, StoreReferenceData) to store files to the Grid. However, the Shuttle first stores them locally and transfers them after the preprocessor finished. The return code of these two functions has changed from UInt_t to Bool_t which gives you the success of the storing.
 139 In case of an error with the Grid, the Shuttle will retry the storing later, the preprocessor does not need to be run again.
 140
 141 2) The meaning of the return code of the preprocessor has changed. 0 is now success and any other value means failure. This value is stored in the log and you can use it to keep details about the error condition.
 142
 143 3) New function StoreReferenceFile to _directly_ store a file (without opening it) to the reference storage.
 144
 145 4) The memory usage of the preprocessor is monitored. If it exceeds 2 GB it is terminated.
 146
 147 5) New function AliPreprocessor::ProcessDCS(). If you do not need to have DCS data in all cases, you can skip the processing by implemting this function and returning kFALSE under certain conditions. E.g. if there is a certain run type.
 148 If you always need DCS data (like before), you do not need to implement it.
 149
 150 6) The run type has been added to the monitoring page
 151
 152 Revision 1.33  2007/04/03 13:56:01  acolla
 153 Grid Storage at the end of preprocessing. Added virtual method to disable DCS query according to the
 154 run type.
 155
 156 Revision 1.32  2007/02/28 10:41:56  acolla
 157 Run type field added in SHUTTLE framework. Run type is read from "run type" logbook and retrieved by
 158 AliPreprocessor::GetRunType() function.
 159 Added some ldap definition files.
 160
 161 Revision 1.30  2007/02/13 11:23:21  acolla
 162 Moved getters and setters of Shuttle's main OCDB/Reference, local
 163 OCDB/Reference, temp and log folders to AliShuttleInterface
 164
 165 Revision 1.27  2007/01/30 17:52:42  jgrosseo
 166 adding monalisa monitoring
 167
 168 Revision 1.26  2007/01/23 19:20:03  acolla
 169 Removed old ldif files, added TOF, MCH ldif files. Added some options in
 170 AliShuttleConfig::Print. Added in Ali Shuttle: SetShuttleTempDir and
 171 SetShuttleLogDir
 172
 173 Revision 1.25  2007/01/15 19:13:52  acolla
 174 Moved some AliInfo to AliDebug in SendMail function
 175
 176 Revision 1.21  2006/12/07 08:51:26  jgrosseo
 177 update (alberto):
 178 table, db names in ldap configuration
 179 added GRP preprocessor
 180 DCS data can also be retrieved by data point
 181
 182 Revision 1.20  2006/11/16 16:16:48  jgrosseo
 183 introducing strict run ordering flag
 184 removed giving preprocessor name to preprocessor, they have to know their name themselves ;-)
 185
 186 Revision 1.19  2006/11/06 14:23:04  jgrosseo
 187 major update (Alberto)
 188 o) reading of run parameters from the logbook
 189 o) online offline naming conversion
 190 o) standalone DCSclient package
 191
 192 Revision 1.18  2006/10/20 15:22:59  jgrosseo
 193 o) Adding time out to the execution of the preprocessors: The Shuttle forks and the parent process monitors the child
 194 o) Merging Collect, CollectAll, CollectNew function
 195 o) Removing implementation of empty copy constructors (declaration still there!)
 196
 197 Revision 1.17  2006/10/05 16:20:55  jgrosseo
 198 adapting to new CDB classes
 199
 200 Revision 1.16  2006/10/05 15:46:26  jgrosseo
 201 applying to the new interface
 202
 203 Revision 1.15  2006/10/02 16:38:39  jgrosseo
 204 update (alberto):
 205 fixed memory leaks
 206 storing of objects that failed to be stored to the grid before
 207 interfacing of shuttle status table in daq system
 208
 209 Revision 1.14  2006/08/29 09:16:05  jgrosseo
 210 small update
 211
 212 Revision 1.13  2006/08/15 10:50:00  jgrosseo
 213 effc++ corrections (alberto)
 214
 215 Revision 1.12  2006/08/08 14:19:29  jgrosseo
 216 Update to shuttle classes (Alberto)
 217
 218 - Possibility to set the full object's path in the Preprocessor's and
 219 Shuttle's  Store functions
 220 - Possibility to extend the object's run validity in the same classes
 221 ("startValidity" and "validityInfinite" parameters)
 222 - Implementation of the StoreReferenceData function to store reference
 223 data in a dedicated CDB storage.
 224
 225 Revision 1.11  2006/07/21 07:37:20  jgrosseo
 226 last run is stored after each run
 227
 228 Revision 1.10  2006/07/20 09:54:40  jgrosseo
 229 introducing status management: The processing per subdetector is divided into several steps,
 230 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
 231 can keep track of the number of failures and skips further processing after a certain threshold is
 232 exceeded. These thresholds can be configured in LDAP.
 233
 234 Revision 1.9  2006/07/19 10:09:55  jgrosseo
 235 new configuration, accesst to DAQ FES (Alberto)
 236
 237 Revision 1.8  2006/07/11 12:44:36  jgrosseo
 238 adding parameters for extended validity range of data produced by preprocessor
 239
 240 Revision 1.7  2006/07/10 14:37:09  jgrosseo
 241 small fix + todo comment
 242
 243 Revision 1.6  2006/07/10 13:01:41  jgrosseo
 244 enhanced storing of last sucessfully processed run (alberto)
 245
 246 Revision 1.5  2006/07/04 14:59:57  jgrosseo
 247 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
 248
 249 Revision 1.4  2006/06/12 09:11:16  jgrosseo
 250 coding conventions (Alberto)
 251
 252 Revision 1.3  2006/06/06 14:26:40  jgrosseo
 253 o) removed files that were moved to STEER
 254 o) shuttle updated to follow the new interface (Alberto)
 255
 256 Revision 1.2  2006/03/07 07:52:34  hristov
 257 New version (B.Yordanov)
 258
 259 Revision 1.6  2005/11/19 17:19:14  byordano
 260 RetrieveDATEEntries and RetrieveConditionsData added
 261
 262 Revision 1.5  2005/11/19 11:09:27  byordano
 263 AliShuttle declaration added
 264
 265 Revision 1.4  2005/11/17 17:47:34  byordano
 266 TList changed to TObjArray
 267
 268 Revision 1.3  2005/11/17 14:43:23  byordano
 269 import to local CVS
 270
 271 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
 272 Initial import as subdirectory in AliRoot
 273
 274 Revision 1.2  2005/09/13 08:41:15  byordano
 275 default startTime endTime added
 276
 277 Revision 1.4  2005/08/30 09:13:02  byordano
 278 some docs added
 279
 280 Revision 1.3  2005/08/29 21:15:47  byordano
 281 some docs added
 282
 283 */
 284
 285 //
 286 // This class is the main manager for AliShuttle.
 287 // It organizes the data retrieval from DCS and call the
 288 // interface methods of AliPreprocessor.
 289 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
 290 // data for its set of aliases is retrieved. If there is registered
 291 // AliPreprocessor for this detector then it will be used
 292 // accroding to the schema (see AliPreprocessor).
 293 // If there isn't registered AliPreprocessor than the retrieved
 294 // data is stored automatically to the undelying AliCDBStorage.
 295 // For detSpec is used the alias name.
 296 //
 297
 298 #include "AliShuttle.h"
 299
 300 #include "AliCDBManager.h"
 301 #include "AliCDBStorage.h"
 302 #include "AliCDBId.h"
 303 #include "AliCDBRunRange.h"
 304 #include "AliCDBPath.h"
 305 #include "AliCDBEntry.h"
 306 #include "AliShuttleConfig.h"
 307 #include "DCSClient/AliDCSClient.h"
 308 #include "AliLog.h"
 309 #include "AliPreprocessor.h"
 310 #include "AliShuttleStatus.h"
 311 #include "AliShuttleLogbookEntry.h"
 312
 313 #include <TSystem.h>
 314 #include <TObject.h>
 315 #include <TString.h>
 316 #include <TTimeStamp.h>
 317 #include <TObjString.h>
 318 #include <TSQLServer.h>
 319 #include <TSQLResult.h>
 320 #include <TSQLRow.h>
 321 #include <TMutex.h>
 322 #include <TSystemDirectory.h>
 323 #include <TSystemFile.h>
 324 #include <TFile.h>
 325 #include <TGrid.h>
 326 #include <TGridResult.h>
 327
 328 #include <TMonaLisaWriter.h>
 329
 330 #include <fstream>
 331
 332 #include <sys/types.h>
 333 #include <sys/wait.h>
 334
 335 ClassImp(AliShuttle)
 336
 337 //______________________________________________________________________________________________
 338 AliShuttle::AliShuttle(const AliShuttleConfig* config,
 339                 UInt_t timeout, Int_t retries):
 340 fConfig(config),
 341 fTimeout(timeout), fRetries(retries),
 342 fPreprocessorMap(),
 343 fLogbookEntry(0),
 344 fCurrentDetector(),
 345 fStatusEntry(0),
 346 fMonitoringMutex(0),
 347 fLastActionTime(0),
 348 fLastAction(),
 349 fMonaLisa(0),
 350 fTestMode(kNone),
 351 fReadTestMode(kFALSE),
 352 fOutputRedirected(kFALSE)
 353 {
 354         //
 355         // config: AliShuttleConfig used
 356         // timeout: timeout used for AliDCSClient connection
 357         // retries: the number of retries in case of connection error.
 358         //
 359
 360         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
 361         for(int iSys=0;iSys<4;iSys++) {
 362                 fServer[iSys]=0;
 363                 if (iSys < 3)
 364                         fFXSlist[iSys].SetOwner(kTRUE);
 365         }
 366         fPreprocessorMap.SetOwner(kTRUE);
 367
 368         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
 369                 fFirstUnprocessed[iDet] = kFALSE;
 370
 371         fMonitoringMutex = new TMutex();
 372 }
 373
 374 //______________________________________________________________________________________________
 375 AliShuttle::~AliShuttle()
 376 {
 377         //
 378         // destructor
 379         //
 380
 381         fPreprocessorMap.DeleteAll();
 382         for(int iSys=0;iSys<4;iSys++)
 383                 if(fServer[iSys]) {
 384                         fServer[iSys]->Close();
 385                         delete fServer[iSys];
 386                         fServer[iSys] = 0;
 387                 }
 388
 389         if (fStatusEntry){
 390                 delete fStatusEntry;
 391                 fStatusEntry = 0;
 392         }
 393
 394         if (fMonitoringMutex)
 395         {
 396                 delete fMonitoringMutex;
 397                 fMonitoringMutex = 0;
 398         }
 399 }
 400
 401 //______________________________________________________________________________________________
 402 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
 403 {
 404         //
 405         // Registers new AliPreprocessor.
 406         // It uses GetName() for indentificator of the pre processor.
 407         // The pre processor is registered it there isn't any other
 408         // with the same identificator (GetName()).
 409         //
 410
 411         const char* detName = preprocessor->GetName();
 412         if(GetDetPos(detName) < 0)
 413                 AliFatal(Form("********** !!!!! Invalid detector name: %s !!!!! **********", detName));
 414
 415         if (fPreprocessorMap.GetValue(detName)) {
 416                 AliWarning(Form("AliPreprocessor %s is already registered!", detName));
 417                 return;
 418         }
 419
 420         fPreprocessorMap.Add(new TObjString(detName), preprocessor);
 421 }
 422 //______________________________________________________________________________________________
 423 Bool_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
 424                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
 425 {
 426         // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
 427         // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
 428         // using this function. Use StoreReferenceData instead!
 429         // It calls StoreLocally function which temporarily stores the data locally; when the preprocessor
 430         // finishes the data are transferred to the main storage (Grid).
 431
 432         return StoreLocally(fgkLocalCDB, path, object, metaData, validityStart, validityInfinite);
 433 }
 434
 435 //______________________________________________________________________________________________
 436 Bool_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData)
 437 {
 438         // Stores a CDB object in the storage for reference data. This objects will not be available during
 439         // offline reconstrunction. Use this function for reference data only!
 440         // It calls StoreLocally function which temporarily stores the data locally; when the preprocessor
 441         // finishes the data are transferred to the main storage (Grid).
 442
 443         return StoreLocally(fgkLocalRefStorage, path, object, metaData);
 444 }
 445
 446 //______________________________________________________________________________________________
 447 Bool_t AliShuttle::StoreLocally(const TString& localUri,
 448                         const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
 449                         Int_t validityStart, Bool_t validityInfinite)
 450 {
 451         // Store object temporarily in local storage. Parameters are passed by Store and StoreReferenceData functions.
 452         // when the preprocessor finishes the data are transferred to the main storage (Grid).
 453         // The parameters are:
 454         //   1) Uri of the backup storage (Local)
 455         //   2) the object's path.
 456         //   3) the object to be stored
 457         //   4) the metaData to be associated with the object
 458         //   5) the validity start run number w.r.t. the current run,
 459         //      if the data is valid only for this run leave the default 0
 460         //   6) specifies if the calibration data is valid for infinity (this means until updated),
 461         //      typical for calibration runs, the default is kFALSE
 462         //
 463         // returns 0 if fail, 1 otherwise
 464
 465         if (fTestMode & kErrorStorage)
 466         {
 467                 Log(fCurrentDetector, "StoreLocally - In TESTMODE - Simulating error while storing locally");
 468                 return kFALSE;
 469         }
 470
 471         const char* cdbType = (localUri == fgkLocalCDB) ? "CDB" : "Reference";
 472
 473         Int_t firstRun = GetCurrentRun() - validityStart;
 474         if(firstRun < 0) {
 475                 AliWarning("First valid run happens to be less than 0! Setting it to 0.");
 476                 firstRun=0;
 477         }
 478
 479         Int_t lastRun = -1;
 480         if(validityInfinite) {
 481                 lastRun = AliCDBRunRange::Infinity();
 482         } else {
 483                 lastRun = GetCurrentRun();
 484         }
 485
 486         // Version is set to current run, it will be used later to transfer data to Grid
 487         AliCDBId id(path, firstRun, lastRun, GetCurrentRun(), -1);
 488
 489         if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
 490                 TObjString runUsed = Form("%d", GetCurrentRun());
 491                 metaData->SetProperty("RunUsed(TObjString)", runUsed.Clone());
 492         }
 493
 494         Bool_t result = kFALSE;
 495
 496         if (!(AliCDBManager::Instance()->GetStorage(localUri))) {
 497                 Log("SHUTTLE", Form("StoreLocally - Cannot activate local %s storage", cdbType));
 498         } else {
 499                 result = AliCDBManager::Instance()->GetStorage(localUri)
 500                                         ->Put(object, id, metaData);
 501         }
 502
 503         if(!result) {
 504
 505                 Log(fCurrentDetector, Form("StoreLocally - Can't store object <%s>!", id.ToString().Data()));
 506         }
 507
 508         return result;
 509 }
 510
 511 //______________________________________________________________________________________________
 512 Bool_t AliShuttle::StoreOCDB()
 513 {
 514         //
 515         // Called when preprocessor ends successfully or when previous storage attempt failed (kStoreError status)
 516         // Calls underlying StoreOCDB(const char*) function twice, for OCDB and Reference storage.
 517         // Then calls StoreRefFilesToGrid to store reference files.
 518         //
 519
 520         if (fTestMode & kErrorGrid)
 521         {
 522                 Log("SHUTTLE", "StoreOCDB - In TESTMODE - Simulating error while storing in the Grid");
 523                 Log(fCurrentDetector, "StoreOCDB - In TESTMODE - Simulating error while storing in the Grid");
 524                 return kFALSE;
 525         }
 526
 527         Log("SHUTTLE","StoreOCDB - Storing OCDB data ...");
 528         Bool_t resultCDB = StoreOCDB(fgkMainCDB);
 529
 530         Log("SHUTTLE","StoreOCDB - Storing reference data ...");
 531         Bool_t resultRef = StoreOCDB(fgkMainRefStorage);
 532
 533         Log("SHUTTLE","StoreOCDB - Storing reference files ...");
 534         Bool_t resultRefFiles = CopyFilesToGrid("reference");
 535
 536         Bool_t resultMetadata = kTRUE;
 537         if(fCurrentDetector == "GRP")
 538         {
 539                 Log("StoreOCDB - SHUTTLE","Storing Run Metadata file ...");
 540                 resultMetadata = CopyFilesToGrid("metadata");
 541         }
 542
 543         return resultCDB && resultRef && resultRefFiles && resultMetadata;
 544 }
 545
 546 //______________________________________________________________________________________________
 547 Bool_t AliShuttle::StoreOCDB(const TString& gridURI)
 548 {
 549         //
 550         // Called by StoreOCDB(), performs actual storage to the main OCDB and reference storages (Grid)
 551         //
 552
 553         TObjArray* gridIds=0;
 554
 555         Bool_t result = kTRUE;
 556
 557         const char* type = 0;
 558         TString localURI;
 559         if(gridURI == fgkMainCDB) {
 560                 type = "OCDB";
 561                 localURI = fgkLocalCDB;
 562         } else if(gridURI == fgkMainRefStorage) {
 563                 type = "reference";
 564                 localURI = fgkLocalRefStorage;
 565         } else {
 566                 AliError(Form("Invalid storage URI: %s", gridURI.Data()));
 567                 return kFALSE;
 568         }
 569
 570         AliCDBManager* man = AliCDBManager::Instance();
 571
 572         AliCDBStorage *gridSto = man->GetStorage(gridURI);
 573         if(!gridSto) {
 574                 Log("SHUTTLE",
 575                         Form("StoreOCDB - cannot activate main %s storage", type));
 576                 return kFALSE;
 577         }
 578
 579         gridIds = gridSto->GetQueryCDBList();
 580
 581         // get objects previously stored in local CDB
 582         AliCDBStorage *localSto = man->GetStorage(localURI);
 583         if(!localSto) {
 584                 Log("SHUTTLE",
 585                         Form("StoreOCDB - cannot activate local %s storage", type));
 586                 return kFALSE;
 587         }
 588         AliCDBPath aPath(GetOfflineDetName(fCurrentDetector.Data()),"*","*");
 589         // Local objects were stored with current run as Grid version!
 590         TList* localEntries = localSto->GetAll(aPath.GetPath(), GetCurrentRun(), GetCurrentRun());
 591         localEntries->SetOwner(1);
 592
 593         // loop on local stored objects
 594         TIter localIter(localEntries);
 595         AliCDBEntry *aLocEntry = 0;
 596         while((aLocEntry = dynamic_cast<AliCDBEntry*> (localIter.Next()))){
 597                 aLocEntry->SetOwner(1);
 598                 AliCDBId aLocId = aLocEntry->GetId();
 599                 aLocEntry->SetVersion(-1);
 600                 aLocEntry->SetSubVersion(-1);
 601
 602                 // If local object is valid up to infinity we store it only if it is
 603                 // the first unprocessed run!
 604                 if (aLocId.GetLastRun() == AliCDBRunRange::Infinity() &&
 605                         !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
 606                 {
 607                         Log("SHUTTLE", Form("StoreOCDB - %s: object %s has validity infinite but "
 608                                                 "there are previous unprocessed runs!",
 609                                                 fCurrentDetector.Data(), aLocId.GetPath().Data()));
 610                         continue;
 611                 }
 612
 613                 // loop on Grid valid Id's
 614                 Bool_t store = kTRUE;
 615                 TIter gridIter(gridIds);
 616                 AliCDBId* aGridId = 0;
 617                 while((aGridId = dynamic_cast<AliCDBId*> (gridIter.Next()))){
 618                         if(aGridId->GetPath() != aLocId.GetPath()) continue;
 619                         // skip all objects valid up to infinity
 620                         if(aGridId->GetLastRun() == AliCDBRunRange::Infinity()) continue;
 621                         // if we get here, it means there's already some more recent object stored on Grid!
 622                         store = kFALSE;
 623                         break;
 624                 }
 625
 626                 // If we get here, the file can be stored!
 627                 Bool_t storeOk = gridSto->Put(aLocEntry);
 628                 if(!store || storeOk){
 629
 630                         if (!store)
 631                         {
 632                                 Log(fCurrentDetector.Data(),
 633                                         Form("StoreOCDB - A more recent object already exists in %s storage: <%s>",
 634                                                 type, aGridId->ToString().Data()));
 635                         } else {
 636                                 Log("SHUTTLE",
 637                                         Form("StoreOCDB - Object <%s> successfully put into %s storage",
 638                                                 aLocId.ToString().Data(), type));
 639                                 Log(fCurrentDetector.Data(),
 640                                         Form("StoreOCDB - Object <%s> successfully put into %s storage",
 641                                                 aLocId.ToString().Data(), type));
 642                         }
 643
 644                         // removing local filename...
 645                         TString filename;
 646                         localSto->IdToFilename(aLocId, filename);
 647                         Log("SHUTTLE", Form("StoreOCDB - Removing local file %s", filename.Data()));
 648                         RemoveFile(filename.Data());
 649                         continue;
 650                 } else  {
 651                         Log("SHUTTLE",
 652                                 Form("StoreOCDB - Grid %s storage of object <%s> failed",
 653                                         type, aLocId.ToString().Data()));
 654                         Log(fCurrentDetector.Data(),
 655                                 Form("StoreOCDB - Grid %s storage of object <%s> failed",
 656                                         type, aLocId.ToString().Data()));
 657                         result = kFALSE;
 658                 }
 659         }
 660         localEntries->Clear();
 661
 662         return result;
 663 }
 664
 665 //______________________________________________________________________________________________
 666 Bool_t AliShuttle::CleanReferenceStorage(const char* detector)
 667 {
 668         // clears the directory used to store reference files of a given subdetector
 669
 670         AliCDBManager* man = AliCDBManager::Instance();
 671         AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
 672         TString localBaseFolder = sto->GetBaseFolder();
 673
 674         TString targetDir = GetRefFilePrefix(localBaseFolder.Data(), detector);
 675
 676         Log("SHUTTLE", Form("CleanReferenceStorage - Cleaning %s", targetDir.Data()));
 677
 678         TString begin;
 679         begin.Form("%d_", GetCurrentRun());
 680
 681         TSystemDirectory* baseDir = new TSystemDirectory("/", targetDir);
 682         if (!baseDir)
 683                 return kTRUE;
 684
 685         TList* dirList = baseDir->GetListOfFiles();
 686         delete baseDir;
 687
 688         if (!dirList) return kTRUE;
 689
 690         if (dirList->GetEntries() < 3)
 691         {
 692                 delete dirList;
 693                 return kTRUE;
 694         }
 695
 696         Int_t nDirs = 0, nDel = 0;
 697         TIter dirIter(dirList);
 698         TSystemFile* entry = 0;
 699
 700         Bool_t success = kTRUE;
 701
 702         while ((entry = dynamic_cast<TSystemFile*> (dirIter.Next())))
 703         {
 704                 if (entry->IsDirectory())
 705                         continue;
 706
 707                 TString fileName(entry->GetName());
 708                 if (!fileName.BeginsWith(begin))
 709                         continue;
 710
 711                 nDirs++;
 712
 713                 // delete file
 714                 Int_t result = gSystem->Unlink(fileName.Data());
 715
 716                 if (result)
 717                 {
 718                         Log("SHUTTLE", Form("CleanReferenceStorage - Could not delete file %s!", fileName.Data()));
 719                         success = kFALSE;
 720                 } else {
 721                         nDel++;
 722                 }
 723         }
 724
 725         if(nDirs > 0)
 726                 Log("SHUTTLE", Form("CleanReferenceStorage - %d (over %d) reference files in folder %s were deleted.",
 727                         nDel, nDirs, targetDir.Data()));
 728
 729
 730         delete dirList;
 731         return success;
 732
 733
 734
 735
 736
 737
 738   Int_t result = gSystem->GetPathInfo(targetDir, 0, (Long64_t*) 0, 0, 0);
 739   if (result == 0)
 740   {
 741     // delete directory
 742     result = gSystem->Exec(Form("rm -rf %s", targetDir.Data()));
 743     if (result != 0)
 744     {
 745       Log("SHUTTLE", Form("CleanReferenceStorage - Could not clean directory %s", targetDir.Data()));
 746       return kFALSE;
 747     }
 748   }
 749
 750   result = gSystem->mkdir(targetDir, kTRUE);
 751   if (result != 0)
 752   {
 753     Log("SHUTTLE", Form("CleanReferenceStorage - Error creating base directory %s", targetDir.Data()));
 754     return kFALSE;
 755   }
 756
 757   return kTRUE;
 758 }
 759
 760 //______________________________________________________________________________________________
 761 Bool_t AliShuttle::StoreReferenceFile(const char* detector, const char* localFile, const char* gridFileName)
 762 {
 763         //
 764         // Stores reference file directly (without opening it). This function stores the file locally.
 765         //
 766         // The file is stored under the following location:
 767         // <base folder of local reference storage>/<DET>/<RUN#>_<gridFileName>
 768         // where <gridFileName> is the second parameter given to the function
 769         //
 770
 771         if (fTestMode & kErrorStorage)
 772         {
 773                 Log(fCurrentDetector, "StoreReferenceFile - In TESTMODE - Simulating error while storing locally");
 774                 return kFALSE;
 775         }
 776
 777         AliCDBManager* man = AliCDBManager::Instance();
 778         AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
 779
 780         TString localBaseFolder = sto->GetBaseFolder();
 781
 782         TString target = GetRefFilePrefix(localBaseFolder.Data(), detector);
 783         target.Append(Form("/%d_%s", GetCurrentRun(), gridFileName));
 784
 785         return CopyFileLocally(localFile, target);
 786 }
 787
 788 //______________________________________________________________________________________________
 789 Bool_t AliShuttle::StoreRunMetadataFile(const char* localFile, const char* gridFileName)
 790 {
 791         //
 792         // Stores Run metadata file to the Grid, in the run folder
 793         //
 794         // Only GRP can call this function.
 795
 796         if (fTestMode & kErrorStorage)
 797         {
 798                 Log(fCurrentDetector, "StoreRunMetaDataFile - In TESTMODE - Simulating error while storing locally");
 799                 return kFALSE;
 800         }
 801
 802         AliCDBManager* man = AliCDBManager::Instance();
 803         AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
 804
 805         TString localBaseFolder = sto->GetBaseFolder();
 806
 807         // Build Run level folder
 808         // folder = /alice/data/year/lhcPeriod/runNb/Raw
 809
 810
 811         TString lhcPeriod = GetLHCPeriod();
 812         if (lhcPeriod.Length() == 0)
 813         {
 814                 Log("SHUTTLE","StoreRunMetaDataFile - LHCPeriod not found in logbook!");
 815                 return 0;
 816         }
 817
 818         TString target = Form("%s/GRP/RunMetadata/alice/data/%d/%s/%09d/Raw/%s",
 819                                 localBaseFolder.Data(), GetCurrentYear(),
 820                                 lhcPeriod.Data(), GetCurrentRun(), gridFileName);
 821
 822         return CopyFileLocally(localFile, target);
 823 }
 824
 825 //______________________________________________________________________________________________
 826 Bool_t AliShuttle::CopyFileLocally(const char* localFile, const TString& target)
 827 {
 828         //
 829         // Stores file locally. Called by StoreReferenceFile and StoreRunMetadataFile
 830         // Files are temporarily stored in the local reference storage. When the preprocessor
 831         // finishes, the Shuttle calls CopyFilesToGrid to transfer the files to AliEn
 832         // (in reference or run level folders)
 833         //
 834
 835         TString targetDir(target(0, target.Last('/')));
 836
 837         //try to open base dir folder, if it does not exist
 838         void* dir = gSystem->OpenDirectory(targetDir.Data());
 839         if (dir == NULL) {
 840                 if (gSystem->mkdir(targetDir.Data(), kTRUE)) {
 841                         Log("SHUTTLE", Form("StoreFileLocally - Can't open directory <%s>", targetDir.Data()));
 842                         return kFALSE;
 843                 }
 844
 845         } else {
 846                 gSystem->FreeDirectory(dir);
 847         }
 848
 849         Int_t result = 0;
 850
 851         result = gSystem->GetPathInfo(localFile, 0, (Long64_t*) 0, 0, 0);
 852         if (result)
 853         {
 854                 Log("SHUTTLE", Form("StoreFileLocally - %s does not exist", localFile));
 855                 return kFALSE;
 856         }
 857
 858         result = gSystem->GetPathInfo(target, 0, (Long64_t*) 0, 0, 0);
 859         if (!result)
 860         {
 861                 Log("SHUTTLE", Form("StoreFileLocally - target file %s already exist, removing...", target.Data()));
 862                 if (gSystem->Unlink(target.Data()))
 863                 {
 864                         Log("SHUTTLE", Form("StoreFileLocally - Could not remove existing target file %s!", target.Data()));
 865                         return kFALSE;
 866                 }
 867         }
 868
 869         result = gSystem->CopyFile(localFile, target);
 870
 871         if (result == 0)
 872         {
 873                 Log("SHUTTLE", Form("StoreFileLocally - File %s stored locally to %s", localFile, target.Data()));
 874                 return kTRUE;
 875         }
 876         else
 877         {
 878                 Log("SHUTTLE", Form("StoreFileLocally - Could not store file %s to %s! Error code = %d",
 879                                 localFile, target.Data(), result));
 880                 return kFALSE;
 881         }
 882
 883
 884
 885 }
 886
 887 //______________________________________________________________________________________________
 888 Bool_t AliShuttle::CopyFilesToGrid(const char* type)
 889 {
 890         //
 891         // Transfers local files to the Grid. Local files can be reference files
 892         // or run metadata file (from GRP only).
 893         //
 894         // According to the type (ref, metadata) the files are stored under the following location:
 895         // ref --> <base folder of reference storage>/<DET>/<RUN#>_<gridFileName>
 896         // metadata --> <run data folder>/<MetadataFileName>
 897         //
 898
 899         AliCDBManager* man = AliCDBManager::Instance();
 900         AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
 901         if (!sto)
 902                 return kFALSE;
 903         TString localBaseFolder = sto->GetBaseFolder();
 904
 905         TString dir;
 906         TString alienDir;
 907         TString begin;
 908
 909         if (strcmp(type, "reference") == 0)
 910         {
 911                 dir = GetRefFilePrefix(localBaseFolder.Data(), fCurrentDetector.Data());
 912                 AliCDBStorage* gridSto = man->GetStorage(fgkMainRefStorage);
 913                 if (!gridSto)
 914                         return kFALSE;
 915                 TString gridBaseFolder = gridSto->GetBaseFolder();
 916                 alienDir = GetRefFilePrefix(gridBaseFolder.Data(), fCurrentDetector.Data());
 917                 begin = Form("%d_", GetCurrentRun());
 918         }
 919         else if (strcmp(type, "metadata") == 0)
 920         {
 921
 922                 TString lhcPeriod = GetLHCPeriod();
 923
 924                 if (lhcPeriod.Length() == 0)
 925                 {
 926                         Log("SHUTTLE","CopyFilesToGrid - LHCPeriod not found in logbook!");
 927                         return 0;
 928                 }
 929
 930                 dir = Form("%s/GRP/RunMetadata/alice/data/%d/%s/%09d/Raw",
 931                                 localBaseFolder.Data(), GetCurrentYear(),
 932                                 lhcPeriod.Data(), GetCurrentRun());
 933                 alienDir = dir(dir.Index("/alice/data/"), dir.Length());
 934
 935                 begin = "";
 936         }
 937         else
 938         {
 939                 Log("SHUTTLE", "CopyFilesToGrid - Unexpected: type label must be reference or metadata!");
 940                 return kFALSE;
 941         }
 942
 943         TSystemDirectory* baseDir = new TSystemDirectory("/", dir);
 944         if (!baseDir)
 945                 return kTRUE;
 946
 947         TList* dirList = baseDir->GetListOfFiles();
 948         delete baseDir;
 949
 950         if (!dirList) return kTRUE;
 951
 952         if (dirList->GetEntries() < 3)
 953         {
 954                 delete dirList;
 955                 return kTRUE;
 956         }
 957
 958         if (!gGrid)
 959         {
 960                 Log("SHUTTLE", "CopyFilesToGrid - Connection to Grid failed: Cannot continue!");
 961                 delete dirList;
 962                 return kFALSE;
 963         }
 964
 965         Int_t nDirs = 0, nTransfer = 0;
 966         TIter dirIter(dirList);
 967         TSystemFile* entry = 0;
 968
 969         Bool_t success = kTRUE;
 970         Bool_t first = kTRUE;
 971
 972         while ((entry = dynamic_cast<TSystemFile*> (dirIter.Next())))
 973         {
 974                 if (entry->IsDirectory())
 975                         continue;
 976
 977                 TString fileName(entry->GetName());
 978                 if (!fileName.BeginsWith(begin))
 979                         continue;
 980
 981                 nDirs++;
 982
 983                 if (first)
 984                 {
 985                         first = kFALSE;
 986                         // check that folder exists, otherwise create it
 987                         TGridResult* result = gGrid->Ls(alienDir.Data(), "a");
 988
 989                         if (!result)
 990                         {
 991                                 delete dirList;
 992                                 return kFALSE;
 993                         }
 994
 995                         if (!result->GetFileName(1)) // TODO: It looks like element 0 is always 0!!
 996                         {
 997                                 // TODO It does not work currently! Bug in TAliEn::Mkdir
 998                                 // TODO Manually fixed in local root v5-16-00
 999                                 if (!gGrid->Mkdir(alienDir.Data(),"-p",0))
1000                                 {
1001                                         Log("SHUTTLE", Form("CopyFilesToGrid - Cannot create directory %s",
1002                                                         alienDir.Data()));
1003                                         delete dirList;
1004                                         return kFALSE;
1005                                 } else {
1006                                         Log("SHUTTLE",Form("CopyFilesToGrid - Folder %s created", alienDir.Data()));
1007                                 }
1008
1009                         } else {
1010                                         Log("SHUTTLE",Form("CopyFilesToGrid - Folder %s found", alienDir.Data()));
1011                         }
1012                 }
1013
1014                 TString fullLocalPath;
1015                 fullLocalPath.Form("%s/%s", dir.Data(), fileName.Data());
1016
1017                 TString fullGridPath;
1018                 fullGridPath.Form("alien://%s/%s", alienDir.Data(), fileName.Data());
1019
1020                 Bool_t result = TFile::Cp(fullLocalPath, fullGridPath);
1021
1022                 if (result)
1023                 {
1024                         Log("SHUTTLE", Form("CopyFilesToGrid - Copying local file %s to %s succeeded!",
1025                                                 fullLocalPath.Data(), fullGridPath.Data()));
1026                         RemoveFile(fullLocalPath);
1027                         nTransfer++;
1028                 }
1029                 else
1030                 {
1031                         Log("SHUTTLE", Form("CopyFilesToGrid - Copying local file %s to %s FAILED!",
1032                                                 fullLocalPath.Data(), fullGridPath.Data()));
1033                         success = kFALSE;
1034                 }
1035         }
1036
1037         Log("SHUTTLE", Form("CopyFilesToGrid - %d (over %d) files in folder %s copied to Grid.",
1038                                                 nTransfer, nDirs, dir.Data()));
1039
1040
1041         delete dirList;
1042         return success;
1043 }
1044
1045 //______________________________________________________________________________________________
1046 const char* AliShuttle::GetRefFilePrefix(const char* base, const char* detector)
1047 {
1048         //
1049         // Get folder name of reference files
1050         //
1051
1052         TString offDetStr(GetOfflineDetName(detector));
1053         TString dir;
1054         if (offDetStr == "ITS" || offDetStr == "MUON" || offDetStr == "PHOS")
1055         {
1056                 dir.Form("%s/%s/%s", base, offDetStr.Data(), detector);
1057         } else {
1058                 dir.Form("%s/%s", base, offDetStr.Data());
1059         }
1060
1061         return dir.Data();
1062
1063
1064 }
1065
1066 //______________________________________________________________________________________________
1067 void AliShuttle::CleanLocalStorage(const TString& uri)
1068 {
1069         //
1070         // Called in case the preprocessor is declared failed. Remove remaining objects from the local storages.
1071         //
1072
1073         const char* type = 0;
1074         if(uri == fgkLocalCDB) {
1075                 type = "OCDB";
1076         } else if(uri == fgkLocalRefStorage) {
1077                 type = "Reference";
1078         } else {
1079                 AliError(Form("Invalid storage URI: %s", uri.Data()));
1080                 return;
1081         }
1082
1083         AliCDBManager* man = AliCDBManager::Instance();
1084
1085         // open local storage
1086         AliCDBStorage *localSto = man->GetStorage(uri);
1087         if(!localSto) {
1088                 Log("SHUTTLE",
1089                         Form("CleanLocalStorage - cannot activate local %s storage", type));
1090                 return;
1091         }
1092
1093         TString filename(Form("%s/%s/*/Run*_v%d_s*.root",
1094                 localSto->GetBaseFolder().Data(), GetOfflineDetName(fCurrentDetector.Data()), GetCurrentRun()));
1095
1096         AliDebug(2, Form("filename = %s", filename.Data()));
1097
1098         Log("SHUTTLE", Form("Removing remaining local files for run %d and detector %s ...",
1099                 GetCurrentRun(), fCurrentDetector.Data()));
1100
1101         RemoveFile(filename.Data());
1102
1103 }
1104
1105 //______________________________________________________________________________________________
1106 void AliShuttle::RemoveFile(const char* filename)
1107 {
1108         //
1109         // removes local file
1110         //
1111
1112         TString command(Form("rm -f %s", filename));
1113
1114         Int_t result = gSystem->Exec(command.Data());
1115         if(result != 0)
1116         {
1117                 Log("SHUTTLE", Form("RemoveFile - %s: Cannot remove file %s!",
1118                         fCurrentDetector.Data(), filename));
1119         }
1120 }
1121
1122 //______________________________________________________________________________________________
1123 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
1124 {
1125         //
1126         // Reads the AliShuttleStatus from the CDB
1127         //
1128
1129         if (fStatusEntry){
1130                 delete fStatusEntry;
1131                 fStatusEntry = 0;
1132         }
1133
1134         fStatusEntry = AliCDBManager::Instance()->GetStorage(GetLocalCDB())
1135                 ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), GetCurrentRun());
1136
1137         if (!fStatusEntry) return 0;
1138         fStatusEntry->SetOwner(1);
1139
1140         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
1141         if (!status) {
1142                 AliError("Invalid object stored to CDB!");
1143                 return 0;
1144         }
1145
1146         return status;
1147 }
1148
1149 //______________________________________________________________________________________________
1150 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
1151 {
1152         //
1153         // writes the status for one subdetector
1154         //
1155
1156         if (fStatusEntry){
1157                 delete fStatusEntry;
1158                 fStatusEntry = 0;
1159         }
1160
1161         Int_t run = GetCurrentRun();
1162
1163         AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), run, run);
1164
1165         fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
1166         fStatusEntry->SetOwner(1);
1167
1168         UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
1169
1170         if (!result) {
1171                 Log("SHUTTLE", Form("WriteShuttleStatus - Failed for %s, run %d",
1172                                                 fCurrentDetector.Data(), run));
1173                 return kFALSE;
1174         }
1175
1176         SendMLInfo();
1177
1178         return kTRUE;
1179 }
1180
1181 //______________________________________________________________________________________________
1182 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
1183 {
1184         //
1185         // changes the AliShuttleStatus for the given detector and run to the given status
1186         //
1187
1188         if (!fStatusEntry){
1189                 AliError("UNEXPECTED: fStatusEntry empty");
1190                 return;
1191         }
1192
1193         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
1194
1195         if (!status){
1196                 Log("SHUTTLE", "UpdateShuttleStatus - UNEXPECTED: status could not be read from current CDB entry");
1197                 return;
1198         }
1199
1200         TString actionStr = Form("UpdateShuttleStatus - %s: Changing state from %s to %s",
1201                                 fCurrentDetector.Data(),
1202                                 status->GetStatusName(),
1203                                 status->GetStatusName(newStatus));
1204         Log("SHUTTLE", actionStr);
1205         SetLastAction(actionStr);
1206
1207         status->SetStatus(newStatus);
1208         if (increaseCount) status->IncreaseCount();
1209
1210         AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
1211
1212         SendMLInfo();
1213 }
1214
1215 //______________________________________________________________________________________________
1216 void AliShuttle::SendMLInfo()
1217 {
1218         //
1219         // sends ML information about the current status of the current detector being processed
1220         //
1221
1222         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
1223
1224         if (!status){
1225                 Log("SHUTTLE", "SendMLInfo - UNEXPECTED: status could not be read from current CDB entry");
1226                 return;
1227         }
1228
1229         TMonaLisaText  mlStatus(Form("%s_status", fCurrentDetector.Data()), status->GetStatusName());
1230         TMonaLisaValue mlRetryCount(Form("%s_count", fCurrentDetector.Data()), status->GetCount());
1231
1232         TList mlList;
1233         mlList.Add(&mlStatus);
1234         mlList.Add(&mlRetryCount);
1235
1236         fMonaLisa->SendParameters(&mlList);
1237 }
1238
1239 //______________________________________________________________________________________________
1240 Bool_t AliShuttle::ContinueProcessing()
1241 {
1242         // this function reads the AliShuttleStatus information from CDB and
1243         // checks if the processing should be continued
1244         // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
1245
1246         if (!fConfig->HostProcessDetector(fCurrentDetector)) return kFALSE;
1247
1248         AliPreprocessor* aPreprocessor =
1249                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
1250         if (!aPreprocessor)
1251         {
1252                 Log("SHUTTLE", Form("ContinueProcessing - %s: no preprocessor registered", fCurrentDetector.Data()));
1253                 return kFALSE;
1254         }
1255
1256         AliShuttleLogbookEntry::Status entryStatus =
1257                 fLogbookEntry->GetDetectorStatus(fCurrentDetector);
1258
1259         if(entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
1260                 Log("SHUTTLE", Form("ContinueProcessing - %s is %s",
1261                                 fCurrentDetector.Data(),
1262                                 fLogbookEntry->GetDetectorStatusName(entryStatus)));
1263                 return kFALSE;
1264         }
1265
1266         // if we get here, according to Shuttle logbook subdetector is in UNPROCESSED state
1267
1268         // check if current run is first unprocessed run for current detector
1269         if (fConfig->StrictRunOrder(fCurrentDetector) &&
1270                 !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
1271         {
1272                 if (fTestMode == kNone)
1273                 {
1274                         Log("SHUTTLE", Form("ContinueProcessing - %s requires strict run ordering"
1275                                         " but this is not the first unprocessed run!"));
1276                         return kFALSE;
1277                 }
1278                 else
1279                 {
1280                         Log("SHUTTLE", Form("ContinueProcessing - In TESTMODE - "
1281                                         "Although %s requires strict run ordering "
1282                                         "and this is not the first unprocessed run, "
1283                                         "the SHUTTLE continues"));
1284                 }
1285         }
1286
1287         AliShuttleStatus* status = ReadShuttleStatus();
1288         if (!status) {
1289                 // first time
1290                 Log("SHUTTLE", Form("ContinueProcessing - %s: Processing first time",
1291                                 fCurrentDetector.Data()));
1292                 status = new AliShuttleStatus(AliShuttleStatus::kStarted);
1293                 return WriteShuttleStatus(status);
1294         }
1295
1296         // The following two cases shouldn't happen if Shuttle Logbook was correctly updated.
1297         // If it happens it may mean Logbook updating failed... let's do it now!
1298         if (status->GetStatus() == AliShuttleStatus::kDone ||
1299             status->GetStatus() == AliShuttleStatus::kFailed){
1300                 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s. Updating Shuttle Logbook",
1301                                         fCurrentDetector.Data(),
1302                                         status->GetStatusName(status->GetStatus())));
1303                 UpdateShuttleLogbook(fCurrentDetector.Data(),
1304                                         status->GetStatusName(status->GetStatus()));
1305                 return kFALSE;
1306         }
1307
1308         if (status->GetStatus() == AliShuttleStatus::kStoreError) {
1309                 Log("SHUTTLE",
1310                         Form("ContinueProcessing - %s: Grid storage of one or more "
1311                                 "objects failed. Trying again now",
1312                                 fCurrentDetector.Data()));
1313                 UpdateShuttleStatus(AliShuttleStatus::kStoreStarted);
1314                 if (StoreOCDB()){
1315                         Log("SHUTTLE", Form("ContinueProcessing - %s: all objects "
1316                                 "successfully stored into main storage",
1317                                 fCurrentDetector.Data()));
1318                         UpdateShuttleStatus(AliShuttleStatus::kDone);
1319                         UpdateShuttleLogbook(fCurrentDetector.Data(), "DONE");
1320                 } else {
1321                         Log("SHUTTLE",
1322                                 Form("ContinueProcessing - %s: Grid storage failed again",
1323                                         fCurrentDetector.Data()));
1324                         UpdateShuttleStatus(AliShuttleStatus::kStoreError);
1325                 }
1326                 return kFALSE;
1327         }
1328
1329         // if we get here, there is a restart
1330         Bool_t cont = kFALSE;
1331
1332         // abort conditions
1333         if (status->GetCount() >= fConfig->GetMaxRetries()) {
1334                 Log("SHUTTLE", Form("ContinueProcessing - %s failed %d times in status %s - "
1335                                 "Updating Shuttle Logbook", fCurrentDetector.Data(),
1336                                 status->GetCount(), status->GetStatusName()));
1337                 UpdateShuttleLogbook(fCurrentDetector.Data(), "FAILED");
1338                 UpdateShuttleStatus(AliShuttleStatus::kFailed);
1339
1340                 // there may still be objects in local OCDB and reference storage
1341                 // and FXS databases may be not updated: do it now!
1342
1343                 // TODO Currently disabled, we want to keep files in case of failure!
1344                 // CleanLocalStorage(fgkLocalCDB);
1345                 // CleanLocalStorage(fgkLocalRefStorage);
1346                 // UpdateTableFailCase();
1347
1348                 // Send mail to detector expert!
1349                 Log("SHUTTLE", Form("ContinueProcessing - Sending mail to %s expert...",
1350                                         fCurrentDetector.Data()));
1351                 if (!SendMail())
1352                         Log("SHUTTLE", Form("ContinueProcessing - Could not send mail to %s expert",
1353                                         fCurrentDetector.Data()));
1354
1355         } else {
1356                 Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. "
1357                                 "Aborted before with %s. Retry number %d.", fCurrentDetector.Data(),
1358                                 status->GetStatusName(), status->GetCount()));
1359                 Bool_t increaseCount = kTRUE;
1360                 if (status->GetStatus() == AliShuttleStatus::kDCSError ||
1361                         status->GetStatus() == AliShuttleStatus::kDCSStarted)
1362                                 increaseCount = kFALSE;
1363
1364                 UpdateShuttleStatus(AliShuttleStatus::kStarted, increaseCount);
1365                 cont = kTRUE;
1366         }
1367
1368         return cont;
1369 }
1370
1371 //______________________________________________________________________________________________
1372 Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
1373 {
1374         //
1375         // Makes data retrieval for all detectors in the configuration.
1376         // entry: Shuttle logbook entry, contains run paramenters and status of detectors
1377         // (Unprocessed, Inactive, Failed or Done).
1378         // Returns kFALSE in case of error occured and kTRUE otherwise
1379         //
1380
1381         if (!entry) return kFALSE;
1382
1383         fLogbookEntry = entry;
1384
1385         Log("SHUTTLE", Form("\t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^*",
1386                                         GetCurrentRun()));
1387
1388         // create ML instance that monitors this run
1389         fMonaLisa = new TMonaLisaWriter(Form("%d", GetCurrentRun()), "SHUTTLE", "aliendb1.cern.ch");
1390         // disable monitoring of other parameters that come e.g. from TFile
1391         gMonitoringWriter = 0;
1392
1393         // Send the information to ML
1394         TMonaLisaText  mlStatus("SHUTTLE_status", "Processing");
1395         TMonaLisaText  mlRunType("SHUTTLE_runtype", Form("%s (%s)", entry->GetRunType(), entry->GetRunParameter("log")));
1396
1397         TList mlList;
1398         mlList.Add(&mlStatus);
1399         mlList.Add(&mlRunType);
1400
1401         fMonaLisa->SendParameters(&mlList);
1402
1403         if (fLogbookEntry->IsDone())
1404         {
1405                 Log("SHUTTLE","Process - Shuttle is already DONE. Updating logbook");
1406                 UpdateShuttleLogbook("shuttle_done");
1407                 fLogbookEntry = 0;
1408                 return kTRUE;
1409         }
1410
1411         // read test mode if flag is set
1412         if (fReadTestMode)
1413         {
1414                 fTestMode = kNone;
1415                 TString logEntry(entry->GetRunParameter("log"));
1416                 //printf("log entry = %s\n", logEntry.Data());
1417                 TString searchStr("Testmode: ");
1418                 Int_t pos = logEntry.Index(searchStr.Data());
1419                 //printf("%d\n", pos);
1420                 if (pos >= 0)
1421                 {
1422                         TSubString subStr = logEntry(pos + searchStr.Length(), logEntry.Length());
1423                         //printf("%s\n", subStr.String().Data());
1424                         TString newStr(subStr.Data());
1425                         TObjArray* token = newStr.Tokenize(' ');
1426                         if (token)
1427                         {
1428                                 //token->Print();
1429                                 TObjString* tmpStr = dynamic_cast<TObjString*> (token->First());
1430                                 if (tmpStr)
1431                                 {
1432                                         Int_t testMode = tmpStr->String().Atoi();
1433                                         if (testMode > 0)
1434                                         {
1435                                                 Log("SHUTTLE", Form("Process - Enabling test mode %d", testMode));
1436                                                 SetTestMode((TestMode) testMode);
1437                                         }
1438                                 }
1439                                 delete token;
1440                         }
1441                 }
1442         }
1443
1444         fLogbookEntry->Print("all");
1445
1446         // Initialization
1447         Bool_t hasError = kFALSE;
1448
1449         // Set the CDB and Reference folders according to the year and LHC period
1450         TString lhcPeriod(GetLHCPeriod());
1451         if (lhcPeriod.Length() == 0)
1452         {
1453                 Log("SHUTTLE","StoreRunMetaDataFile - LHCPeriod not found in logbook!");
1454                 return 0;
1455         }
1456
1457         if (fgkMainCDB.Length() == 0)
1458                 fgkMainCDB = Form("alien://folder=/alice/data/%d/%s/OCDB?user=alidaq?cacheFold=/tmp/OCDBCache",
1459                                         GetCurrentYear(), lhcPeriod.Data());
1460
1461         if (fgkMainRefStorage.Length() == 0)
1462                 fgkMainRefStorage = Form("alien://folder=/alice/data/%d/%s/Reference?user=alidaq?cacheFold=/tmp/OCDBCache",
1463                                         GetCurrentYear(), lhcPeriod.Data());
1464
1465         AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
1466         if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
1467         AliCDBStorage *mainRefSto = AliCDBManager::Instance()->GetStorage(fgkMainRefStorage);
1468         if(mainRefSto) mainRefSto->QueryCDB(GetCurrentRun());
1469
1470         // Loop on detectors in the configuration
1471         TIter iter(fConfig->GetDetectors());
1472         TObjString* aDetector = 0;
1473
1474         while ((aDetector = (TObjString*) iter.Next()))
1475         {
1476                 fCurrentDetector = aDetector->String();
1477
1478                 if (ContinueProcessing() == kFALSE) continue;
1479
1480                 Log("SHUTTLE", Form("\t\t\t****** run %d - %s: START  ******",
1481                                                 GetCurrentRun(), aDetector->GetName()));
1482
1483                 for(Int_t iSys=0;iSys<3;iSys++) fFXSCalled[iSys]=kFALSE;
1484
1485                 Log(fCurrentDetector.Data(), "Process - Starting processing");
1486
1487                 Int_t pid = fork();
1488
1489                 if (pid < 0)
1490                 {
1491                         Log("SHUTTLE", "Process - ERROR: Forking failed");
1492                 }
1493                 else if (pid > 0)
1494                 {
1495                         // parent
1496                         Log("SHUTTLE", Form("Process - In parent process of %d - %s: Starting monitoring",
1497                                                         GetCurrentRun(), aDetector->GetName()));
1498
1499                         Long_t begin = time(0);
1500
1501                         int status; // to be used with waitpid, on purpose an int (not Int_t)!
1502                         while (waitpid(pid, &status, WNOHANG) == 0)
1503                         {
1504                                 Long_t expiredTime = time(0) - begin;
1505
1506                                 if (expiredTime > fConfig->GetPPTimeOut())
1507                                 {
1508                                         TString tmp;
1509                                         tmp.Form("Process - Process of %s time out. "
1510                                                         "Run time: %d seconds. Killing...",
1511                                                         fCurrentDetector.Data(), expiredTime);
1512                                         Log("SHUTTLE", tmp);
1513                                         Log(fCurrentDetector, tmp);
1514
1515                                         kill(pid, 9);
1516
1517                                         UpdateShuttleStatus(AliShuttleStatus::kPPTimeOut);
1518                                         hasError = kTRUE;
1519
1520                                         gSystem->Sleep(1000);
1521                                 }
1522                                 else
1523                                 {
1524                                         gSystem->Sleep(1000);
1525
1526                                         TString checkStr;
1527                                         checkStr.Form("ps -o vsize --pid %d | tail -n 1", pid);
1528                                         FILE* pipe = gSystem->OpenPipe(checkStr, "r");
1529                                         if (!pipe)
1530                                         {
1531                                                 Log("SHUTTLE", Form("Process - Error: "
1532                                                         "Could not open pipe to %s", checkStr.Data()));
1533                                                 continue;
1534                                         }
1535
1536                                         char buffer[100];
1537                                         if (!fgets(buffer, 100, pipe))
1538                                         {
1539                                                 Log("SHUTTLE", "Process - Error: ps did not return anything");
1540                                                 gSystem->ClosePipe(pipe);
1541                                                 continue;
1542                                         }
1543                                         gSystem->ClosePipe(pipe);
1544
1545                                         //Log("SHUTTLE", Form("ps returned %s", buffer));
1546
1547                                         Int_t mem = 0;
1548                                         if ((sscanf(buffer, "%d\n", &mem) != 1) || !mem)
1549                                         {
1550                                                 Log("SHUTTLE", "Process - Error: Could not parse output of ps");
1551                                                 continue;
1552                                         }
1553
1554                                         if (expiredTime % 60 == 0)
1555                                                 Log("SHUTTLE", Form("Process - %s: Checking process. "
1556                                                         "Run time: %d seconds - Memory consumption: %d KB",
1557                                                         fCurrentDetector.Data(), expiredTime, mem));
1558
1559                                         if (mem > fConfig->GetPPMaxMem())
1560                                         {
1561                                                 TString tmp;
1562                                                 tmp.Form("Process - Process exceeds maximum allowed memory "
1563                                                         "(%d KB > %d KB). Killing...",
1564                                                         mem, fConfig->GetPPMaxMem());
1565                                                 Log("SHUTTLE", tmp);
1566                                                 Log(fCurrentDetector, tmp);
1567
1568                                                 kill(pid, 9);
1569
1570                                                 UpdateShuttleStatus(AliShuttleStatus::kPPOutOfMemory);
1571                                                 hasError = kTRUE;
1572
1573                                                 gSystem->Sleep(1000);
1574                                         }
1575                                 }
1576                         }
1577
1578                         Log("SHUTTLE", Form("Process - In parent process of %d - %s: Client has terminated.",
1579                                                                 GetCurrentRun(), aDetector->GetName()));
1580
1581                         if (WIFEXITED(status))
1582                         {
1583                                 Int_t returnCode = WEXITSTATUS(status);
1584
1585                                 Log("SHUTTLE", Form("Process - %s: the return code is %d", fCurrentDetector.Data(),
1586                                                                                 returnCode));
1587
1588                                 if (returnCode == 0) hasError = kTRUE;
1589                         }
1590                 }
1591                 else if (pid == 0)
1592                 {
1593                         // client
1594                         Log("SHUTTLE", Form("Process - In client process of %d - %s", GetCurrentRun(),
1595                                 aDetector->GetName()));
1596
1597                         Log("SHUTTLE", Form("Process - Redirecting output to %s log",fCurrentDetector.Data()));
1598
1599                         if ((freopen(GetLogFileName(fCurrentDetector), "a", stdout)) == 0)
1600                         {
1601                                 Log("SHUTTLE", "Process - Could not freopen stdout");
1602                         }
1603                         else
1604                         {
1605                                 fOutputRedirected = kTRUE;
1606                                 if ((dup2(fileno(stdout), fileno(stderr))) < 0)
1607                                         Log("SHUTTLE", "Process - Could not redirect stderr");
1608
1609                         }
1610
1611                         TString wd = gSystem->WorkingDirectory();
1612                         TString tmpDir = Form("%s/%s_%d_process", GetShuttleTempDir(),
1613                                 fCurrentDetector.Data(), GetCurrentRun());
1614
1615                         Int_t result = gSystem->GetPathInfo(tmpDir.Data(), 0, (Long64_t*) 0, 0, 0);
1616                         if (!result) // temp dir already exists!
1617                         {
1618                                 Log(fCurrentDetector.Data(),
1619                                         Form("Process - %s dir already exists! Removing...", tmpDir.Data()));
1620                                 gSystem->Exec(Form("rm -rf %s",tmpDir.Data()));
1621                         }
1622
1623                         if (gSystem->mkdir(tmpDir.Data(), 1))
1624                         {
1625                                 Log(fCurrentDetector.Data(), "Process - could not make temp directory!!");
1626                                 gSystem->Exit(1);
1627                         }
1628
1629                         if (!gSystem->ChangeDirectory(tmpDir.Data()))
1630                         {
1631                                 Log(fCurrentDetector.Data(), "Process - could not change directory!!");
1632                                 gSystem->Exit(1);
1633                         }
1634
1635                         Bool_t success = ProcessCurrentDetector();
1636
1637                         gSystem->ChangeDirectory(wd.Data());
1638
1639                         gSystem->Exec(Form("rm -rf %s",tmpDir.Data()));
1640
1641                         if (success) // Preprocessor finished successfully!
1642                         {
1643                                 // Update time_processed field in FXS DB
1644                                 if (UpdateTable() == kFALSE)
1645                                         Log("SHUTTLE", Form("Process - %s: Could not update FXS databases!",
1646                                                         fCurrentDetector.Data()));
1647
1648                                 // Transfer the data from local storage to main storage (Grid)
1649                                 UpdateShuttleStatus(AliShuttleStatus::kStoreStarted);
1650                                 if (StoreOCDB() == kFALSE)
1651                                 {
1652                                         Log("SHUTTLE",
1653                                                 Form("\t\t\t****** run %d - %s: STORAGE ERROR ******",
1654                                                         GetCurrentRun(), aDetector->GetName()));
1655                                         UpdateShuttleStatus(AliShuttleStatus::kStoreError);
1656                                         success = kFALSE;
1657                                 } else {
1658                                         Log("SHUTTLE",
1659                                                 Form("\t\t\t****** run %d - %s: DONE ******",
1660                                                         GetCurrentRun(), aDetector->GetName()));
1661                                         UpdateShuttleStatus(AliShuttleStatus::kDone);
1662                                         UpdateShuttleLogbook(fCurrentDetector, "DONE");
1663                                 }
1664                         } else
1665                         {
1666                                 Log("SHUTTLE",
1667                                         Form("\t\t\t****** run %d - %s: PP ERROR ******",
1668                                                 GetCurrentRun(), aDetector->GetName()));
1669                         }
1670
1671                         for (UInt_t iSys=0; iSys<3; iSys++)
1672                         {
1673                                 if (fFXSCalled[iSys]) fFXSlist[iSys].Clear();
1674                         }
1675
1676                         Log("SHUTTLE", Form("Process - Client process of %d - %s is exiting now with %d.",
1677                                                         GetCurrentRun(), aDetector->GetName(), success));
1678
1679                         // the client exits here
1680                         gSystem->Exit(success);
1681
1682                         AliError("We should never get here!!!");
1683                 }
1684         }
1685
1686         Log("SHUTTLE", Form("\t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^*",
1687                                                         GetCurrentRun()));
1688
1689         //check if shuttle is done for this run, if so update logbook
1690         TObjArray checkEntryArray;
1691         checkEntryArray.SetOwner(1);
1692         TString whereClause = Form("where run=%d", GetCurrentRun());
1693         if (!QueryShuttleLogbook(whereClause.Data(), checkEntryArray) || checkEntryArray.GetEntries() == 0) {
1694                 Log("SHUTTLE", Form("Process - Warning: Cannot check status of run %d on Shuttle logbook!",
1695                                                 GetCurrentRun()));
1696                 return hasError == kFALSE;
1697         }
1698
1699         AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
1700                                                 (checkEntryArray.At(0));
1701
1702         if (checkEntry)
1703         {
1704                 if (checkEntry->IsDone())
1705                 {
1706                         Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
1707                         UpdateShuttleLogbook("shuttle_done");
1708                 }
1709                 else
1710                 {
1711                         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
1712                         {
1713                                 if (checkEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
1714                                 {
1715                                         AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
1716                                                         checkEntry->GetRun(), GetDetName(iDet)));
1717                                         fFirstUnprocessed[iDet] = kFALSE;
1718                                 }
1719                         }
1720                 }
1721         }
1722
1723         // remove ML instance
1724         delete fMonaLisa;
1725         fMonaLisa = 0;
1726
1727         fLogbookEntry = 0;
1728
1729         return hasError == kFALSE;
1730 }
1731
1732 //______________________________________________________________________________________________
1733 Bool_t AliShuttle::ProcessCurrentDetector()
1734 {
1735         //
1736         // Makes data retrieval just for a specific detector (fCurrentDetector).
1737         // Threre should be a configuration for this detector.
1738
1739         Log("SHUTTLE", Form("ProcessCurrentDetector - Retrieving values for %s, run %d",
1740                                                 fCurrentDetector.Data(), GetCurrentRun()));
1741
1742         TString wd = gSystem->WorkingDirectory();
1743
1744         if (!CleanReferenceStorage(fCurrentDetector.Data()))
1745                 return kFALSE;
1746
1747         gSystem->ChangeDirectory(wd.Data());
1748
1749         TMap* dcsMap = new TMap();
1750
1751         // call preprocessor
1752         AliPreprocessor* aPreprocessor =
1753                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
1754
1755         aPreprocessor->Initialize(GetCurrentRun(), GetCurrentStartTime(), GetCurrentEndTime());
1756
1757         Bool_t processDCS = aPreprocessor->ProcessDCS();
1758
1759         if (!processDCS)
1760         {
1761                 Log(fCurrentDetector, "ProcessCurrentDetector -"
1762                         " The preprocessor requested to skip the retrieval of DCS values");
1763         }
1764         else if (fTestMode & kSkipDCS)
1765         {
1766                 Log(fCurrentDetector, "ProcessCurrentDetector - In TESTMODE: Skipping DCS processing");
1767         }
1768         else if (fTestMode & kErrorDCS)
1769         {
1770                 Log(fCurrentDetector, "ProcessCurrentDetector - In TESTMODE: Simulating DCS error");
1771                 UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
1772                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
1773                 delete dcsMap;
1774                 return kFALSE;
1775         } else {
1776
1777                 UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
1778
1779                 // Query DCS archive
1780                 Int_t nServers = fConfig->GetNServers(fCurrentDetector);
1781
1782                 for (int iServ=0; iServ<nServers; iServ++)
1783                 {
1784
1785                         TString host(fConfig->GetDCSHost(fCurrentDetector, iServ));
1786                         Int_t port = fConfig->GetDCSPort(fCurrentDetector, iServ);
1787                         Int_t multiSplit = fConfig->GetMultiSplit(fCurrentDetector, iServ);
1788
1789                         Log(fCurrentDetector, Form("ProcessCurrentDetector -"
1790                                         " Querying DCS Amanda server %s:%d (%d of %d)",
1791                                         host.Data(), port, iServ+1, nServers));
1792
1793                         TMap* aliasMap = 0;
1794                         TMap* dpMap = 0;
1795
1796                         if (fConfig->GetDCSAliases(fCurrentDetector, iServ)->GetEntries() > 0)
1797                         {
1798                                 aliasMap = GetValueSet(host, port,
1799                                                 fConfig->GetDCSAliases(fCurrentDetector, iServ),
1800                                                 kAlias, multiSplit);
1801                                 if (!aliasMap)
1802                                 {
1803                                         Log(fCurrentDetector,
1804                                                 Form("ProcessCurrentDetector -"
1805                                                         " Error retrieving DCS aliases from server %s."
1806                                                         " Sending mail to DCS experts!", host.Data()));
1807                                         UpdateShuttleStatus(AliShuttleStatus::kDCSError);
1808
1809                                         if (!SendMailToDCS())
1810                                                 Log("SHUTTLE", Form("ProcessCurrentDetector - Could not send mail to DCS experts!"));
1811
1812                                         delete dcsMap;
1813                                         return kFALSE;
1814                                 }
1815                         }
1816
1817                         if (fConfig->GetDCSDataPoints(fCurrentDetector, iServ)->GetEntries() > 0)
1818                         {
1819                                 dpMap = GetValueSet(host, port,
1820                                                 fConfig->GetDCSDataPoints(fCurrentDetector, iServ),
1821                                                 kDP, multiSplit);
1822                                 if (!dpMap)
1823                                 {
1824                                         Log(fCurrentDetector,
1825                                                 Form("ProcessCurrentDetector -"
1826                                                         " Error retrieving DCS data points from server %s."
1827                                                         " Sending mail to DCS experts!", host.Data()));
1828                                         UpdateShuttleStatus(AliShuttleStatus::kDCSError);
1829
1830                                         if (!SendMailToDCS())
1831                                                 Log("SHUTTLE", Form("ProcessCurrentDetector - Could not send mail to DCS experts!"));
1832
1833                                         if (aliasMap) delete aliasMap;
1834                                         delete dcsMap;
1835                                         return kFALSE;
1836                                 }
1837                         }
1838
1839                         // merge aliasMap and dpMap into dcsMap
1840                         if(aliasMap) {
1841                                 TIter iter(aliasMap);
1842                                 TObjString* key = 0;
1843                                 while ((key = (TObjString*) iter.Next()))
1844                                         dcsMap->Add(key, aliasMap->GetValue(key->String()));
1845
1846                                 aliasMap->SetOwner(kFALSE);
1847                                 delete aliasMap;
1848                         }
1849
1850                         if(dpMap) {
1851                                 TIter iter(dpMap);
1852                                 TObjString* key = 0;
1853                                 while ((key = (TObjString*) iter.Next()))
1854                                         dcsMap->Add(key, dpMap->GetValue(key->String()));
1855
1856                                 dpMap->SetOwner(kFALSE);
1857                                 delete dpMap;
1858                         }
1859                 }
1860         }
1861
1862         // DCS Archive DB processing successful. Call Preprocessor!
1863         UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
1864
1865         UInt_t returnValue = aPreprocessor->Process(dcsMap);
1866
1867         if (returnValue > 0) // Preprocessor error!
1868         {
1869                 Log(fCurrentDetector, Form("ProcessCurrentDetector - "
1870                                 "Preprocessor failed. Process returned %d.", returnValue));
1871                 UpdateShuttleStatus(AliShuttleStatus::kPPError);
1872                 dcsMap->DeleteAll();
1873                 delete dcsMap;
1874                 return kFALSE;
1875         }
1876
1877         // preprocessor ok!
1878         UpdateShuttleStatus(AliShuttleStatus::kPPDone);
1879         Log(fCurrentDetector, Form("ProcessCurrentDetector - %s preprocessor returned success",
1880                                 fCurrentDetector.Data()));
1881
1882         dcsMap->DeleteAll();
1883         delete dcsMap;
1884
1885         return kTRUE;
1886 }
1887
1888 //______________________________________________________________________________________________
1889 Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
1890                 TObjArray& entries)
1891 {
1892         // Query DAQ's Shuttle logbook and fills detector status object.
1893         // Call QueryRunParameters to query DAQ logbook for run parameters.
1894         //
1895
1896         entries.SetOwner(1);
1897
1898         // check connection, in case connect
1899         if(!Connect(3)) return kFALSE;
1900
1901         TString sqlQuery;
1902         sqlQuery = Form("select * from %s %s order by run", fConfig->GetShuttlelbTable(), whereClause);
1903
1904         TSQLResult* aResult = fServer[3]->Query(sqlQuery);
1905         if (!aResult) {
1906                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
1907                 return kFALSE;
1908         }
1909
1910         AliDebug(2,Form("Query = %s", sqlQuery.Data()));
1911
1912         if(aResult->GetRowCount() == 0) {
1913                 Log("SHUTTLE", "No entries in Shuttle Logbook match request");
1914                 delete aResult;
1915                 return kTRUE;
1916         }
1917
1918         // TODO Check field count!
1919         const UInt_t nCols = 23;
1920         if (aResult->GetFieldCount() != (Int_t) nCols) {
1921                 Log("SHUTTLE", "Invalid SQL result field number!");
1922                 delete aResult;
1923                 return kFALSE;
1924         }
1925
1926         TSQLRow* aRow;
1927         while ((aRow = aResult->Next())) {
1928                 TString runString(aRow->GetField(0), aRow->GetFieldLength(0));
1929                 Int_t run = runString.Atoi();
1930
1931                 AliShuttleLogbookEntry *entry = QueryRunParameters(run);
1932                 if (!entry)
1933                         continue;
1934
1935                 // loop on detectors
1936                 for(UInt_t ii = 0; ii < nCols; ii++)
1937                         entry->SetDetectorStatus(aResult->GetFieldName(ii), aRow->GetField(ii));
1938
1939                 entries.AddLast(entry);
1940                 delete aRow;
1941         }
1942
1943         delete aResult;
1944         return kTRUE;
1945 }
1946
1947 //______________________________________________________________________________________________
1948 AliShuttleLogbookEntry* AliShuttle::QueryRunParameters(Int_t run)
1949 {
1950         //
1951         // Retrieve run parameters written in the DAQ logbook and sets them into AliShuttleLogbookEntry object
1952         //
1953
1954         // check connection, in case connect
1955         if (!Connect(3))
1956                 return 0;
1957
1958         TString sqlQuery;
1959         sqlQuery.Form("select * from %s where run=%d", fConfig->GetDAQlbTable(), run);
1960
1961         TSQLResult* aResult = fServer[3]->Query(sqlQuery);
1962         if (!aResult) {
1963                 Log("SHUTTLE", Form("Can't execute query <%s>!", sqlQuery.Data()));
1964                 return 0;
1965         }
1966
1967         if (aResult->GetRowCount() == 0) {
1968                 Log("SHUTTLE", Form("QueryRunParameters - No entry in DAQ Logbook for run %d. Skipping", run));
1969                 delete aResult;
1970                 return 0;
1971         }
1972
1973         if (aResult->GetRowCount() > 1) {
1974                 Log("SHUTTLE", Form("QueryRunParameters - UNEXPECTED: "
1975                                 "more than one entry in DAQ Logbook for run %d!", run));
1976                 delete aResult;
1977                 return 0;
1978         }
1979
1980         TSQLRow* aRow = aResult->Next();
1981         if (!aRow)
1982         {
1983                 Log("SHUTTLE", Form("QueryRunParameters - Could not retrieve row for run %d. Skipping", run));
1984                 delete aResult;
1985                 return 0;
1986         }
1987
1988         AliShuttleLogbookEntry* entry = new AliShuttleLogbookEntry(run);
1989
1990         for (Int_t ii = 0; ii < aResult->GetFieldCount(); ii++)
1991                 entry->SetRunParameter(aResult->GetFieldName(ii), aRow->GetField(ii));
1992
1993         UInt_t startTime = entry->GetStartTime();
1994         UInt_t endTime = entry->GetEndTime();
1995
1996         if (!startTime || !endTime || startTime > endTime) {
1997                 Log("SHUTTLE",
1998                         Form("QueryRunParameters - Invalid parameters for Run %d: startTime = %d, endTime = %d",
1999                                 run, startTime, endTime));
2000                 delete entry;
2001                 delete aRow;
2002                 delete aResult;
2003                 return 0;
2004         }
2005
2006         delete aRow;
2007         delete aResult;
2008
2009         return entry;
2010 }
2011
2012 //______________________________________________________________________________________________
2013 TMap* AliShuttle::GetValueSet(const char* host, Int_t port, const TSeqCollection* entries,
2014                               DCSType type, Int_t multiSplit)
2015 {
2016         // Retrieve all "entry" data points from the DCS server
2017         // host, port: TSocket connection parameters
2018         // entries: list of name of the alias or data point
2019         // type: kAlias or kDP
2020         // returns TMap of values, 0 when failure
2021
2022         AliDCSClient client(host, port, fTimeout, fRetries, multiSplit);
2023
2024         TMap* result = 0;
2025         if (type == kAlias)
2026         {
2027                 result = client.GetAliasValues(entries, GetCurrentStartTime(),
2028                         GetCurrentEndTime());
2029         }
2030         else if (type == kDP)
2031         {
2032                 result = client.GetDPValues(entries, GetCurrentStartTime(),
2033                         GetCurrentEndTime());
2034         }
2035
2036         if (result == 0)
2037         {
2038                 Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get entries! Reason: %s",
2039                         client.GetErrorString(client.GetResultErrorCode())));
2040                 if (client.GetResultErrorCode() == AliDCSClient::fgkServerError)
2041                         Log(fCurrentDetector.Data(), Form("GetValueSet - Server error code: %s",
2042                                 client.GetServerError().Data()));
2043
2044                 return 0;
2045         }
2046
2047         return result;
2048 }
2049
2050 //______________________________________________________________________________________________
2051 const char* AliShuttle::GetFile(Int_t system, const char* detector,
2052                 const char* id, const char* source)
2053 {
2054         // Get calibration file from file exchange servers
2055         // First queris the FXS database for the file name, using the run, detector, id and source info
2056         // then calls RetrieveFile(filename) for actual copy to local disk
2057         // run: current run being processed (given by Logbook entry fLogbookEntry)
2058         // detector: the Preprocessor name
2059         // id: provided as a parameter by the Preprocessor
2060         // source: provided by the Preprocessor through GetFileSources function
2061
2062         // check if test mode should simulate a FXS error
2063         if (fTestMode & kErrorFXSFiles)
2064         {
2065                 Log(detector, Form("GetFile - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
2066                 return 0;
2067         }
2068
2069         // check connection, in case connect
2070         if (!Connect(system))
2071         {
2072                 Log(detector, Form("GetFile - Couldn't connect to %s FXS database", GetSystemName(system)));
2073                 return 0;
2074         }
2075
2076         // Query preparation
2077         TString sourceName(source);
2078         Int_t nFields = 3;
2079         TString sqlQueryStart = Form("select filePath,size,fileChecksum from %s where",
2080                                                                 fConfig->GetFXSdbTable(system));
2081         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
2082                                                                 GetCurrentRun(), detector, id);
2083
2084         if (system == kDAQ)
2085         {
2086                 whereClause += Form(" and DAQsource=\"%s\"", source);
2087         }
2088         else if (system == kDCS)
2089         {
2090                 sourceName="none";
2091         }
2092         else if (system == kHLT)
2093         {
2094                 whereClause += Form(" and DDLnumbers=\"%s\"", source);
2095                 nFields = 3;
2096         }
2097
2098         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
2099
2100         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2101
2102         // Query execution
2103         TSQLResult* aResult = 0;
2104         aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
2105         if (!aResult) {
2106                 Log(detector, Form("GetFileName - Can't execute SQL query to %s database for: id = %s, source = %s",
2107                                 GetSystemName(system), id, sourceName.Data()));
2108                 return 0;
2109         }
2110
2111         if(aResult->GetRowCount() == 0)
2112         {
2113                 Log(detector,
2114                         Form("GetFileName - No entry in %s FXS db for: id = %s, source = %s",
2115                                 GetSystemName(system), id, sourceName.Data()));
2116                 delete aResult;
2117                 return 0;
2118         }
2119
2120         if (aResult->GetRowCount() > 1) {
2121                 Log(detector,
2122                         Form("GetFileName - More than one entry in %s FXS db for: id = %s, source = %s",
2123                                 GetSystemName(system), id, sourceName.Data()));
2124                 delete aResult;
2125                 return 0;
2126         }
2127
2128         if (aResult->GetFieldCount() != nFields) {
2129                 Log(detector,
2130                         Form("GetFileName - Wrong field count in %s FXS db for: id = %s, source = %s",
2131                                 GetSystemName(system), id, sourceName.Data()));
2132                 delete aResult;
2133                 return 0;
2134         }
2135
2136         TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
2137
2138         if (!aRow){
2139                 Log(detector, Form("GetFileName - Empty set result in %s FXS db from query: id = %s, source = %s",
2140                                 GetSystemName(system), id, sourceName.Data()));
2141                 delete aResult;
2142                 return 0;
2143         }
2144
2145         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
2146         TString fileSize(aRow->GetField(1), aRow->GetFieldLength(1));
2147         TString fileChecksum(aRow->GetField(2), aRow->GetFieldLength(2));
2148
2149         delete aResult;
2150         delete aRow;
2151
2152         AliDebug(2, Form("filePath = %s; size = %s, fileChecksum = %s",
2153                                 filePath.Data(), fileSize.Data(), fileChecksum.Data()));
2154
2155         // retrieved file is renamed to make it unique
2156         TString localFileName = Form("%s/%s_%d_process/%s_%s_%d_%s_%s.shuttle",
2157                                         GetShuttleTempDir(), detector, GetCurrentRun(),
2158                                         GetSystemName(system), detector, GetCurrentRun(),
2159                                         id, sourceName.Data());
2160
2161
2162         // file retrieval from FXS
2163         UInt_t nRetries = 0;
2164         UInt_t maxRetries = 3;
2165         Bool_t result = kFALSE;
2166
2167         // copy!! if successful TSystem::Exec returns 0
2168         while(nRetries++ < maxRetries) {
2169                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
2170                 result = RetrieveFile(system, filePath.Data(), localFileName.Data());
2171                 if(!result)
2172                 {
2173                         Log(detector, Form("GetFileName - Copy of file %s from %s FXS failed",
2174                                         filePath.Data(), GetSystemName(system)));
2175                         continue;
2176                 }
2177
2178                 if (fileChecksum.Length()>0)
2179                 {
2180                         // compare md5sum of local file with the one stored in the FXS DB
2181                         Int_t md5Comp = gSystem->Exec(Form("md5sum %s |grep %s 2>&1 > /dev/null",
2182                                                 localFileName.Data(), fileChecksum.Data()));
2183
2184                         if (md5Comp != 0)
2185                         {
2186                                 Log(detector, Form("GetFileName - md5sum of file %s does not match with local copy!",
2187                                                         filePath.Data()));
2188                                 result = kFALSE;
2189                                 continue;
2190                         }
2191                 } else {
2192                         Log(fCurrentDetector, Form("GetFile - md5sum of file %s not set in %s database, skipping comparison",
2193                                                         filePath.Data(), GetSystemName(system)));
2194                 }
2195                 if (result) break;
2196         }
2197
2198         if(!result) return 0;
2199
2200         fFXSCalled[system]=kTRUE;
2201         TObjString *fileParams = new TObjString(Form("%s#!?!#%s", id, sourceName.Data()));
2202         fFXSlist[system].Add(fileParams);
2203
2204         static TString staticLocalFileName;
2205         staticLocalFileName.Form("%s", localFileName.Data());
2206
2207         Log(fCurrentDetector, Form("GetFile - Retrieved file with id %s and "
2208                         "source %s from %s to %s", id, source,
2209                         GetSystemName(system), localFileName.Data()));
2210
2211         return staticLocalFileName.Data();
2212 }
2213
2214 //______________________________________________________________________________________________
2215 Bool_t AliShuttle::RetrieveFile(UInt_t system, const char* fxsFileName, const char* localFileName)
2216 {
2217         //
2218         // Copies file from FXS to local Shuttle machine
2219         //
2220
2221         // check temp directory: trying to cd to temp; if it does not exist, create it
2222         AliDebug(2, Form("Copy file %s from %s FXS into %s",
2223                         GetSystemName(system), fxsFileName, localFileName));
2224
2225         TString tmpDir(localFileName);
2226
2227         tmpDir = tmpDir(0,tmpDir.Last('/'));
2228
2229         Int_t noDir = gSystem->GetPathInfo(tmpDir.Data(), 0, (Long64_t*) 0, 0, 0);
2230         if (noDir) // temp dir does not exists!
2231         {
2232                 if (gSystem->mkdir(tmpDir.Data(), 1))
2233                 {
2234                         Log(fCurrentDetector.Data(), "RetrieveFile - could not make temp directory!!");
2235                         return kFALSE;
2236                 }
2237         }
2238
2239         TString baseFXSFolder;
2240         if (system == kDAQ)
2241         {
2242                 baseFXSFolder = "FES/";
2243         }
2244         else if (system == kDCS)
2245         {
2246                 baseFXSFolder = "";
2247         }
2248         else if (system == kHLT)
2249         {
2250                 baseFXSFolder = "/opt/FXS/";
2251         }
2252
2253
2254         TString command = Form("scp -oPort=%d -2 %s@%s:%s%s %s",
2255                 fConfig->GetFXSPort(system),
2256                 fConfig->GetFXSUser(system),
2257                 fConfig->GetFXSHost(system),
2258                 baseFXSFolder.Data(),
2259                 fxsFileName,
2260                 localFileName);
2261
2262         AliDebug(2, Form("%s",command.Data()));
2263
2264         Bool_t result = (gSystem->Exec(command.Data()) == 0);
2265
2266         return result;
2267 }
2268
2269 //______________________________________________________________________________________________
2270 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
2271 {
2272         //
2273         // Get sources producing the condition file Id from file exchange servers
2274         // if id is NULL all sources are returned (distinct)
2275         //
2276
2277         Log(detector, Form("GetFileSources - Retrieving sources with id %s from %s", id, GetSystemName(system)));
2278
2279         // check if test mode should simulate a FXS error
2280         if (fTestMode & kErrorFXSSources)
2281         {
2282                 Log(detector, Form("GetFileSources - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
2283                 return 0;
2284         }
2285
2286         if (system == kDCS)
2287         {
2288                 Log(detector, "GetFileSources - WARNING: DCS system has only one source of data!");
2289                 TList *list = new TList();
2290                 list->SetOwner(1);
2291                 list->Add(new TObjString(" "));
2292                 return list;
2293         }
2294
2295         // check connection, in case connect
2296         if (!Connect(system))
2297         {
2298                 Log(detector, Form("GetFileSources - Couldn't connect to %s FXS database", GetSystemName(system)));
2299                 return NULL;
2300         }
2301
2302         TString sourceName = 0;
2303         if (system == kDAQ)
2304         {
2305                 sourceName = "DAQsource";
2306         } else if (system == kHLT)
2307         {
2308                 sourceName = "DDLnumbers";
2309         }
2310
2311         TString sqlQueryStart = Form("select distinct %s from %s where", sourceName.Data(), fConfig->GetFXSdbTable(system));
2312         TString whereClause = Form("run=%d and detector=\"%s\"",
2313                                 GetCurrentRun(), detector);
2314         if (id)
2315                 whereClause += Form(" and fileId=\"%s\"", id);
2316         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
2317
2318         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2319
2320         // Query execution
2321         TSQLResult* aResult;
2322         aResult = fServer[system]->Query(sqlQuery);
2323         if (!aResult) {
2324                 Log(detector, Form("GetFileSources - Can't execute SQL query to %s database for id: %s",
2325                                 GetSystemName(system), id));
2326                 return 0;
2327         }
2328
2329         TList *list = new TList();
2330         list->SetOwner(1);
2331
2332         if (aResult->GetRowCount() == 0)
2333         {
2334                 Log(detector,
2335                         Form("GetFileSources - No entry in %s FXS table for id: %s", GetSystemName(system), id));
2336                 delete aResult;
2337                 return list;
2338         }
2339
2340         Log(detector, Form("GetFileSources - Found %d sources", aResult->GetRowCount()));
2341
2342         TSQLRow* aRow;
2343         while ((aRow = aResult->Next()))
2344         {
2345
2346                 TString source(aRow->GetField(0), aRow->GetFieldLength(0));
2347                 AliDebug(2, Form("%s = %s", sourceName.Data(), source.Data()));
2348                 list->Add(new TObjString(source));
2349                 delete aRow;
2350         }
2351
2352         delete aResult;
2353
2354         return list;
2355 }
2356
2357 //______________________________________________________________________________________________
2358 TList* AliShuttle::GetFileIDs(Int_t system, const char* detector, const char* source)
2359 {
2360         //
2361         // Get all ids of condition files produced by a given source from file exchange servers
2362         //
2363
2364         Log(detector, Form("GetFileIDs - Retrieving ids with source %s with %s", source, GetSystemName(system)));
2365
2366         // check if test mode should simulate a FXS error
2367         if (fTestMode & kErrorFXSSources)
2368         {
2369                 Log(detector, Form("GetFileIDs - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
2370                 return 0;
2371         }
2372
2373         // check connection, in case connect
2374         if (!Connect(system))
2375         {
2376                 Log(detector, Form("GetFileIDs - Couldn't connect to %s FXS database", GetSystemName(system)));
2377                 return NULL;
2378         }
2379
2380         TString sourceName = 0;
2381         if (system == kDAQ)
2382         {
2383                 sourceName = "DAQsource";
2384         } else if (system == kHLT)
2385         {
2386                 sourceName = "DDLnumbers";
2387         }
2388
2389         TString sqlQueryStart = Form("select fileId from %s where", fConfig->GetFXSdbTable(system));
2390         TString whereClause = Form("run=%d and detector=\"%s\"",
2391                                 GetCurrentRun(), detector);
2392         if (sourceName.Length() > 0 && source)
2393                 whereClause += Form(" and %s=\"%s\"", sourceName.Data(), source);
2394         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
2395
2396         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2397
2398         // Query execution
2399         TSQLResult* aResult;
2400         aResult = fServer[system]->Query(sqlQuery);
2401         if (!aResult) {
2402                 Log(detector, Form("GetFileIDs - Can't execute SQL query to %s database for source: %s",
2403                                 GetSystemName(system), source));
2404                 return 0;
2405         }
2406
2407         TList *list = new TList();
2408         list->SetOwner(1);
2409
2410         if (aResult->GetRowCount() == 0)
2411         {
2412                 Log(detector,
2413                         Form("GetFileIDs - No entry in %s FXS table for source: %s", GetSystemName(system), source));
2414                 delete aResult;
2415                 return list;
2416         }
2417
2418         Log(detector, Form("GetFileIDs - Found %d ids", aResult->GetRowCount()));
2419
2420         TSQLRow* aRow;
2421
2422         while ((aRow = aResult->Next()))
2423         {
2424
2425                 TString id(aRow->GetField(0), aRow->GetFieldLength(0));
2426                 AliDebug(2, Form("fileId = %s", id.Data()));
2427                 list->Add(new TObjString(id));
2428                 delete aRow;
2429         }
2430
2431         delete aResult;
2432
2433         return list;
2434 }
2435
2436 //______________________________________________________________________________________________
2437 Bool_t AliShuttle::Connect(Int_t system)
2438 {
2439         // Connect to MySQL Server of the system's FXS MySQL databases
2440         // DAQ Logbook, Shuttle Logbook and DAQ FXS db are on the same host
2441         //
2442
2443         // check connection: if already connected return
2444         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
2445
2446         TString dbHost, dbUser, dbPass, dbName;
2447
2448         if (system < 3) // FXS db servers
2449         {
2450                 dbHost = Form("mysql://%s:%d", fConfig->GetFXSdbHost(system), fConfig->GetFXSdbPort(system));
2451                 dbUser = fConfig->GetFXSdbUser(system);
2452                 dbPass = fConfig->GetFXSdbPass(system);
2453                 dbName =   fConfig->GetFXSdbName(system);
2454         } else { // Run & Shuttle logbook servers
2455         // TODO Will the Shuttle logbook server be the same as the Run logbook server ???
2456                 dbHost = Form("mysql://%s:%d", fConfig->GetDAQlbHost(), fConfig->GetDAQlbPort());
2457                 dbUser = fConfig->GetDAQlbUser();
2458                 dbPass = fConfig->GetDAQlbPass();
2459                 dbName =   fConfig->GetDAQlbDB();
2460         }
2461
2462         fServer[system] = TSQLServer::Connect(dbHost.Data(), dbUser.Data(), dbPass.Data());
2463         if (!fServer[system] || !fServer[system]->IsConnected()) {
2464                 if(system < 3)
2465                 {
2466                 AliError(Form("Can't establish connection to FXS database for %s",
2467                                         AliShuttleInterface::GetSystemName(system)));
2468                 } else {
2469                 AliError("Can't establish connection to Run logbook.");
2470                 }
2471                 if(fServer[system]) delete fServer[system];
2472                 return kFALSE;
2473         }
2474
2475         // Get tables
2476         TSQLResult* aResult=0;
2477         switch(system){
2478                 case kDAQ:
2479                         aResult = fServer[kDAQ]->GetTables(dbName.Data());
2480                         break;
2481                 case kDCS:
2482                         aResult = fServer[kDCS]->GetTables(dbName.Data());
2483                         break;
2484                 case kHLT:
2485                         aResult = fServer[kHLT]->GetTables(dbName.Data());
2486                         break;
2487                 default:
2488                         aResult = fServer[3]->GetTables(dbName.Data());
2489                         break;
2490         }
2491
2492         delete aResult;
2493         return kTRUE;
2494 }
2495
2496 //______________________________________________________________________________________________
2497 Bool_t AliShuttle::UpdateTable()
2498 {
2499         //
2500         // Update FXS table filling time_processed field in all rows corresponding to current run and detector
2501         //
2502
2503         Bool_t result = kTRUE;
2504
2505         for (UInt_t system=0; system<3; system++)
2506         {
2507                 if(!fFXSCalled[system]) continue;
2508
2509                 // check connection, in case connect
2510                 if (!Connect(system))
2511                 {
2512                         Log(fCurrentDetector, Form("UpdateTable - Couldn't connect to %s FXS database", GetSystemName(system)));
2513                         result = kFALSE;
2514                         continue;
2515                 }
2516
2517                 TTimeStamp now; // now
2518
2519                 // Loop on FXS list entries
2520                 TIter iter(&fFXSlist[system]);
2521                 TObjString *aFXSentry=0;
2522                 while ((aFXSentry = dynamic_cast<TObjString*> (iter.Next())))
2523                 {
2524                         TString aFXSentrystr = aFXSentry->String();
2525                         TObjArray *aFXSarray = aFXSentrystr.Tokenize("#!?!#");
2526                         if (!aFXSarray || aFXSarray->GetEntries() != 2 )
2527                         {
2528                                 Log(fCurrentDetector, Form("UpdateTable - error updating %s FXS entry. Check string: <%s>",
2529                                         GetSystemName(system), aFXSentrystr.Data()));
2530                                 if(aFXSarray) delete aFXSarray;
2531                                 result = kFALSE;
2532                                 continue;
2533                         }
2534                         const char* fileId = ((TObjString*) aFXSarray->At(0))->GetName();
2535                         const char* source = ((TObjString*) aFXSarray->At(1))->GetName();
2536
2537                         TString whereClause;
2538                         if (system == kDAQ)
2539                         {
2540                                 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
2541                                                         GetCurrentRun(), fCurrentDetector.Data(), fileId, source);
2542                         }
2543                         else if (system == kDCS)
2544                         {
2545                                 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\";",
2546                                                         GetCurrentRun(), fCurrentDetector.Data(), fileId);
2547                         }
2548                         else if (system == kHLT)
2549                         {
2550                                 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DDLnumbers=\"%s\";",
2551                                                         GetCurrentRun(), fCurrentDetector.Data(), fileId, source);
2552                         }
2553
2554                         delete aFXSarray;
2555
2556                         TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(system),
2557                                                                 now.GetSec(), whereClause.Data());
2558
2559                         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2560
2561                         // Query execution
2562                         TSQLResult* aResult;
2563                         aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
2564                         if (!aResult)
2565                         {
2566                                 Log(fCurrentDetector, Form("UpdateTable - %s db: can't execute SQL query <%s>",
2567                                                                 GetSystemName(system), sqlQuery.Data()));
2568                                 result = kFALSE;
2569                                 continue;
2570                         }
2571                         delete aResult;
2572                 }
2573         }
2574
2575         return result;
2576 }
2577
2578 //______________________________________________________________________________________________
2579 Bool_t AliShuttle::UpdateTableFailCase()
2580 {
2581         // Update FXS table filling time_processed field in all rows corresponding to current run and detector
2582         // this is called in case the preprocessor is declared failed for the current run, because
2583         // the fields are updated only in case of success
2584
2585         Bool_t result = kTRUE;
2586
2587         for (UInt_t system=0; system<3; system++)
2588         {
2589                 // check connection, in case connect
2590                 if (!Connect(system))
2591                 {
2592                         Log(fCurrentDetector, Form("UpdateTableFailCase - Couldn't connect to %s FXS database",
2593                                                         GetSystemName(system)));
2594                         result = kFALSE;
2595                         continue;
2596                 }
2597
2598                 TTimeStamp now; // now
2599
2600                 // Loop on FXS list entries
2601
2602                 TString whereClause = Form("where run=%d and detector=\"%s\";",
2603                                                 GetCurrentRun(), fCurrentDetector.Data());
2604
2605
2606                 TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(system),
2607                                                         now.GetSec(), whereClause.Data());
2608
2609                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2610
2611                 // Query execution
2612                 TSQLResult* aResult;
2613                 aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
2614                 if (!aResult)
2615                 {
2616                         Log(fCurrentDetector, Form("UpdateTableFailCase - %s db: can't execute SQL query <%s>",
2617                                                         GetSystemName(system), sqlQuery.Data()));
2618                         result = kFALSE;
2619                         continue;
2620                 }
2621                 delete aResult;
2622         }
2623
2624         return result;
2625 }
2626
2627 //______________________________________________________________________________________________
2628 Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status)
2629 {
2630         //
2631         // Update Shuttle logbook filling detector or shuttle_done column
2632         // ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
2633         //
2634
2635         // check connection, in case connect
2636         if(!Connect(3)){
2637                 Log("SHUTTLE", "UpdateShuttleLogbook - Couldn't connect to DAQ Logbook.");
2638                 return kFALSE;
2639         }
2640
2641         TString detName(detector);
2642         TString setClause;
2643         if(detName == "shuttle_done")
2644         {
2645                 setClause = "set shuttle_done=1";
2646
2647                 // Send the information to ML
2648                 TMonaLisaText  mlStatus("SHUTTLE_status", "Done");
2649
2650                 TList mlList;
2651                 mlList.Add(&mlStatus);
2652
2653                 fMonaLisa->SendParameters(&mlList);
2654         } else {
2655                 TString statusStr(status);
2656                 if(statusStr.Contains("done", TString::kIgnoreCase) ||
2657                    statusStr.Contains("failed", TString::kIgnoreCase)){
2658                         setClause = Form("set %s=\"%s\"", detector, status);
2659                 } else {
2660                         Log("SHUTTLE",
2661                                 Form("UpdateShuttleLogbook - Invalid status <%s> for detector %s",
2662                                         status, detector));
2663                         return kFALSE;
2664                 }
2665         }
2666
2667         TString whereClause = Form("where run=%d", GetCurrentRun());
2668
2669         TString sqlQuery = Form("update %s %s %s",
2670                                         fConfig->GetShuttlelbTable(), setClause.Data(), whereClause.Data());
2671
2672         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2673
2674         // Query execution
2675         TSQLResult* aResult;
2676         aResult = dynamic_cast<TSQLResult*> (fServer[3]->Query(sqlQuery));
2677         if (!aResult) {
2678                 Log("SHUTTLE", Form("UpdateShuttleLogbook - Can't execute query <%s>", sqlQuery.Data()));
2679                 return kFALSE;
2680         }
2681         delete aResult;
2682
2683         return kTRUE;
2684 }
2685
2686 //______________________________________________________________________________________________
2687 Int_t AliShuttle::GetCurrentRun() const
2688 {
2689         //
2690         // Get current run from logbook entry
2691         //
2692
2693         return fLogbookEntry ? fLogbookEntry->GetRun() : -1;
2694 }
2695
2696 //______________________________________________________________________________________________
2697 UInt_t AliShuttle::GetCurrentStartTime() const
2698 {
2699         //
2700         // get current start time
2701         //
2702
2703         return fLogbookEntry ? fLogbookEntry->GetStartTime() : 0;
2704 }
2705
2706 //______________________________________________________________________________________________
2707 UInt_t AliShuttle::GetCurrentEndTime() const
2708 {
2709         //
2710         // get current end time from logbook entry
2711         //
2712
2713         return fLogbookEntry ? fLogbookEntry->GetEndTime() : 0;
2714 }
2715
2716 //______________________________________________________________________________________________
2717 UInt_t AliShuttle::GetCurrentYear() const
2718 {
2719         //
2720         // Get current year from logbook entry
2721         //
2722
2723         if (!fLogbookEntry) return 0;
2724
2725         TTimeStamp startTime(GetCurrentStartTime());
2726         TString year =  Form("%d",startTime.GetDate());
2727         year = year(0,4);
2728
2729         return year.Atoi();
2730 }
2731
2732 //______________________________________________________________________________________________
2733 const char* AliShuttle::GetLHCPeriod() const
2734 {
2735         //
2736         // Get current LHC period from logbook entry
2737         //
2738
2739         if (!fLogbookEntry) return 0;
2740
2741         return fLogbookEntry->GetRunParameter("LHCperiod");
2742 }
2743
2744 //______________________________________________________________________________________________
2745 void AliShuttle::Log(const char* detector, const char* message)
2746 {
2747         //
2748         // Fill log string with a message
2749         //
2750
2751         void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
2752         if (dir == NULL) {
2753                 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE)) {
2754                         AliError(Form("Can't open directory <%s>", GetShuttleLogDir()));
2755                         return;
2756                 }
2757
2758         } else {
2759                 gSystem->FreeDirectory(dir);
2760         }
2761
2762         TString toLog = Form("%s (%d): %s - ", TTimeStamp(time(0)).AsString("s"), getpid(), detector);
2763         if (GetCurrentRun() >= 0)
2764                 toLog += Form("run %d - ", GetCurrentRun());
2765         toLog += Form("%s", message);
2766
2767         AliInfo(toLog.Data());
2768
2769         // if we redirect the log output already to the file, leave here
2770         if (fOutputRedirected && strcmp(detector, "SHUTTLE") != 0)
2771                 return;
2772
2773         TString fileName = GetLogFileName(detector);
2774
2775         gSystem->ExpandPathName(fileName);
2776
2777         ofstream logFile;
2778         logFile.open(fileName, ofstream::out | ofstream::app);
2779
2780         if (!logFile.is_open()) {
2781                 AliError(Form("Could not open file %s", fileName.Data()));
2782                 return;
2783         }
2784
2785         logFile << toLog.Data() << "\n";
2786
2787         logFile.close();
2788 }
2789
2790 //______________________________________________________________________________________________
2791 TString AliShuttle::GetLogFileName(const char* detector) const
2792 {
2793         //
2794         // returns the name of the log file for a given sub detector
2795         //
2796
2797         TString fileName;
2798
2799         if (GetCurrentRun() >= 0)
2800                 fileName.Form("%s/%s_%d.log", GetShuttleLogDir(), detector, GetCurrentRun());
2801         else
2802                 fileName.Form("%s/%s.log", GetShuttleLogDir(), detector);
2803
2804         return fileName;
2805 }
2806
2807 //______________________________________________________________________________________________
2808 Bool_t AliShuttle::Collect(Int_t run)
2809 {
2810         //
2811         // Collects conditions data for all UNPROCESSED run written to DAQ LogBook in case of run = -1 (default)
2812         // If a dedicated run is given this run is processed
2813         //
2814         // In operational mode, this is the Shuttle function triggered by the EOR signal.
2815         //
2816
2817         if (run == -1)
2818                 Log("SHUTTLE","Collect - Shuttle called. Collecting conditions data for unprocessed runs");
2819         else
2820                 Log("SHUTTLE", Form("Collect - Shuttle called. Collecting conditions data for run %d", run));
2821
2822         SetLastAction("Starting");
2823
2824         TString whereClause("where shuttle_done=0");
2825         if (run != -1)
2826                 whereClause += Form(" and run=%d", run);
2827
2828         TObjArray shuttleLogbookEntries;
2829         if (!QueryShuttleLogbook(whereClause, shuttleLogbookEntries))
2830         {
2831                 Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
2832                 return kFALSE;
2833         }
2834
2835         if (shuttleLogbookEntries.GetEntries() == 0)
2836         {
2837                 if (run == -1)
2838                         Log("SHUTTLE","Collect - Found no UNPROCESSED runs in Shuttle logbook");
2839                 else
2840                         Log("SHUTTLE", Form("Collect - Run %d is already DONE "
2841                                                 "or it does not exist in Shuttle logbook", run));
2842                 return kTRUE;
2843         }
2844
2845         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
2846                 fFirstUnprocessed[iDet] = kTRUE;
2847
2848         if (run != -1)
2849         {
2850                 // query Shuttle logbook for earlier runs, check if some detectors are unprocessed,
2851                 // flag them into fFirstUnprocessed array
2852                 TString whereClause(Form("where shuttle_done=0 and run < %d", run));
2853                 TObjArray tmpLogbookEntries;
2854                 if (!QueryShuttleLogbook(whereClause, tmpLogbookEntries))
2855                 {
2856                         Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
2857                         return kFALSE;
2858                 }
2859
2860                 TIter iter(&tmpLogbookEntries);
2861                 AliShuttleLogbookEntry* anEntry = 0;
2862                 while ((anEntry = dynamic_cast<AliShuttleLogbookEntry*> (iter.Next())))
2863                 {
2864                         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
2865                         {
2866                                 if (anEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
2867                                 {
2868                                         AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
2869                                                         anEntry->GetRun(), GetDetName(iDet)));
2870                                         fFirstUnprocessed[iDet] = kFALSE;
2871                                 }
2872                         }
2873
2874                 }
2875
2876         }
2877
2878         if (!RetrieveConditionsData(shuttleLogbookEntries))
2879         {
2880                 Log("SHUTTLE", "Collect - Process of at least one run failed");
2881                 return kFALSE;
2882         }
2883
2884         Log("SHUTTLE", "Collect - Requested run(s) successfully processed");
2885         return kTRUE;
2886 }
2887
2888 //______________________________________________________________________________________________
2889 Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
2890 {
2891         //
2892         // Retrieve conditions data for all runs that aren't processed yet
2893         //
2894
2895         Bool_t hasError = kFALSE;
2896
2897         TIter iter(&dateEntries);
2898         AliShuttleLogbookEntry* anEntry;
2899
2900         while ((anEntry = (AliShuttleLogbookEntry*) iter.Next())){
2901                 if (!Process(anEntry)){
2902                         hasError = kTRUE;
2903                 }
2904
2905                 // clean SHUTTLE temp directory
2906                 //TString filename = Form("%s/*.shuttle", GetShuttleTempDir());
2907                 //RemoveFile(filename.Data());
2908         }
2909
2910         return hasError == kFALSE;
2911 }
2912
2913 //______________________________________________________________________________________________
2914 ULong_t AliShuttle::GetTimeOfLastAction() const
2915 {
2916         //
2917         // Gets time of last action
2918         //
2919
2920         ULong_t tmp;
2921
2922         fMonitoringMutex->Lock();
2923
2924         tmp = fLastActionTime;
2925
2926         fMonitoringMutex->UnLock();
2927
2928         return tmp;
2929 }
2930
2931 //______________________________________________________________________________________________
2932 const TString AliShuttle::GetLastAction() const
2933 {
2934         //
2935         // returns a string description of the last action
2936         //
2937
2938         TString tmp;
2939
2940         fMonitoringMutex->Lock();
2941
2942         tmp = fLastAction;
2943
2944         fMonitoringMutex->UnLock();
2945
2946         return tmp;
2947 }
2948
2949 //______________________________________________________________________________________________
2950 void AliShuttle::SetLastAction(const char* action)
2951 {
2952         //
2953         // updates the monitoring variables
2954         //
2955
2956         fMonitoringMutex->Lock();
2957
2958         fLastAction = action;
2959         fLastActionTime = time(0);
2960
2961         fMonitoringMutex->UnLock();
2962 }
2963
2964 //______________________________________________________________________________________________
2965 const char* AliShuttle::GetRunParameter(const char* param)
2966 {
2967         //
2968         // returns run parameter read from DAQ logbook
2969         //
2970
2971         if(!fLogbookEntry) {
2972                 AliError("No logbook entry!");
2973                 return 0;
2974         }
2975
2976         return fLogbookEntry->GetRunParameter(param);
2977 }
2978
2979 //______________________________________________________________________________________________
2980 AliCDBEntry* AliShuttle::GetFromOCDB(const char* detector, const AliCDBPath& path)
2981 {
2982         //
2983         // returns object from OCDB valid for current run
2984         //
2985
2986         if (fTestMode & kErrorOCDB)
2987         {
2988                 Log(detector, "GetFromOCDB - In TESTMODE - Simulating error with OCDB");
2989                 return 0;
2990         }
2991
2992         AliCDBStorage *sto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
2993         if (!sto)
2994         {
2995                 Log(detector, "GetFromOCDB - Cannot activate main OCDB for query!");
2996                 return 0;
2997         }
2998
2999         return dynamic_cast<AliCDBEntry*> (sto->Get(path, GetCurrentRun()));
3000 }
3001
3002 //______________________________________________________________________________________________
3003 Bool_t AliShuttle::SendMail()
3004 {
3005         //
3006         // sends a mail to the subdetector expert in case of preprocessor error
3007         //
3008
3009         if (fTestMode != kNone)
3010                 return kTRUE;
3011
3012         void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
3013         if (dir == NULL)
3014         {
3015                 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE))
3016                 {
3017                         Log("SHUTTLE", Form("SendMail - Can't open directory <%s>", GetShuttleLogDir()));
3018                         return kFALSE;
3019                 }
3020
3021         } else {
3022                 gSystem->FreeDirectory(dir);
3023         }
3024
3025         TString bodyFileName;
3026         bodyFileName.Form("%s/mail.body", GetShuttleLogDir());
3027         gSystem->ExpandPathName(bodyFileName);
3028
3029         ofstream mailBody;
3030         mailBody.open(bodyFileName, ofstream::out);
3031
3032         if (!mailBody.is_open())
3033         {
3034                 Log("SHUTTLE", Form("Could not open mail body file %s", bodyFileName.Data()));
3035                 return kFALSE;
3036         }
3037
3038         TString to="";
3039         TIter iterExperts(fConfig->GetResponsibles(fCurrentDetector));
3040         TObjString *anExpert=0;
3041         while ((anExpert = (TObjString*) iterExperts.Next()))
3042         {
3043                 to += Form("%s,", anExpert->GetName());
3044         }
3045         to.Remove(to.Length()-1);
3046         AliDebug(2, Form("to: %s",to.Data()));
3047
3048         if (to.IsNull()) {
3049                 Log("SHUTTLE", "List of detector responsibles not yet set!");
3050                 return kFALSE;
3051         }
3052
3053         TString cc="alberto.colla@cern.ch";
3054
3055         TString subject = Form("%s Shuttle preprocessor FAILED in run %d !",
3056                                 fCurrentDetector.Data(), GetCurrentRun());
3057         AliDebug(2, Form("subject: %s", subject.Data()));
3058
3059         TString body = Form("Dear %s expert(s), \n\n", fCurrentDetector.Data());
3060         body += Form("SHUTTLE just detected that your preprocessor "
3061                         "failed processing run %d!!\n\n", GetCurrentRun());
3062         body += Form("Please check %s status on the SHUTTLE monitoring page: \n\n", fCurrentDetector.Data());
3063         body += Form("\thttp://pcalimonitor.cern.ch:8889/shuttle.jsp?time=168 \n\n");
3064         body += Form("Find the %s log for the current run on \n\n"
3065                 "\thttp://pcalishuttle01.cern.ch:8880/logs/%s_%d.log \n\n",
3066                 fCurrentDetector.Data(), fCurrentDetector.Data(), GetCurrentRun());
3067         body += Form("The last 10 lines of %s log file are following:\n\n");
3068
3069         AliDebug(2, Form("Body begin: %s", body.Data()));
3070
3071         mailBody << body.Data();
3072         mailBody.close();
3073         mailBody.open(bodyFileName, ofstream::out | ofstream::app);
3074
3075         TString logFileName = Form("%s/%s_%d.log", GetShuttleLogDir(), fCurrentDetector.Data(), GetCurrentRun());
3076         TString tailCommand = Form("tail -n 10 %s >> %s", logFileName.Data(), bodyFileName.Data());
3077         if (gSystem->Exec(tailCommand.Data()))
3078         {
3079                 mailBody << Form("%s log file not found ...\n\n", fCurrentDetector.Data());
3080         }
3081
3082         TString endBody = Form("------------------------------------------------------\n\n");
3083         endBody += Form("In case of problems please contact the SHUTTLE core team.\n\n");
3084         endBody += "Please do not answer this message directly, it is automatically generated.\n\n";
3085         endBody += "Greetings,\n\n \t\t\tthe SHUTTLE\n";
3086
3087         AliDebug(2, Form("Body end: %s", endBody.Data()));
3088
3089         mailBody << endBody.Data();
3090
3091         mailBody.close();
3092
3093         // send mail!
3094         TString mailCommand = Form("mail -s \"%s\" -c %s %s < %s",
3095                                                 subject.Data(),
3096                                                 cc.Data(),
3097                                                 to.Data(),
3098                                                 bodyFileName.Data());
3099         AliDebug(2, Form("mail command: %s", mailCommand.Data()));
3100
3101         Bool_t result = gSystem->Exec(mailCommand.Data());
3102
3103         return result == 0;
3104 }
3105
3106 //______________________________________________________________________________________________
3107 Bool_t AliShuttle::SendMailToDCS()
3108 {
3109         //
3110         // sends a mail to the DCS experts in case of DCS error
3111         //
3112
3113         if (fTestMode != kNone)
3114                 return kTRUE;
3115
3116         void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
3117         if (dir == NULL)
3118         {
3119                 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE))
3120                 {
3121                         Log("SHUTTLE", Form("SendMailToDCS - Can't open directory <%s>", GetShuttleLogDir()));
3122                         return kFALSE;
3123                 }
3124
3125         } else {
3126                 gSystem->FreeDirectory(dir);
3127         }
3128
3129         TString bodyFileName;
3130         bodyFileName.Form("%s/mail.body", GetShuttleLogDir());
3131         gSystem->ExpandPathName(bodyFileName);
3132
3133         ofstream mailBody;
3134         mailBody.open(bodyFileName, ofstream::out);
3135
3136         if (!mailBody.is_open())
3137         {
3138                 Log("SHUTTLE", Form("SendMailToDCS - Could not open mail body file %s", bodyFileName.Data()));
3139                 return kFALSE;
3140         }
3141
3142         TString to="Vladimir.Fekete@cern.ch, Svetozar.Kapusta@cern.ch";
3143         //TString to="alberto.colla@cern.ch";
3144         AliDebug(2, Form("to: %s",to.Data()));
3145
3146         if (to.IsNull()) {
3147                 Log("SHUTTLE", "List of detector responsibles not yet set!");
3148                 return kFALSE;
3149         }
3150
3151         TString cc="alberto.colla@cern.ch";
3152
3153         TString subject = Form("Retrieval of data points for %s FAILED in run %d !",
3154                                 fCurrentDetector.Data(), GetCurrentRun());
3155         AliDebug(2, Form("subject: %s", subject.Data()));
3156
3157         TString body = Form("Dear DCS experts, \n\n");
3158         body += Form("SHUTTLE couldn\'t retrieve the data points for detector %s "
3159                         "in run %d!!\n\n", fCurrentDetector.Data(), GetCurrentRun());
3160         body += Form("Please check %s status on the SHUTTLE monitoring page: \n\n", fCurrentDetector.Data());
3161         body += Form("\thttp://pcalimonitor.cern.ch:8889/shuttle.jsp?time=168 \n\n");
3162         body += Form("Find the %s log for the current run on \n\n"
3163                 "\thttp://pcalishuttle01.cern.ch:8880/logs/%s_%d.log \n\n",
3164                 fCurrentDetector.Data(), fCurrentDetector.Data(), GetCurrentRun());
3165         body += Form("The last 10 lines of %s log file are following:\n\n");
3166
3167         AliDebug(2, Form("Body begin: %s", body.Data()));
3168
3169         mailBody << body.Data();
3170         mailBody.close();
3171         mailBody.open(bodyFileName, ofstream::out | ofstream::app);
3172
3173         TString logFileName = Form("%s/%s_%d.log", GetShuttleLogDir(), fCurrentDetector.Data(), GetCurrentRun());
3174         TString tailCommand = Form("tail -n 10 %s >> %s", logFileName.Data(), bodyFileName.Data());
3175         if (gSystem->Exec(tailCommand.Data()))
3176         {
3177                 mailBody << Form("%s log file not found ...\n\n", fCurrentDetector.Data());
3178         }
3179
3180         TString endBody = Form("------------------------------------------------------\n\n");
3181         endBody += Form("In case of problems please contact the SHUTTLE core team.\n\n");
3182         endBody += "Please do not answer this message directly, it is automatically generated.\n\n";
3183         endBody += "Greetings,\n\n \t\t\tthe SHUTTLE\n";
3184
3185         AliDebug(2, Form("Body end: %s", endBody.Data()));
3186
3187         mailBody << endBody.Data();
3188
3189         mailBody.close();
3190
3191         // send mail!
3192         TString mailCommand = Form("mail -s \"%s\" -c %s %s < %s",
3193                                                 subject.Data(),
3194                                                 cc.Data(),
3195                                                 to.Data(),
3196                                                 bodyFileName.Data());
3197         AliDebug(2, Form("mail command: %s", mailCommand.Data()));
3198
3199         Bool_t result = gSystem->Exec(mailCommand.Data());
3200
3201         return result == 0;
3202 }
3203
3204 //______________________________________________________________________________________________
3205 const char* AliShuttle::GetRunType()
3206 {
3207         //
3208         // returns run type read from "run type" logbook
3209         //
3210
3211         if(!fLogbookEntry) {
3212                 AliError("No logbook entry!");
3213                 return 0;
3214         }
3215
3216         return fLogbookEntry->GetRunType();
3217 }
3218
3219 //______________________________________________________________________________________________
3220 Bool_t AliShuttle::GetHLTStatus()
3221 {
3222         // Return HLT status (ON=1 OFF=0)
3223         // Converts the HLT status from the status string read in the run logbook (not just a bool)
3224
3225         if(!fLogbookEntry) {
3226                 AliError("No logbook entry!");
3227                 return 0;
3228         }
3229
3230         // TODO implement when HLTStatus is inserted in run logbook
3231         //TString hltStatus = fLogbookEntry->GetRunParameter("HLTStatus");
3232         //if(hltStatus == "OFF") {return kFALSE};
3233
3234         return kTRUE;
3235 }
3236
3237 //______________________________________________________________________________________________
3238 void AliShuttle::SetShuttleTempDir(const char* tmpDir)
3239 {
3240         //
3241         // sets Shuttle temp directory
3242         //
3243
3244         fgkShuttleTempDir = gSystem->ExpandPathName(tmpDir);
3245 }
3246
3247 //______________________________________________________________________________________________
3248 void AliShuttle::SetShuttleLogDir(const char* logDir)
3249 {
3250         //
3251         // sets Shuttle log directory
3252         //
3253
3254         fgkShuttleLogDir = gSystem->ExpandPathName(logDir);
3255 }