]> git.uio.no Git - u/mrichter/AliRoot.git/blame - SHUTTLE/AliShuttle.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
AliDebug instead of AliInfo
[u/mrichter/AliRoot.git] / SHUTTLE / AliShuttle.cxx
CommitLineData
73abe331 1/**************************************************************************
2 * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
ff3781ad 16/* $Id$ */
73abe331 17
18//
19// This class is the main manager for AliShuttle.
20// It organizes the data retrieval from DCS and call the
b948db8d 21// interface methods of AliPreprocessor.
73abe331 22// For every detector in AliShuttleConfgi (see AliShuttleConfig),
23// data for its set of aliases is retrieved. If there is registered
b948db8d 24// AliPreprocessor for this detector then it will be used
25// accroding to the schema (see AliPreprocessor).
26// If there isn't registered AliPreprocessor than the retrieved
73abe331 27// data is stored automatically to the undelying AliCDBStorage.
28// For detSpec is used the alias name.
29//
30
31#include "AliShuttle.h"
32
33#include "AliCDBManager.h"
34#include "AliCDBStorage.h"
35#include "AliCDBId.h"
84090f85 36#include "AliCDBRunRange.h"
37#include "AliCDBPath.h"
5164a766 38#include "AliCDBEntry.h"
73abe331 39#include "AliShuttleConfig.h"
eba76848 40#include "DCSClient/AliDCSClient.h"
73abe331 41#include "AliLog.h"
b948db8d 42#include "AliPreprocessor.h"
5164a766 43#include "AliShuttleStatus.h"
2bb7b766 44#include "AliShuttleLogbookEntry.h"
73abe331 45
57f50b3c 46#include <TSystem.h>
58bc3020 47#include <TObject.h>
b948db8d 48#include <TString.h>
57f50b3c 49#include <TTimeStamp.h>
73abe331 50#include <TObjString.h>
57f50b3c 51#include <TSQLServer.h>
52#include <TSQLResult.h>
53#include <TSQLRow.h>
cb343cfd 54#include <TMutex.h>
9827400b 55#include <TSystemDirectory.h>
56#include <TSystemFile.h>
a986b218 57#include <TFile.h>
9827400b 58#include <TGrid.h>
59#include <TGridResult.h>
73abe331 60
e7f62f16 61#include <TMonaLisaWriter.h>
62
5164a766 63#include <fstream>
64
cb343cfd 65#include <sys/types.h>
66#include <sys/wait.h>
67
1bd183ba 68#include <signal.h>
69
73abe331 70ClassImp(AliShuttle)
71
b948db8d 72//______________________________________________________________________________________________
73AliShuttle::AliShuttle(const AliShuttleConfig* config,
74 UInt_t timeout, Int_t retries):
4f0ab988 75fConfig(config),
76fTimeout(timeout), fRetries(retries),
77fPreprocessorMap(),
2bb7b766 78fLogbookEntry(0),
eba76848 79fCurrentDetector(),
926cbc0e 80fFirstProcessing(0),
85a80aa9 81fStatusEntry(0),
cb343cfd 82fMonitoringMutex(0),
eba76848 83fLastActionTime(0),
e7f62f16 84fLastAction(),
9827400b 85fMonaLisa(0),
86fTestMode(kNone),
ffa29e93 87fReadTestMode(kFALSE),
88fOutputRedirected(kFALSE)
73abe331 89{
90 //
91 // config: AliShuttleConfig used
73abe331 92 // timeout: timeout used for AliDCSClient connection
93 // retries: the number of retries in case of connection error.
94 //
95
57f50b3c 96 if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
be48e3ea 97 for(int iSys=0;iSys<4;iSys++) {
57f50b3c 98 fServer[iSys]=0;
be48e3ea 99 if (iSys < 3)
2c15234c 100 fFXSlist[iSys].SetOwner(kTRUE);
57f50b3c 101 }
2bb7b766 102 fPreprocessorMap.SetOwner(kTRUE);
be48e3ea 103
104 for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
105 fFirstUnprocessed[iDet] = kFALSE;
106
cb343cfd 107 fMonitoringMutex = new TMutex();
58bc3020 108}
109
b948db8d 110//______________________________________________________________________________________________
57f50b3c 111AliShuttle::~AliShuttle()
58bc3020 112{
9827400b 113 //
114 // destructor
115 //
58bc3020 116
b948db8d 117 fPreprocessorMap.DeleteAll();
be48e3ea 118 for(int iSys=0;iSys<4;iSys++)
57f50b3c 119 if(fServer[iSys]) {
120 fServer[iSys]->Close();
121 delete fServer[iSys];
eba76848 122 fServer[iSys] = 0;
57f50b3c 123 }
2bb7b766 124
125 if (fStatusEntry){
126 delete fStatusEntry;
127 fStatusEntry = 0;
128 }
cb343cfd 129
130 if (fMonitoringMutex)
131 {
132 delete fMonitoringMutex;
133 fMonitoringMutex = 0;
134 }
73abe331 135}
136
b948db8d 137//______________________________________________________________________________________________
57f50b3c 138void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
58bc3020 139{
73abe331 140 //
b948db8d 141 // Registers new AliPreprocessor.
73abe331 142 // It uses GetName() for indentificator of the pre processor.
143 // The pre processor is registered it there isn't any other
144 // with the same identificator (GetName()).
145 //
146
eba76848 147 const char* detName = preprocessor->GetName();
148 if(GetDetPos(detName) < 0)
149 AliFatal(Form("********** !!!!! Invalid detector name: %s !!!!! **********", detName));
150
151 if (fPreprocessorMap.GetValue(detName)) {
152 AliWarning(Form("AliPreprocessor %s is already registered!", detName));
73abe331 153 return;
154 }
155
eba76848 156 fPreprocessorMap.Add(new TObjString(detName), preprocessor);
73abe331 157}
b948db8d 158//______________________________________________________________________________________________
3301427a 159Bool_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
84090f85 160 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
73abe331 161{
9827400b 162 // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
163 // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
164 // using this function. Use StoreReferenceData instead!
165 // It calls StoreLocally function which temporarily stores the data locally; when the preprocessor
166 // finishes the data are transferred to the main storage (Grid).
b948db8d 167
3301427a 168 return StoreLocally(fgkLocalCDB, path, object, metaData, validityStart, validityInfinite);
84090f85 169}
170
171//______________________________________________________________________________________________
3301427a 172Bool_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData)
84090f85 173{
9827400b 174 // Stores a CDB object in the storage for reference data. This objects will not be available during
175 // offline reconstrunction. Use this function for reference data only!
176 // It calls StoreLocally function which temporarily stores the data locally; when the preprocessor
177 // finishes the data are transferred to the main storage (Grid).
85a80aa9 178
3301427a 179 return StoreLocally(fgkLocalRefStorage, path, object, metaData);
85a80aa9 180}
181
182//______________________________________________________________________________________________
3301427a 183Bool_t AliShuttle::StoreLocally(const TString& localUri,
85a80aa9 184 const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
185 Int_t validityStart, Bool_t validityInfinite)
186{
9827400b 187 // Store object temporarily in local storage. Parameters are passed by Store and StoreReferenceData functions.
188 // when the preprocessor finishes the data are transferred to the main storage (Grid).
189 // The parameters are:
190 // 1) Uri of the backup storage (Local)
191 // 2) the object's path.
192 // 3) the object to be stored
193 // 4) the metaData to be associated with the object
194 // 5) the validity start run number w.r.t. the current run,
195 // if the data is valid only for this run leave the default 0
196 // 6) specifies if the calibration data is valid for infinity (this means until updated),
197 // typical for calibration runs, the default is kFALSE
198 //
199 // returns 0 if fail, 1 otherwise
84090f85 200
9827400b 201 if (fTestMode & kErrorStorage)
202 {
203 Log(fCurrentDetector, "StoreLocally - In TESTMODE - Simulating error while storing locally");
204 return kFALSE;
205 }
206
3301427a 207 const char* cdbType = (localUri == fgkLocalCDB) ? "CDB" : "Reference";
2bb7b766 208
85a80aa9 209 Int_t firstRun = GetCurrentRun() - validityStart;
84090f85 210 if(firstRun < 0) {
9827400b 211 AliWarning("First valid run happens to be less than 0! Setting it to 0.");
84090f85 212 firstRun=0;
213 }
214
215 Int_t lastRun = -1;
216 if(validityInfinite) {
217 lastRun = AliCDBRunRange::Infinity();
218 } else {
219 lastRun = GetCurrentRun();
220 }
221
3301427a 222 // Version is set to current run, it will be used later to transfer data to Grid
223 AliCDBId id(path, firstRun, lastRun, GetCurrentRun(), -1);
2bb7b766 224
225 if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
226 TObjString runUsed = Form("%d", GetCurrentRun());
9e080f92 227 metaData->SetProperty("RunUsed(TObjString)", runUsed.Clone());
2bb7b766 228 }
84090f85 229
3301427a 230 Bool_t result = kFALSE;
84090f85 231
3301427a 232 if (!(AliCDBManager::Instance()->GetStorage(localUri))) {
233 Log("SHUTTLE", Form("StoreLocally - Cannot activate local %s storage", cdbType));
84090f85 234 } else {
3301427a 235 result = AliCDBManager::Instance()->GetStorage(localUri)
84090f85 236 ->Put(object, id, metaData);
237 }
238
239 if(!result) {
240
9827400b 241 Log(fCurrentDetector, Form("StoreLocally - Can't store object <%s>!", id.ToString().Data()));
3301427a 242 }
2bb7b766 243
3301427a 244 return result;
245}
84090f85 246
3301427a 247//______________________________________________________________________________________________
248Bool_t AliShuttle::StoreOCDB()
249{
9827400b 250 //
251 // Called when preprocessor ends successfully or when previous storage attempt failed (kStoreError status)
252 // Calls underlying StoreOCDB(const char*) function twice, for OCDB and Reference storage.
253 // Then calls StoreRefFilesToGrid to store reference files.
254 //
255
5e993b6f 256 UpdateShuttleStatus(AliShuttleStatus::kStoreStarted);
257
9827400b 258 if (fTestMode & kErrorGrid)
259 {
260 Log("SHUTTLE", "StoreOCDB - In TESTMODE - Simulating error while storing in the Grid");
261 Log(fCurrentDetector, "StoreOCDB - In TESTMODE - Simulating error while storing in the Grid");
262 return kFALSE;
263 }
264
c88ad5db 265 Log("SHUTTLE","StoreOCDB - Storing OCDB data ...");
5e993b6f 266 Int_t resultCDB = StoreOCDB(fgkMainCDB);
86aa42c3 267
c88ad5db 268 Log("SHUTTLE","StoreOCDB - Storing reference data ...");
5e993b6f 269 Int_t resultRef = StoreOCDB(fgkMainRefStorage);
9827400b 270
c88ad5db 271 Log("SHUTTLE","StoreOCDB - Storing reference files ...");
272 Bool_t resultRefFiles = CopyFilesToGrid("reference");
273
274 Bool_t resultMetadata = kTRUE;
275 if(fCurrentDetector == "GRP")
276 {
277 Log("StoreOCDB - SHUTTLE","Storing Run Metadata file ...");
278 resultMetadata = CopyFilesToGrid("metadata");
279 }
9827400b 280
5e993b6f 281 Int_t storeResult = 0;
282
283 if (resultCDB < 0 || resultRef < 0 || resultRefFiles == kFALSE || resultMetadata == kFALSE)
284 storeResult = -1;
285 else if (resultCDB > 0 || resultRef > 0)
286 storeResult = 1;
287
288 if (storeResult < 0)
289 {
290 Log("SHUTTLE",
291 Form("\t\t\t****** run %d - %s: STORAGE ERROR ******",
292 GetCurrentRun(), fCurrentDetector.Data()));
293 UpdateShuttleStatus(AliShuttleStatus::kStoreError);
294 }
295 else if (storeResult > 0)
296 {
297 Log("SHUTTLE",
298 Form("\t\t\t****** run %d - %s: STORAGE DELAYED ******",
299 GetCurrentRun(), fCurrentDetector.Data()));
300 UpdateShuttleStatus(AliShuttleStatus::kStoreDelayed);
301 }
302 else if (storeResult == 0)
303 {
304 Log("SHUTTLE",
305 Form("\t\t\t****** run %d - %s: DONE ******",
306 GetCurrentRun(), fCurrentDetector.Data()));
307 UpdateShuttleStatus(AliShuttleStatus::kDone);
308 UpdateShuttleLogbook(fCurrentDetector, "DONE");
309 }
310
311 return (storeResult == 0);
3301427a 312}
313
314//______________________________________________________________________________________________
5e993b6f 315Int_t AliShuttle::StoreOCDB(const TString& gridURI)
3301427a 316{
317 //
318 // Called by StoreOCDB(), performs actual storage to the main OCDB and reference storages (Grid)
319 //
5e993b6f 320 // Return code:
321 // -2 initialization error
322 // -1 storage error
323 // 0 success
324 // 1 storage delayed (e.g. previous unprocessed runs)
325 //
3301427a 326
327 TObjArray* gridIds=0;
328
329 Bool_t result = kTRUE;
5e993b6f 330 Bool_t delayed = kFALSE;
3301427a 331
332 const char* type = 0;
333 TString localURI;
334 if(gridURI == fgkMainCDB) {
335 type = "OCDB";
336 localURI = fgkLocalCDB;
337 } else if(gridURI == fgkMainRefStorage) {
338 type = "reference";
339 localURI = fgkLocalRefStorage;
340 } else {
341 AliError(Form("Invalid storage URI: %s", gridURI.Data()));
5e993b6f 342 return -2;
3301427a 343 }
344
345 AliCDBManager* man = AliCDBManager::Instance();
346
347 AliCDBStorage *gridSto = man->GetStorage(gridURI);
348 if(!gridSto) {
349 Log("SHUTTLE",
350 Form("StoreOCDB - cannot activate main %s storage", type));
5e993b6f 351 return -2;
3301427a 352 }
353
354 gridIds = gridSto->GetQueryCDBList();
355
356 // get objects previously stored in local CDB
357 AliCDBStorage *localSto = man->GetStorage(localURI);
358 if(!localSto) {
359 Log("SHUTTLE",
360 Form("StoreOCDB - cannot activate local %s storage", type));
5e993b6f 361 return -2;
3301427a 362 }
363 AliCDBPath aPath(GetOfflineDetName(fCurrentDetector.Data()),"*","*");
364 // Local objects were stored with current run as Grid version!
365 TList* localEntries = localSto->GetAll(aPath.GetPath(), GetCurrentRun(), GetCurrentRun());
366 localEntries->SetOwner(1);
367
368 // loop on local stored objects
369 TIter localIter(localEntries);
370 AliCDBEntry *aLocEntry = 0;
371 while((aLocEntry = dynamic_cast<AliCDBEntry*> (localIter.Next()))){
372 aLocEntry->SetOwner(1);
373 AliCDBId aLocId = aLocEntry->GetId();
374 aLocEntry->SetVersion(-1);
375 aLocEntry->SetSubVersion(-1);
376
377 // If local object is valid up to infinity we store it only if it is
378 // the first unprocessed run!
379 if (aLocId.GetLastRun() == AliCDBRunRange::Infinity() &&
380 !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
381 {
382 Log("SHUTTLE", Form("StoreOCDB - %s: object %s has validity infinite but "
383 "there are previous unprocessed runs!",
384 fCurrentDetector.Data(), aLocId.GetPath().Data()));
5e993b6f 385 Log(fCurrentDetector.Data(), Form("StoreOCDB - %s: object %s has validity infinite but "
386 "there are previous unprocessed runs!",
387 fCurrentDetector.Data(), aLocId.GetPath().Data()));
388 delayed = kTRUE;
3301427a 389 continue;
390 }
391
392 // loop on Grid valid Id's
393 Bool_t store = kTRUE;
394 TIter gridIter(gridIds);
395 AliCDBId* aGridId = 0;
396 while((aGridId = dynamic_cast<AliCDBId*> (gridIter.Next()))){
397 if(aGridId->GetPath() != aLocId.GetPath()) continue;
398 // skip all objects valid up to infinity
399 if(aGridId->GetLastRun() == AliCDBRunRange::Infinity()) continue;
400 // if we get here, it means there's already some more recent object stored on Grid!
401 store = kFALSE;
402 break;
403 }
404
405 // If we get here, the file can be stored!
406 Bool_t storeOk = gridSto->Put(aLocEntry);
407 if(!store || storeOk){
408
409 if (!store)
410 {
411 Log(fCurrentDetector.Data(),
412 Form("StoreOCDB - A more recent object already exists in %s storage: <%s>",
413 type, aGridId->ToString().Data()));
414 } else {
415 Log("SHUTTLE",
416 Form("StoreOCDB - Object <%s> successfully put into %s storage",
417 aLocId.ToString().Data(), type));
2d9019b4 418 Log(fCurrentDetector.Data(),
419 Form("StoreOCDB - Object <%s> successfully put into %s storage",
420 aLocId.ToString().Data(), type));
3301427a 421 }
84090f85 422
3301427a 423 // removing local filename...
424 TString filename;
425 localSto->IdToFilename(aLocId, filename);
c88ad5db 426 Log("SHUTTLE", Form("StoreOCDB - Removing local file %s", filename.Data()));
3301427a 427 RemoveFile(filename.Data());
428 continue;
429 } else {
430 Log("SHUTTLE",
431 Form("StoreOCDB - Grid %s storage of object <%s> failed",
432 type, aLocId.ToString().Data()));
2d9019b4 433 Log(fCurrentDetector.Data(),
434 Form("StoreOCDB - Grid %s storage of object <%s> failed",
435 type, aLocId.ToString().Data()));
3301427a 436 result = kFALSE;
b948db8d 437 }
438 }
3301427a 439 localEntries->Clear();
2bb7b766 440
5e993b6f 441 Int_t returnCode = 0;
442
443 if (result == kFALSE)
444 returnCode = -1;
445 else if (delayed != kFALSE)
446 returnCode = 1;
447
448 Log("SHUTTLE", Form("StoreOCDB - Returning with %d (result = %d, delayed = %d)", returnCode, result, delayed));
449 Log(fCurrentDetector.Data(), Form("StoreOCDB - Returning with %d (result = %d, delayed = %d)", returnCode, result, delayed));
450
451 return returnCode;
3301427a 452}
453
546242fb 454//______________________________________________________________________________________________
455Bool_t AliShuttle::CleanReferenceStorage(const char* detector)
456{
2d9019b4 457 // clears the directory used to store reference files of a given subdetector
546242fb 458
459 AliCDBManager* man = AliCDBManager::Instance();
460 AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
2d9019b4 461 TString localBaseFolder = sto->GetBaseFolder();
462
463 TString targetDir = GetRefFilePrefix(localBaseFolder.Data(), detector);
464
d524ade6 465 Log("SHUTTLE", Form("CleanReferenceStorage - Cleaning %s", targetDir.Data()));
2d9019b4 466
467 TString begin;
468 begin.Form("%d_", GetCurrentRun());
469
470 TSystemDirectory* baseDir = new TSystemDirectory("/", targetDir);
471 if (!baseDir)
472 return kTRUE;
473
474 TList* dirList = baseDir->GetListOfFiles();
475 delete baseDir;
476
477 if (!dirList) return kTRUE;
478
479 if (dirList->GetEntries() < 3)
480 {
481 delete dirList;
482 return kTRUE;
483 }
484
485 Int_t nDirs = 0, nDel = 0;
486 TIter dirIter(dirList);
487 TSystemFile* entry = 0;
546242fb 488
2d9019b4 489 Bool_t success = kTRUE;
546242fb 490
2d9019b4 491 while ((entry = dynamic_cast<TSystemFile*> (dirIter.Next())))
492 {
493 if (entry->IsDirectory())
494 continue;
495
496 TString fileName(entry->GetName());
497 if (!fileName.BeginsWith(begin))
498 continue;
499
500 nDirs++;
501
502 // delete file
503 Int_t result = gSystem->Unlink(fileName.Data());
504
505 if (result)
506 {
d524ade6 507 Log("SHUTTLE", Form("CleanReferenceStorage - Could not delete file %s!", fileName.Data()));
2d9019b4 508 success = kFALSE;
509 } else {
510 nDel++;
511 }
512 }
513
514 if(nDirs > 0)
515 Log("SHUTTLE", Form("CleanReferenceStorage - %d (over %d) reference files in folder %s were deleted.",
516 nDel, nDirs, targetDir.Data()));
517
518
519 delete dirList;
520 return success;
521
522
523
524
525
546242fb 526
527 Int_t result = gSystem->GetPathInfo(targetDir, 0, (Long64_t*) 0, 0, 0);
528 if (result == 0)
529 {
530 // delete directory
d524ade6 531 result = gSystem->Exec(Form("rm -rf %s", targetDir.Data()));
546242fb 532 if (result != 0)
533 {
d524ade6 534 Log("SHUTTLE", Form("CleanReferenceStorage - Could not clean directory %s", targetDir.Data()));
546242fb 535 return kFALSE;
536 }
537 }
538
539 result = gSystem->mkdir(targetDir, kTRUE);
540 if (result != 0)
541 {
c88ad5db 542 Log("SHUTTLE", Form("CleanReferenceStorage - Error creating base directory %s", targetDir.Data()));
546242fb 543 return kFALSE;
544 }
545
546 return kTRUE;
547}
548
9827400b 549//______________________________________________________________________________________________
550Bool_t AliShuttle::StoreReferenceFile(const char* detector, const char* localFile, const char* gridFileName)
551{
552 //
3c2a21c8 553 // Stores reference file directly (without opening it). This function stores the file locally.
9827400b 554 //
3c2a21c8 555 // The file is stored under the following location:
556 // <base folder of local reference storage>/<DET>/<RUN#>_<gridFileName>
557 // where <gridFileName> is the second parameter given to the function
558 //
9827400b 559
560 if (fTestMode & kErrorStorage)
561 {
562 Log(fCurrentDetector, "StoreReferenceFile - In TESTMODE - Simulating error while storing locally");
563 return kFALSE;
564 }
565
566 AliCDBManager* man = AliCDBManager::Instance();
567 AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
568
569 TString localBaseFolder = sto->GetBaseFolder();
570
d524ade6 571 TString target = GetRefFilePrefix(localBaseFolder.Data(), detector);
572 target.Append(Form("/%d_%s", GetCurrentRun(), gridFileName));
9827400b 573
d524ade6 574 return CopyFileLocally(localFile, target);
c88ad5db 575}
576
577//______________________________________________________________________________________________
578Bool_t AliShuttle::StoreRunMetadataFile(const char* localFile, const char* gridFileName)
579{
580 //
581 // Stores Run metadata file to the Grid, in the run folder
582 //
583 // Only GRP can call this function.
584
585 if (fTestMode & kErrorStorage)
586 {
587 Log(fCurrentDetector, "StoreRunMetaDataFile - In TESTMODE - Simulating error while storing locally");
588 return kFALSE;
589 }
590
591 AliCDBManager* man = AliCDBManager::Instance();
592 AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
593
594 TString localBaseFolder = sto->GetBaseFolder();
595
596 // Build Run level folder
955d2abc 597 // folder = /alice/data/year/lhcPeriod/runNb/raw
c88ad5db 598
c88ad5db 599
675f64cd 600 TString lhcPeriod = GetLHCPeriod();
c88ad5db 601 if (lhcPeriod.Length() == 0)
602 {
603 Log("SHUTTLE","StoreRunMetaDataFile - LHCPeriod not found in logbook!");
604 return 0;
605 }
a1c5723b 606
607 // TODO partitions with one detector only write data into LHCperiod_DET
2694f253 608 TString partition = GetRunParameter("detector");
a1c5723b 609
610 if (partition.Length() > 0 && partition != "ALICE")
611 {
612 lhcPeriod.Append(Form("_%s", partition.Data()));
613 Log(fCurrentDetector, Form("Run data tags merged file will be written in %s",
614 lhcPeriod.Data()));
615 }
8884d71e 616
955d2abc 617 TString target = Form("%s/GRP/RunMetadata/alice/data/%d/%s/%09d/raw/%s",
675f64cd 618 localBaseFolder.Data(), GetCurrentYear(),
d524ade6 619 lhcPeriod.Data(), GetCurrentRun(), gridFileName);
c88ad5db 620
d524ade6 621 return CopyFileLocally(localFile, target);
c88ad5db 622}
623
624//______________________________________________________________________________________________
d524ade6 625Bool_t AliShuttle::CopyFileLocally(const char* localFile, const TString& target)
c88ad5db 626{
627 //
628 // Stores file locally. Called by StoreReferenceFile and StoreRunMetadataFile
d524ade6 629 // Files are temporarily stored in the local reference storage. When the preprocessor
630 // finishes, the Shuttle calls CopyFilesToGrid to transfer the files to AliEn
631 // (in reference or run level folders)
c88ad5db 632 //
633
d524ade6 634 TString targetDir(target(0, target.Last('/')));
635
636 //try to open base dir folder, if it does not exist
2d9019b4 637 void* dir = gSystem->OpenDirectory(targetDir.Data());
638 if (dir == NULL) {
639 if (gSystem->mkdir(targetDir.Data(), kTRUE)) {
a1c5723b 640 Log("SHUTTLE", Form("CopyFileLocally - Can't open directory <%s>", targetDir.Data()));
2d9019b4 641 return kFALSE;
642 }
643
644 } else {
645 gSystem->FreeDirectory(dir);
646 }
9827400b 647
7d43a416 648 Int_t result = 0;
649
650 result = gSystem->GetPathInfo(localFile, 0, (Long64_t*) 0, 0, 0);
9827400b 651 if (result)
652 {
a1c5723b 653 Log("SHUTTLE", Form("CopyFileLocally - %s does not exist", localFile));
546242fb 654 return kFALSE;
9827400b 655 }
546242fb 656
7d43a416 657 result = gSystem->GetPathInfo(target, 0, (Long64_t*) 0, 0, 0);
658 if (!result)
659 {
a1c5723b 660 Log("SHUTTLE", Form("CopyFileLocally - target file %s already exist, removing...", target.Data()));
7d43a416 661 if (gSystem->Unlink(target.Data()))
662 {
a1c5723b 663 Log("SHUTTLE", Form("CopyFileLocally - Could not remove existing target file %s!", target.Data()));
7d43a416 664 return kFALSE;
665 }
666 }
667
9827400b 668 result = gSystem->CopyFile(localFile, target);
669
670 if (result == 0)
671 {
a1c5723b 672 Log("SHUTTLE", Form("CopyFileLocally - File %s stored locally to %s", localFile, target.Data()));
9827400b 673 return kTRUE;
674 }
675 else
676 {
a1c5723b 677 Log("SHUTTLE", Form("CopyFileLocally - Could not store file %s to %s! Error code = %d",
546242fb 678 localFile, target.Data(), result));
9827400b 679 return kFALSE;
680 }
c88ad5db 681
682
683
9827400b 684}
685
686//______________________________________________________________________________________________
c88ad5db 687Bool_t AliShuttle::CopyFilesToGrid(const char* type)
9827400b 688{
689 //
c88ad5db 690 // Transfers local files to the Grid. Local files can be reference files
691 // or run metadata file (from GRP only).
9827400b 692 //
c88ad5db 693 // According to the type (ref, metadata) the files are stored under the following location:
694 // ref --> <base folder of reference storage>/<DET>/<RUN#>_<gridFileName>
695 // metadata --> <run data folder>/<MetadataFileName>
86aa42c3 696 //
c88ad5db 697
9827400b 698 AliCDBManager* man = AliCDBManager::Instance();
699 AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
700 if (!sto)
701 return kFALSE;
702 TString localBaseFolder = sto->GetBaseFolder();
9827400b 703
c88ad5db 704 TString dir;
705 TString alienDir;
9827400b 706 TString begin;
9827400b 707
c88ad5db 708 if (strcmp(type, "reference") == 0)
709 {
710 dir = GetRefFilePrefix(localBaseFolder.Data(), fCurrentDetector.Data());
711 AliCDBStorage* gridSto = man->GetStorage(fgkMainRefStorage);
712 if (!gridSto)
713 return kFALSE;
714 TString gridBaseFolder = gridSto->GetBaseFolder();
715 alienDir = GetRefFilePrefix(gridBaseFolder.Data(), fCurrentDetector.Data());
716 begin = Form("%d_", GetCurrentRun());
717 }
718 else if (strcmp(type, "metadata") == 0)
719 {
c88ad5db 720
675f64cd 721 TString lhcPeriod = GetLHCPeriod();
c88ad5db 722
723 if (lhcPeriod.Length() == 0)
724 {
725 Log("SHUTTLE","CopyFilesToGrid - LHCPeriod not found in logbook!");
726 return 0;
727 }
728
a1c5723b 729 // TODO partitions with one detector only write data into LHCperiod_DET
2694f253 730 TString partition = GetRunParameter("detector");
a1c5723b 731
732 if (partition.Length() > 0 && partition != "ALICE")
733 {
734 lhcPeriod.Append(Form("_%s", partition.Data()));
735 }
736
955d2abc 737 dir = Form("%s/GRP/RunMetadata/alice/data/%d/%s/%09d/raw",
675f64cd 738 localBaseFolder.Data(), GetCurrentYear(),
c88ad5db 739 lhcPeriod.Data(), GetCurrentRun());
675f64cd 740 alienDir = dir(dir.Index("/alice/data/"), dir.Length());
741
c88ad5db 742 begin = "";
743 }
744 else
745 {
746 Log("SHUTTLE", "CopyFilesToGrid - Unexpected: type label must be reference or metadata!");
747 return kFALSE;
748 }
749
9827400b 750 TSystemDirectory* baseDir = new TSystemDirectory("/", dir);
3d8bc902 751 if (!baseDir)
752 return kTRUE;
753
2d9019b4 754 TList* dirList = baseDir->GetListOfFiles();
755 delete baseDir;
756
757 if (!dirList) return kTRUE;
758
759 if (dirList->GetEntries() < 3)
3d8bc902 760 {
2d9019b4 761 delete dirList;
9827400b 762 return kTRUE;
3d8bc902 763 }
2d9019b4 764
546242fb 765 if (!gGrid)
766 {
c88ad5db 767 Log("SHUTTLE", "CopyFilesToGrid - Connection to Grid failed: Cannot continue!");
2d9019b4 768 delete dirList;
546242fb 769 return kFALSE;
770 }
771
2d9019b4 772 Int_t nDirs = 0, nTransfer = 0;
773 TIter dirIter(dirList);
774 TSystemFile* entry = 0;
775
9827400b 776 Bool_t success = kTRUE;
3d8bc902 777 Bool_t first = kTRUE;
9827400b 778
2d9019b4 779 while ((entry = dynamic_cast<TSystemFile*> (dirIter.Next())))
780 {
9827400b 781 if (entry->IsDirectory())
782 continue;
783
784 TString fileName(entry->GetName());
785 if (!fileName.BeginsWith(begin))
786 continue;
787
2d9019b4 788 nDirs++;
789
3d8bc902 790 if (first)
791 {
792 first = kFALSE;
c88ad5db 793 // check that folder exists, otherwise create it
3d8bc902 794 TGridResult* result = gGrid->Ls(alienDir.Data(), "a");
795
796 if (!result)
2d9019b4 797 {
798 delete dirList;
3d8bc902 799 return kFALSE;
2d9019b4 800 }
3d8bc902 801
546242fb 802 if (!result->GetFileName(1)) // TODO: It looks like element 0 is always 0!!
3d8bc902 803 {
675f64cd 804 // TODO It does not work currently! Bug in TAliEn::Mkdir
805 // TODO Manually fixed in local root v5-16-00
c88ad5db 806 if (!gGrid->Mkdir(alienDir.Data(),"-p",0))
3d8bc902 807 {
c88ad5db 808 Log("SHUTTLE", Form("CopyFilesToGrid - Cannot create directory %s",
3d8bc902 809 alienDir.Data()));
2d9019b4 810 delete dirList;
3d8bc902 811 return kFALSE;
546242fb 812 } else {
c88ad5db 813 Log("SHUTTLE",Form("CopyFilesToGrid - Folder %s created", alienDir.Data()));
3d8bc902 814 }
815
546242fb 816 } else {
c88ad5db 817 Log("SHUTTLE",Form("CopyFilesToGrid - Folder %s found", alienDir.Data()));
3d8bc902 818 }
819 }
820
9827400b 821 TString fullLocalPath;
822 fullLocalPath.Form("%s/%s", dir.Data(), fileName.Data());
823
824 TString fullGridPath;
825 fullGridPath.Form("alien://%s/%s", alienDir.Data(), fileName.Data());
826
a986b218 827 Bool_t result = TFile::Cp(fullLocalPath, fullGridPath);
9827400b 828
829 if (result)
830 {
c88ad5db 831 Log("SHUTTLE", Form("CopyFilesToGrid - Copying local file %s to %s succeeded!",
832 fullLocalPath.Data(), fullGridPath.Data()));
9827400b 833 RemoveFile(fullLocalPath);
2d9019b4 834 nTransfer++;
9827400b 835 }
836 else
837 {
c88ad5db 838 Log("SHUTTLE", Form("CopyFilesToGrid - Copying local file %s to %s FAILED!",
839 fullLocalPath.Data(), fullGridPath.Data()));
9827400b 840 success = kFALSE;
841 }
842 }
2d9019b4 843
c88ad5db 844 Log("SHUTTLE", Form("CopyFilesToGrid - %d (over %d) files in folder %s copied to Grid.",
845 nTransfer, nDirs, dir.Data()));
2d9019b4 846
847
848 delete dirList;
9827400b 849 return success;
850}
851
2d9019b4 852//______________________________________________________________________________________________
853const char* AliShuttle::GetRefFilePrefix(const char* base, const char* detector)
854{
855 //
856 // Get folder name of reference files
857 //
858
859 TString offDetStr(GetOfflineDetName(detector));
860 TString dir;
861 if (offDetStr == "ITS" || offDetStr == "MUON" || offDetStr == "PHOS")
862 {
863 dir.Form("%s/%s/%s", base, offDetStr.Data(), detector);
864 } else {
865 dir.Form("%s/%s", base, offDetStr.Data());
866 }
867
868 return dir.Data();
869
870
871}
c88ad5db 872
3301427a 873//______________________________________________________________________________________________
874void AliShuttle::CleanLocalStorage(const TString& uri)
875{
9827400b 876 //
877 // Called in case the preprocessor is declared failed. Remove remaining objects from the local storages.
878 //
3301427a 879
880 const char* type = 0;
881 if(uri == fgkLocalCDB) {
882 type = "OCDB";
883 } else if(uri == fgkLocalRefStorage) {
546242fb 884 type = "Reference";
3301427a 885 } else {
886 AliError(Form("Invalid storage URI: %s", uri.Data()));
887 return;
888 }
889
890 AliCDBManager* man = AliCDBManager::Instance();
b948db8d 891
3301427a 892 // open local storage
893 AliCDBStorage *localSto = man->GetStorage(uri);
894 if(!localSto) {
895 Log("SHUTTLE",
896 Form("CleanLocalStorage - cannot activate local %s storage", type));
897 return;
898 }
899
900 TString filename(Form("%s/%s/*/Run*_v%d_s*.root",
546242fb 901 localSto->GetBaseFolder().Data(), GetOfflineDetName(fCurrentDetector.Data()), GetCurrentRun()));
3301427a 902
c88ad5db 903 AliDebug(2, Form("filename = %s", filename.Data()));
3301427a 904
c88ad5db 905 Log("SHUTTLE", Form("Removing remaining local files for run %d and detector %s ...",
3301427a 906 GetCurrentRun(), fCurrentDetector.Data()));
907
908 RemoveFile(filename.Data());
909
910}
911
912//______________________________________________________________________________________________
913void AliShuttle::RemoveFile(const char* filename)
914{
9827400b 915 //
916 // removes local file
917 //
3301427a 918
919 TString command(Form("rm -f %s", filename));
920
921 Int_t result = gSystem->Exec(command.Data());
922 if(result != 0)
923 {
924 Log("SHUTTLE", Form("RemoveFile - %s: Cannot remove file %s!",
925 fCurrentDetector.Data(), filename));
926 }
73abe331 927}
928
b948db8d 929//______________________________________________________________________________________________
5164a766 930AliShuttleStatus* AliShuttle::ReadShuttleStatus()
931{
9827400b 932 //
933 // Reads the AliShuttleStatus from the CDB
934 //
5164a766 935
2bb7b766 936 if (fStatusEntry){
937 delete fStatusEntry;
938 fStatusEntry = 0;
939 }
5164a766 940
10a5a932 941 fStatusEntry = AliCDBManager::Instance()->GetStorage(GetLocalCDB())
2bb7b766 942 ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), GetCurrentRun());
5164a766 943
2bb7b766 944 if (!fStatusEntry) return 0;
945 fStatusEntry->SetOwner(1);
5164a766 946
2bb7b766 947 AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
948 if (!status) {
949 AliError("Invalid object stored to CDB!");
950 return 0;
951 }
5164a766 952
2bb7b766 953 return status;
5164a766 954}
955
956//______________________________________________________________________________________________
7bfb2090 957Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
5164a766 958{
9827400b 959 //
960 // writes the status for one subdetector
961 //
2bb7b766 962
963 if (fStatusEntry){
964 delete fStatusEntry;
965 fStatusEntry = 0;
966 }
5164a766 967
2bb7b766 968 Int_t run = GetCurrentRun();
5164a766 969
2bb7b766 970 AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), run, run);
5164a766 971
2bb7b766 972 fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
973 fStatusEntry->SetOwner(1);
5164a766 974
2bb7b766 975 UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
7bfb2090 976
2bb7b766 977 if (!result) {
3301427a 978 Log("SHUTTLE", Form("WriteShuttleStatus - Failed for %s, run %d",
979 fCurrentDetector.Data(), run));
2bb7b766 980 return kFALSE;
981 }
e7f62f16 982
983 SendMLInfo();
7bfb2090 984
2bb7b766 985 return kTRUE;
5164a766 986}
987
988//______________________________________________________________________________________________
989void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
990{
9827400b 991 //
992 // changes the AliShuttleStatus for the given detector and run to the given status
993 //
5164a766 994
2bb7b766 995 if (!fStatusEntry){
996 AliError("UNEXPECTED: fStatusEntry empty");
997 return;
998 }
5164a766 999
2bb7b766 1000 AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
5164a766 1001
2bb7b766 1002 if (!status){
c88ad5db 1003 Log("SHUTTLE", "UpdateShuttleStatus - UNEXPECTED: status could not be read from current CDB entry");
2bb7b766 1004 return;
1005 }
5164a766 1006
2c15234c 1007 TString actionStr = Form("UpdateShuttleStatus - %s: Changing state from %s to %s",
eba76848 1008 fCurrentDetector.Data(),
36c99a6a 1009 status->GetStatusName(),
eba76848 1010 status->GetStatusName(newStatus));
cb343cfd 1011 Log("SHUTTLE", actionStr);
1012 SetLastAction(actionStr);
5164a766 1013
2bb7b766 1014 status->SetStatus(newStatus);
1015 if (increaseCount) status->IncreaseCount();
5164a766 1016
2bb7b766 1017 AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
e7f62f16 1018
1019 SendMLInfo();
5164a766 1020}
e7f62f16 1021
1022//______________________________________________________________________________________________
1023void AliShuttle::SendMLInfo()
1024{
1025 //
1026 // sends ML information about the current status of the current detector being processed
1027 //
1028
1029 AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
1030
1031 if (!status){
3301427a 1032 Log("SHUTTLE", "SendMLInfo - UNEXPECTED: status could not be read from current CDB entry");
e7f62f16 1033 return;
1034 }
1035
1036 TMonaLisaText mlStatus(Form("%s_status", fCurrentDetector.Data()), status->GetStatusName());
1037 TMonaLisaValue mlRetryCount(Form("%s_count", fCurrentDetector.Data()), status->GetCount());
1038
1039 TList mlList;
1040 mlList.Add(&mlStatus);
1041 mlList.Add(&mlRetryCount);
1042
ee6f7523 1043 TString mlID;
1044 mlID.Form("%d", GetCurrentRun());
1045 fMonaLisa->SendParameters(&mlList, mlID);
e7f62f16 1046}
1047
5164a766 1048//______________________________________________________________________________________________
1049Bool_t AliShuttle::ContinueProcessing()
1050{
9827400b 1051 // this function reads the AliShuttleStatus information from CDB and
1052 // checks if the processing should be continued
1053 // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
2bb7b766 1054
926cbc0e 1055 if (!fConfig->HostProcessDetector(fCurrentDetector))
1056 return kFALSE;
57c1a579 1057
1058 AliPreprocessor* aPreprocessor =
1059 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
1060 if (!aPreprocessor)
1061 {
c88ad5db 1062 Log("SHUTTLE", Form("ContinueProcessing - %s: no preprocessor registered", fCurrentDetector.Data()));
57c1a579 1063 return kFALSE;
1064 }
1065
2bb7b766 1066 AliShuttleLogbookEntry::Status entryStatus =
eba76848 1067 fLogbookEntry->GetDetectorStatus(fCurrentDetector);
2bb7b766 1068
5e993b6f 1069 if (entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
c88ad5db 1070 Log("SHUTTLE", Form("ContinueProcessing - %s is %s",
2bb7b766 1071 fCurrentDetector.Data(),
1072 fLogbookEntry->GetDetectorStatusName(entryStatus)));
1073 return kFALSE;
1074 }
1075
1076 // if we get here, according to Shuttle logbook subdetector is in UNPROCESSED state
be48e3ea 1077
1078 // check if current run is first unprocessed run for current detector
1079 if (fConfig->StrictRunOrder(fCurrentDetector) &&
1080 !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
1081 {
86aa42c3 1082 if (fTestMode == kNone)
1083 {
c88ad5db 1084 Log("SHUTTLE", Form("ContinueProcessing - %s requires strict run ordering"
1085 " but this is not the first unprocessed run!"));
86aa42c3 1086 return kFALSE;
1087 }
1088 else
1089 {
c88ad5db 1090 Log("SHUTTLE", Form("ContinueProcessing - In TESTMODE - "
1091 "Although %s requires strict run ordering "
1092 "and this is not the first unprocessed run, "
1093 "the SHUTTLE continues"));
86aa42c3 1094 }
be48e3ea 1095 }
1096
926cbc0e 1097 // Is the subdetector processed first time for this run?
1098 fFirstProcessing = kFALSE;
1099
2bb7b766 1100 AliShuttleStatus* status = ReadShuttleStatus();
1101 if (!status) {
1102 // first time
1103 Log("SHUTTLE", Form("ContinueProcessing - %s: Processing first time",
1104 fCurrentDetector.Data()));
1105 status = new AliShuttleStatus(AliShuttleStatus::kStarted);
926cbc0e 1106 fFirstProcessing = kTRUE;
2bb7b766 1107 return WriteShuttleStatus(status);
1108 }
1109
1110 // The following two cases shouldn't happen if Shuttle Logbook was correctly updated.
1111 // If it happens it may mean Logbook updating failed... let's do it now!
1112 if (status->GetStatus() == AliShuttleStatus::kDone ||
1113 status->GetStatus() == AliShuttleStatus::kFailed){
1114 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s. Updating Shuttle Logbook",
1115 fCurrentDetector.Data(),
1116 status->GetStatusName(status->GetStatus())));
1117 UpdateShuttleLogbook(fCurrentDetector.Data(),
1118 status->GetStatusName(status->GetStatus()));
1119 return kFALSE;
1120 }
1121
5e993b6f 1122 if (status->GetStatus() == AliShuttleStatus::kStoreStarted || status->GetStatus() == AliShuttleStatus::kStoreDelayed ||status->GetStatus() == AliShuttleStatus::kStoreError) {
2bb7b766 1123 Log("SHUTTLE",
c88ad5db 1124 Form("ContinueProcessing - %s: Grid storage of one or more "
1125 "objects failed. Trying again now",
2bb7b766 1126 fCurrentDetector.Data()));
5e993b6f 1127 StoreOCDB();
2bb7b766 1128 return kFALSE;
1129 }
1130
1131 // if we get here, there is a restart
57c1a579 1132 Bool_t cont = kFALSE;
2bb7b766 1133
1134 // abort conditions
cb343cfd 1135 if (status->GetCount() >= fConfig->GetMaxRetries()) {
57c1a579 1136 Log("SHUTTLE", Form("ContinueProcessing - %s failed %d times in status %s - "
1137 "Updating Shuttle Logbook", fCurrentDetector.Data(),
2bb7b766 1138 status->GetCount(), status->GetStatusName()));
1139 UpdateShuttleLogbook(fCurrentDetector.Data(), "FAILED");
e7f62f16 1140 UpdateShuttleStatus(AliShuttleStatus::kFailed);
3301427a 1141
1142 // there may still be objects in local OCDB and reference storage
1143 // and FXS databases may be not updated: do it now!
9827400b 1144
1145 // TODO Currently disabled, we want to keep files in case of failure!
1146 // CleanLocalStorage(fgkLocalCDB);
1147 // CleanLocalStorage(fgkLocalRefStorage);
1148 // UpdateTableFailCase();
1149
1150 // Send mail to detector expert!
c88ad5db 1151 Log("SHUTTLE", Form("ContinueProcessing - Sending mail to %s expert...",
926cbc0e 1152 fCurrentDetector.Data()));
9827400b 1153 if (!SendMail())
1154 Log("SHUTTLE", Form("ContinueProcessing - Could not send mail to %s expert",
926cbc0e 1155 fCurrentDetector.Data()));
3301427a 1156
57c1a579 1157 } else {
1158 Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. "
1159 "Aborted before with %s. Retry number %d.", fCurrentDetector.Data(),
1160 status->GetStatusName(), status->GetCount()));
9827400b 1161 Bool_t increaseCount = kTRUE;
c88ad5db 1162 if (status->GetStatus() == AliShuttleStatus::kDCSError ||
1163 status->GetStatus() == AliShuttleStatus::kDCSStarted)
1164 increaseCount = kFALSE;
675f64cd 1165
9827400b 1166 UpdateShuttleStatus(AliShuttleStatus::kStarted, increaseCount);
57c1a579 1167 cont = kTRUE;
2bb7b766 1168 }
1169
57c1a579 1170 return cont;
5164a766 1171}
1172
1173//______________________________________________________________________________________________
2bb7b766 1174Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
58bc3020 1175{
73abe331 1176 //
b948db8d 1177 // Makes data retrieval for all detectors in the configuration.
2bb7b766 1178 // entry: Shuttle logbook entry, contains run paramenters and status of detectors
1179 // (Unprocessed, Inactive, Failed or Done).
d477ad88 1180 // Returns kFALSE in case of error occured and kTRUE otherwise
73abe331 1181 //
1182
9827400b 1183 if (!entry) return kFALSE;
2bb7b766 1184
1185 fLogbookEntry = entry;
1186
c88ad5db 1187 Log("SHUTTLE", Form("\t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^*",
9827400b 1188 GetCurrentRun()));
2bb7b766 1189
e7f62f16 1190 // Send the information to ML
1191 TMonaLisaText mlStatus("SHUTTLE_status", "Processing");
9827400b 1192 TMonaLisaText mlRunType("SHUTTLE_runtype", Form("%s (%s)", entry->GetRunType(), entry->GetRunParameter("log")));
e7f62f16 1193
1194 TList mlList;
1195 mlList.Add(&mlStatus);
9827400b 1196 mlList.Add(&mlRunType);
e7f62f16 1197
ee6f7523 1198 TString mlID;
1199 mlID.Form("%d", GetCurrentRun());
1200 fMonaLisa->SendParameters(&mlList, mlID);
3301427a 1201
9827400b 1202 if (fLogbookEntry->IsDone())
1203 {
1204 Log("SHUTTLE","Process - Shuttle is already DONE. Updating logbook");
1205 UpdateShuttleLogbook("shuttle_done");
1206 fLogbookEntry = 0;
1207 return kTRUE;
1208 }
1209
1210 // read test mode if flag is set
1211 if (fReadTestMode)
1212 {
3d8bc902 1213 fTestMode = kNone;
9827400b 1214 TString logEntry(entry->GetRunParameter("log"));
1215 //printf("log entry = %s\n", logEntry.Data());
1216 TString searchStr("Testmode: ");
1217 Int_t pos = logEntry.Index(searchStr.Data());
1218 //printf("%d\n", pos);
1219 if (pos >= 0)
1220 {
1221 TSubString subStr = logEntry(pos + searchStr.Length(), logEntry.Length());
1222 //printf("%s\n", subStr.String().Data());
1223 TString newStr(subStr.Data());
1224 TObjArray* token = newStr.Tokenize(' ');
1225 if (token)
1226 {
1227 //token->Print();
1228 TObjString* tmpStr = dynamic_cast<TObjString*> (token->First());
1229 if (tmpStr)
1230 {
1231 Int_t testMode = tmpStr->String().Atoi();
1232 if (testMode > 0)
1233 {
c88ad5db 1234 Log("SHUTTLE", Form("Process - Enabling test mode %d", testMode));
9827400b 1235 SetTestMode((TestMode) testMode);
1236 }
1237 }
1238 delete token;
1239 }
1240 }
1241 }
c88ad5db 1242
eba76848 1243 fLogbookEntry->Print("all");
57f50b3c 1244
1245 // Initialization
d477ad88 1246 Bool_t hasError = kFALSE;
5164a766 1247
675f64cd 1248 // Set the CDB and Reference folders according to the year and LHC period
1249 TString lhcPeriod(GetLHCPeriod());
1250 if (lhcPeriod.Length() == 0)
1251 {
7d4cf768 1252 Log("SHUTTLE","Process - LHCPeriod not found in logbook!");
1253 return 0;
675f64cd 1254 }
1255
1256 if (fgkMainCDB.Length() == 0)
1257 fgkMainCDB = Form("alien://folder=/alice/data/%d/%s/OCDB?user=alidaq?cacheFold=/tmp/OCDBCache",
1258 GetCurrentYear(), lhcPeriod.Data());
1259
1260 if (fgkMainRefStorage.Length() == 0)
1261 fgkMainRefStorage = Form("alien://folder=/alice/data/%d/%s/Reference?user=alidaq?cacheFold=/tmp/OCDBCache",
1262 GetCurrentYear(), lhcPeriod.Data());
1263
57f50b3c 1264 // Loop on detectors in the configuration
b948db8d 1265 TIter iter(fConfig->GetDetectors());
2bb7b766 1266 TObjString* aDetector = 0;
b948db8d 1267
4508ab8b 1268 Bool_t first = kTRUE;
1269
be48e3ea 1270 while ((aDetector = (TObjString*) iter.Next()))
1271 {
7bfb2090 1272 fCurrentDetector = aDetector->String();
5164a766 1273
5e993b6f 1274 if (ContinueProcessing() == kFALSE)
1275 continue;
4508ab8b 1276
1277 if (first)
1278 {
1279 // only read QueryCDB when needed and only once
1280 AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
1281 if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
1282 AliCDBStorage *mainRefSto = AliCDBManager::Instance()->GetStorage(fgkMainRefStorage);
1283 if(mainRefSto) mainRefSto->QueryCDB(GetCurrentRun());
1284 first = kFALSE;
1285 }
9e080f92 1286
c88ad5db 1287 Log("SHUTTLE", Form("\t\t\t****** run %d - %s: START ******",
2bb7b766 1288 GetCurrentRun(), aDetector->GetName()));
1289
9d733021 1290 for(Int_t iSys=0;iSys<3;iSys++) fFXSCalled[iSys]=kFALSE;
1291
c88ad5db 1292 Log(fCurrentDetector.Data(), "Process - Starting processing");
85a80aa9 1293
be48e3ea 1294 Int_t pid = fork();
1295
1296 if (pid < 0)
1297 {
c88ad5db 1298 Log("SHUTTLE", "Process - ERROR: Forking failed");
be48e3ea 1299 }
1300 else if (pid > 0)
1301 {
1302 // parent
c88ad5db 1303 Log("SHUTTLE", Form("Process - In parent process of %d - %s: Starting monitoring",
be48e3ea 1304 GetCurrentRun(), aDetector->GetName()));
1305
1306 Long_t begin = time(0);
1307
1308 int status; // to be used with waitpid, on purpose an int (not Int_t)!
1309 while (waitpid(pid, &status, WNOHANG) == 0)
1310 {
1311 Long_t expiredTime = time(0) - begin;
1312
1313 if (expiredTime > fConfig->GetPPTimeOut())
1314 {
9827400b 1315 TString tmp;
c88ad5db 1316 tmp.Form("Process - Process of %s time out. "
1317 "Run time: %d seconds. Killing...",
1318 fCurrentDetector.Data(), expiredTime);
9827400b 1319 Log("SHUTTLE", tmp);
1320 Log(fCurrentDetector, tmp);
be48e3ea 1321
1322 kill(pid, 9);
1323
3301427a 1324 UpdateShuttleStatus(AliShuttleStatus::kPPTimeOut);
be48e3ea 1325 hasError = kTRUE;
1326
1327 gSystem->Sleep(1000);
1328 }
1329 else
1330 {
be48e3ea 1331 gSystem->Sleep(1000);
9827400b 1332
1333 TString checkStr;
1334 checkStr.Form("ps -o vsize --pid %d | tail -n 1", pid);
1335 FILE* pipe = gSystem->OpenPipe(checkStr, "r");
1336 if (!pipe)
1337 {
c88ad5db 1338 Log("SHUTTLE", Form("Process - Error: "
1339 "Could not open pipe to %s", checkStr.Data()));
9827400b 1340 continue;
1341 }
1342
1343 char buffer[100];
1344 if (!fgets(buffer, 100, pipe))
1345 {
c88ad5db 1346 Log("SHUTTLE", "Process - Error: ps did not return anything");
9827400b 1347 gSystem->ClosePipe(pipe);
1348 continue;
1349 }
1350 gSystem->ClosePipe(pipe);
1351
1352 //Log("SHUTTLE", Form("ps returned %s", buffer));
1353
1354 Int_t mem = 0;
1355 if ((sscanf(buffer, "%d\n", &mem) != 1) || !mem)
1356 {
c88ad5db 1357 Log("SHUTTLE", "Process - Error: Could not parse output of ps");
9827400b 1358 continue;
1359 }
1360
1361 if (expiredTime % 60 == 0)
ee6f7523 1362 {
c88ad5db 1363 Log("SHUTTLE", Form("Process - %s: Checking process. "
1364 "Run time: %d seconds - Memory consumption: %d KB",
1365 fCurrentDetector.Data(), expiredTime, mem));
ee6f7523 1366 SendAlive();
1367 }
9827400b 1368
1369 if (mem > fConfig->GetPPMaxMem())
1370 {
1371 TString tmp;
c88ad5db 1372 tmp.Form("Process - Process exceeds maximum allowed memory "
1373 "(%d KB > %d KB). Killing...",
9827400b 1374 mem, fConfig->GetPPMaxMem());
1375 Log("SHUTTLE", tmp);
1376 Log(fCurrentDetector, tmp);
1377
1378 kill(pid, 9);
1379
1380 UpdateShuttleStatus(AliShuttleStatus::kPPOutOfMemory);
1381 hasError = kTRUE;
1382
1383 gSystem->Sleep(1000);
1384 }
be48e3ea 1385 }
1386 }
1387
c88ad5db 1388 Log("SHUTTLE", Form("Process - In parent process of %d - %s: Client has terminated.",
be48e3ea 1389 GetCurrentRun(), aDetector->GetName()));
1390
1391 if (WIFEXITED(status))
1392 {
1393 Int_t returnCode = WEXITSTATUS(status);
1394
c88ad5db 1395 Log("SHUTTLE", Form("Process - %s: the return code is %d", fCurrentDetector.Data(),
3301427a 1396 returnCode));
be48e3ea 1397
9827400b 1398 if (returnCode == 0) hasError = kTRUE;
be48e3ea 1399 }
1400 }
1401 else if (pid == 0)
1402 {
1403 // client
c88ad5db 1404 Log("SHUTTLE", Form("Process - In client process of %d - %s", GetCurrentRun(),
1405 aDetector->GetName()));
be48e3ea 1406
c88ad5db 1407 Log("SHUTTLE", Form("Process - Redirecting output to %s log",fCurrentDetector.Data()));
ffa29e93 1408
546242fb 1409 if ((freopen(GetLogFileName(fCurrentDetector), "a", stdout)) == 0)
ffa29e93 1410 {
c88ad5db 1411 Log("SHUTTLE", "Process - Could not freopen stdout");
ffa29e93 1412 }
1413 else
1414 {
1415 fOutputRedirected = kTRUE;
1416 if ((dup2(fileno(stdout), fileno(stderr))) < 0)
c88ad5db 1417 Log("SHUTTLE", "Process - Could not redirect stderr");
ffa29e93 1418
1419 }
1420
5bac2bde 1421 TString wd = gSystem->WorkingDirectory();
675f64cd 1422 TString tmpDir = Form("%s/%s_%d_process", GetShuttleTempDir(),
1423 fCurrentDetector.Data(), GetCurrentRun());
5bac2bde 1424
d524ade6 1425 Int_t result = gSystem->GetPathInfo(tmpDir.Data(), 0, (Long64_t*) 0, 0, 0);
1426 if (!result) // temp dir already exists!
1427 {
1428 Log(fCurrentDetector.Data(),
1429 Form("Process - %s dir already exists! Removing...", tmpDir.Data()));
1430 gSystem->Exec(Form("rm -rf %s",tmpDir.Data()));
675f64cd 1431 }
1432
1433 if (gSystem->mkdir(tmpDir.Data(), 1))
1434 {
1435 Log(fCurrentDetector.Data(), "Process - could not make temp directory!!");
1436 gSystem->Exit(1);
d524ade6 1437 }
1438
1439 if (!gSystem->ChangeDirectory(tmpDir.Data()))
1440 {
1441 Log(fCurrentDetector.Data(), "Process - could not change directory!!");
1442 gSystem->Exit(1);
1443 }
5bac2bde 1444
9827400b 1445 Bool_t success = ProcessCurrentDetector();
5bac2bde 1446
1447 gSystem->ChangeDirectory(wd.Data());
b1d18693 1448
9827400b 1449 if (success) // Preprocessor finished successfully!
1450 {
fb40ead4 1451 // remove temporary folder or DCS map
1452 if (!fConfig->KeepTempFolder())
1453 {
1454 gSystem->Exec(Form("rm -rf %s",tmpDir.Data()));
1455 } else if (!fConfig->KeepDCSMap())
1456 {
1457 gSystem->Exec(Form("rm -f %s/DCSMap.root",tmpDir.Data()));
1458 }
b1d18693 1459
3301427a 1460 // Update time_processed field in FXS DB
1461 if (UpdateTable() == kFALSE)
5bac2bde 1462 Log("SHUTTLE", Form("Process - %s: Could not update FXS databases!",
1463 fCurrentDetector.Data()));
3301427a 1464
1465 // Transfer the data from local storage to main storage (Grid)
3301427a 1466 if (StoreOCDB() == kFALSE)
9827400b 1467 success = kFALSE;
5e993b6f 1468 }
1469 else
c88ad5db 1470 {
1471 Log("SHUTTLE",
1472 Form("\t\t\t****** run %d - %s: PP ERROR ******",
1473 GetCurrentRun(), aDetector->GetName()));
be48e3ea 1474 }
1475
4b95672b 1476 for (UInt_t iSys=0; iSys<3; iSys++)
1477 {
1478 if (fFXSCalled[iSys]) fFXSlist[iSys].Clear();
1479 }
1480
c88ad5db 1481 Log("SHUTTLE", Form("Process - Client process of %d - %s is exiting now with %d.",
9827400b 1482 GetCurrentRun(), aDetector->GetName(), success));
be48e3ea 1483
1484 // the client exits here
9827400b 1485 gSystem->Exit(success);
be48e3ea 1486
1487 AliError("We should never get here!!!");
1488 }
7bfb2090 1489 }
5164a766 1490
c88ad5db 1491 Log("SHUTTLE", Form("\t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^*",
2bb7b766 1492 GetCurrentRun()));
1493
1494 //check if shuttle is done for this run, if so update logbook
1495 TObjArray checkEntryArray;
1496 checkEntryArray.SetOwner(1);
9e080f92 1497 TString whereClause = Form("where run=%d", GetCurrentRun());
b0e53b15 1498 if (!QueryShuttleLogbook(whereClause.Data(), checkEntryArray) ||
1499 checkEntryArray.GetEntries() == 0) {
9e080f92 1500 Log("SHUTTLE", Form("Process - Warning: Cannot check status of run %d on Shuttle logbook!",
1501 GetCurrentRun()));
1502 return hasError == kFALSE;
1503 }
b948db8d 1504
9e080f92 1505 AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
1506 (checkEntryArray.At(0));
2bb7b766 1507
9e080f92 1508 if (checkEntry)
1509 {
1510 if (checkEntry->IsDone())
be48e3ea 1511 {
9e080f92 1512 Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
1513 UpdateShuttleLogbook("shuttle_done");
1514 }
1515 else
1516 {
1517 for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
be48e3ea 1518 {
9e080f92 1519 if (checkEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
be48e3ea 1520 {
9e080f92 1521 AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
1522 checkEntry->GetRun(), GetDetName(iDet)));
1523 fFirstUnprocessed[iDet] = kFALSE;
be48e3ea 1524 }
1525 }
2bb7b766 1526 }
1527 }
1528
1529 fLogbookEntry = 0;
85a80aa9 1530
a7160fe9 1531 return hasError == kFALSE;
73abe331 1532}
1533
b948db8d 1534//______________________________________________________________________________________________
9827400b 1535Bool_t AliShuttle::ProcessCurrentDetector()
73abe331 1536{
1537 //
2bb7b766 1538 // Makes data retrieval just for a specific detector (fCurrentDetector).
73abe331 1539 // Threre should be a configuration for this detector.
73abe331 1540
1d172743 1541 Log("SHUTTLE", Form("ProcessCurrentDetector - Retrieving values for %s, run %d",
1542 fCurrentDetector.Data(), GetCurrentRun()));
73abe331 1543
d524ade6 1544 TString wd = gSystem->WorkingDirectory();
1545
2d9019b4 1546 if (!CleanReferenceStorage(fCurrentDetector.Data()))
546242fb 1547 return kFALSE;
d524ade6 1548
1549 gSystem->ChangeDirectory(wd.Data());
1550
1d172743 1551 TMap* dcsMap = new TMap();
3301427a 1552
1553 // call preprocessor
1554 AliPreprocessor* aPreprocessor =
1555 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
1556
1557 aPreprocessor->Initialize(GetCurrentRun(), GetCurrentStartTime(), GetCurrentEndTime());
1558
1559 Bool_t processDCS = aPreprocessor->ProcessDCS();
d477ad88 1560
651fdaab 1561 if (!processDCS)
1562 {
1d172743 1563 Log(fCurrentDetector, "ProcessCurrentDetector -"
1564 " The preprocessor requested to skip the retrieval of DCS values");
651fdaab 1565 }
8b739301 1566 else if (fTestMode & kSkipDCS)
2c15234c 1567 {
1d172743 1568 Log(fCurrentDetector, "ProcessCurrentDetector - In TESTMODE: Skipping DCS processing");
9827400b 1569 }
1570 else if (fTestMode & kErrorDCS)
1571 {
1d172743 1572 Log(fCurrentDetector, "ProcessCurrentDetector - In TESTMODE: Simulating DCS error");
3d8bc902 1573 UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
9827400b 1574 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
1d172743 1575 delete dcsMap;
9827400b 1576 return kFALSE;
2c15234c 1577 } else {
3301427a 1578
1579 UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
1580
1d172743 1581 // Query DCS archive
1582 Int_t nServers = fConfig->GetNServers(fCurrentDetector);
a038aa70 1583
1d172743 1584 for (int iServ=0; iServ<nServers; iServ++)
2c15234c 1585 {
1d172743 1586
1587 TString host(fConfig->GetDCSHost(fCurrentDetector, iServ));
1588 Int_t port = fConfig->GetDCSPort(fCurrentDetector, iServ);
542b6cc8 1589 Int_t multiSplit = fConfig->GetMultiSplit(fCurrentDetector, iServ);
1590
1790d4b7 1591 Log(fCurrentDetector, Form("ProcessCurrentDetector -"
1592 " Querying DCS Amanda server %s:%d (%d of %d)",
1593 host.Data(), port, iServ+1, nServers));
1d172743 1594
1595 TMap* aliasMap = 0;
1596 TMap* dpMap = 0;
1597
1598 if (fConfig->GetDCSAliases(fCurrentDetector, iServ)->GetEntries() > 0)
2c15234c 1599 {
1d172743 1600 aliasMap = GetValueSet(host, port,
542b6cc8 1601 fConfig->GetDCSAliases(fCurrentDetector, iServ),
1602 kAlias, multiSplit);
1d172743 1603 if (!aliasMap)
1604 {
1605 Log(fCurrentDetector,
1606 Form("ProcessCurrentDetector -"
675f64cd 1607 " Error retrieving DCS aliases from server %s."
1608 " Sending mail to DCS experts!", host.Data()));
1d172743 1609 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
675f64cd 1610
fb40ead4 1611 if (!SendMailToDCS())
1612 Log("SHUTTLE", Form("ProcessCurrentDetector - "
95d4aaef 1613 "Could not send mail to DCS experts!"));
675f64cd 1614
a038aa70 1615 delete dcsMap;
1d172743 1616 return kFALSE;
1617 }
2c15234c 1618 }
a038aa70 1619
1d172743 1620 if (fConfig->GetDCSDataPoints(fCurrentDetector, iServ)->GetEntries() > 0)
a038aa70 1621 {
1d172743 1622 dpMap = GetValueSet(host, port,
542b6cc8 1623 fConfig->GetDCSDataPoints(fCurrentDetector, iServ),
1624 kDP, multiSplit);
1d172743 1625 if (!dpMap)
1626 {
1627 Log(fCurrentDetector,
1628 Form("ProcessCurrentDetector -"
675f64cd 1629 " Error retrieving DCS data points from server %s."
1630 " Sending mail to DCS experts!", host.Data()));
1d172743 1631 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
675f64cd 1632
fb40ead4 1633 if (!SendMailToDCS())
1634 Log("SHUTTLE", Form("ProcessCurrentDetector - "
95d4aaef 1635 "Could not send mail to DCS experts!"));
675f64cd 1636
1d172743 1637 if (aliasMap) delete aliasMap;
1638 delete dcsMap;
1639 return kFALSE;
1640 }
a038aa70 1641 }
1d172743 1642
1643 // merge aliasMap and dpMap into dcsMap
1644 if(aliasMap) {
1645 TIter iter(aliasMap);
a038aa70 1646 TObjString* key = 0;
1647 while ((key = (TObjString*) iter.Next()))
1d172743 1648 dcsMap->Add(key, aliasMap->GetValue(key->String()));
1649
1650 aliasMap->SetOwner(kFALSE);
1651 delete aliasMap;
1652 }
1653
1654 if(dpMap) {
1655 TIter iter(dpMap);
1656 TObjString* key = 0;
1657 while ((key = (TObjString*) iter.Next()))
1658 dcsMap->Add(key, dpMap->GetValue(key->String()));
1659
1660 dpMap->SetOwner(kFALSE);
1661 delete dpMap;
a038aa70 1662 }
73abe331 1663 }
1664 }
dc25836b 1665
b1d18693 1666 // save map into file, to help debugging in case of preprocessor error
fbc112e3 1667 TFile* f = TFile::Open("DCSMap.root","recreate");
b1d18693 1668 f->cd();
1669 dcsMap->Write("DCSMap", TObject::kSingleKey);
1670 f->Close();
fbc112e3 1671 delete f;
b1d18693 1672
2bb7b766 1673 // DCS Archive DB processing successful. Call Preprocessor!
85a80aa9 1674 UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
a7160fe9 1675
a038aa70 1676 UInt_t returnValue = aPreprocessor->Process(dcsMap);
b948db8d 1677
3301427a 1678 if (returnValue > 0) // Preprocessor error!
1679 {
c88ad5db 1680 Log(fCurrentDetector, Form("ProcessCurrentDetector - "
1681 "Preprocessor failed. Process returned %d.", returnValue));
cb343cfd 1682 UpdateShuttleStatus(AliShuttleStatus::kPPError);
a038aa70 1683 dcsMap->DeleteAll();
1684 delete dcsMap;
9827400b 1685 return kFALSE;
1686 }
1687
1688 // preprocessor ok!
1689 UpdateShuttleStatus(AliShuttleStatus::kPPDone);
1690 Log(fCurrentDetector, Form("ProcessCurrentDetector - %s preprocessor returned success",
1691 fCurrentDetector.Data()));
b948db8d 1692
a038aa70 1693 dcsMap->DeleteAll();
1694 delete dcsMap;
b948db8d 1695
9827400b 1696 return kTRUE;
2bb7b766 1697}
1698
fbc112e3 1699//______________________________________________________________________________________________
1700void AliShuttle::CountOpenRuns()
1701{
1702 // Query DAQ's Shuttle logbook and sends the number of open runs to ML
1703
1704 // check connection, in case connect
1705 if (!Connect(3))
1706 return;
1707
1708 TString sqlQuery;
1709 sqlQuery = Form("select count(*) from %s where shuttle_done=0", fConfig->GetShuttlelbTable());
1710
1711 TSQLResult* aResult = fServer[3]->Query(sqlQuery);
1712 if (!aResult) {
1713 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
1714 return;
1715 }
1716
1717 AliDebug(2,Form("Query = %s", sqlQuery.Data()));
1718
1719 if (aResult->GetRowCount() == 0) {
1720 AliError(Form("No result for query %s received", sqlQuery.Data()));
1721 return;
1722 }
1723
1724 if (aResult->GetFieldCount() != 1) {
1725 AliError(Form("Invalid field count for query %s received", sqlQuery.Data()));
1726 return;
1727 }
1728
1729 TSQLRow* aRow = aResult->Next();
1730 if (!aRow) {
1731 AliError(Form("Could not receive result of query %s", sqlQuery.Data()));
1732 return;
1733 }
1734
1735 TString result(aRow->GetField(0), aRow->GetFieldLength(0));
1736 Int_t count = result.Atoi();
1737
1738 Log("SHUTTLE", Form("%d unprocessed runs", count));
1739
1740 delete aRow;
1741 delete aResult;
1742
1743 TMonaLisaValue mlStatus("SHUTTLE_openruns", count);
1744
1745 TList mlList;
1746 mlList.Add(&mlStatus);
1747
1748 fMonaLisa->SendParameters(&mlList, "__PROCESSINGINFO__");
1749}
1750
2bb7b766 1751//______________________________________________________________________________________________
1752Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
1753 TObjArray& entries)
1754{
9827400b 1755 // Query DAQ's Shuttle logbook and fills detector status object.
1756 // Call QueryRunParameters to query DAQ logbook for run parameters.
1757 //
2bb7b766 1758
fc5a4708 1759 entries.SetOwner(1);
1760
2bb7b766 1761 // check connection, in case connect
fbc112e3 1762 if (!Connect(3)) return kFALSE;
2bb7b766 1763
1764 TString sqlQuery;
441b0e9c 1765 sqlQuery = Form("select * from %s %s order by run", fConfig->GetShuttlelbTable(), whereClause);
2bb7b766 1766
be48e3ea 1767 TSQLResult* aResult = fServer[3]->Query(sqlQuery);
2bb7b766 1768 if (!aResult) {
1769 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
1770 return kFALSE;
1771 }
1772
fc5a4708 1773 AliDebug(2,Form("Query = %s", sqlQuery.Data()));
1774
2bb7b766 1775 if(aResult->GetRowCount() == 0) {
c88ad5db 1776 Log("SHUTTLE", "No entries in Shuttle Logbook match request");
9827400b 1777 delete aResult;
1778 return kTRUE;
2bb7b766 1779 }
1780
1781 // TODO Check field count!
db99d43e 1782 const UInt_t nCols = 23;
2bb7b766 1783 if (aResult->GetFieldCount() != (Int_t) nCols) {
c88ad5db 1784 Log("SHUTTLE", "Invalid SQL result field number!");
2bb7b766 1785 delete aResult;
1786 return kFALSE;
1787 }
1788
2bb7b766 1789 TSQLRow* aRow;
1790 while ((aRow = aResult->Next())) {
1791 TString runString(aRow->GetField(0), aRow->GetFieldLength(0));
1792 Int_t run = runString.Atoi();
1793
eba76848 1794 AliShuttleLogbookEntry *entry = QueryRunParameters(run);
1795 if (!entry)
1796 continue;
2bb7b766 1797
1798 // loop on detectors
eba76848 1799 for(UInt_t ii = 0; ii < nCols; ii++)
1800 entry->SetDetectorStatus(aResult->GetFieldName(ii), aRow->GetField(ii));
2bb7b766 1801
eba76848 1802 entries.AddLast(entry);
2bb7b766 1803 delete aRow;
1804 }
1805
2bb7b766 1806 delete aResult;
1807 return kTRUE;
1808}
1809
1810//______________________________________________________________________________________________
eba76848 1811AliShuttleLogbookEntry* AliShuttle::QueryRunParameters(Int_t run)
2bb7b766 1812{
eba76848 1813 //
1814 // Retrieve run parameters written in the DAQ logbook and sets them into AliShuttleLogbookEntry object
1815 //
2bb7b766 1816
1817 // check connection, in case connect
be48e3ea 1818 if (!Connect(3))
eba76848 1819 return 0;
2bb7b766 1820
1821 TString sqlQuery;
2c15234c 1822 sqlQuery.Form("select * from %s where run=%d", fConfig->GetDAQlbTable(), run);
2bb7b766 1823
be48e3ea 1824 TSQLResult* aResult = fServer[3]->Query(sqlQuery);
2bb7b766 1825 if (!aResult) {
c88ad5db 1826 Log("SHUTTLE", Form("Can't execute query <%s>!", sqlQuery.Data()));
eba76848 1827 return 0;
2bb7b766 1828 }
1829
eba76848 1830 if (aResult->GetRowCount() == 0) {
2bb7b766 1831 Log("SHUTTLE", Form("QueryRunParameters - No entry in DAQ Logbook for run %d. Skipping", run));
1832 delete aResult;
eba76848 1833 return 0;
2bb7b766 1834 }
1835
eba76848 1836 if (aResult->GetRowCount() > 1) {
c88ad5db 1837 Log("SHUTTLE", Form("QueryRunParameters - UNEXPECTED: "
1838 "more than one entry in DAQ Logbook for run %d!", run));
2bb7b766 1839 delete aResult;
eba76848 1840 return 0;
2bb7b766 1841 }
1842
eba76848 1843 TSQLRow* aRow = aResult->Next();
1844 if (!aRow)
1845 {
c88ad5db 1846 Log("SHUTTLE", Form("QueryRunParameters - Could not retrieve row for run %d. Skipping", run));
eba76848 1847 delete aResult;
1848 return 0;
1849 }
2bb7b766 1850
eba76848 1851 AliShuttleLogbookEntry* entry = new AliShuttleLogbookEntry(run);
2bb7b766 1852
eba76848 1853 for (Int_t ii = 0; ii < aResult->GetFieldCount(); ii++)
1854 entry->SetRunParameter(aResult->GetFieldName(ii), aRow->GetField(ii));
2bb7b766 1855
eba76848 1856 UInt_t startTime = entry->GetStartTime();
1857 UInt_t endTime = entry->GetEndTime();
1858
1abfbb60 1859// if (!startTime || !endTime || startTime > endTime)
1860// {
1861// Log("SHUTTLE",
1862// Form("QueryRunParameters - Invalid parameters for Run %d: startTime = %d, endTime = %d. Skipping!",
1863// run, startTime, endTime));
1864//
1865// Log("SHUTTLE", Form("Marking SHUTTLE done for run %d", run));
1866// fLogbookEntry = entry;
1867// if (!UpdateShuttleLogbook("shuttle_done"))
1868// {
1869// AliError(Form("Could not update logbook for run %d !", run));
1870// }
1871// fLogbookEntry = 0;
1872//
1873// delete entry;
1874// delete aRow;
1875// delete aResult;
1876// return 0;
1877// }
1878
1879 if (!startTime)
b0e53b15 1880 {
1881 Log("SHUTTLE",
1abfbb60 1882 Form("QueryRunParameters - Invalid parameters for Run %d: "
1883 "startTime = %d, endTime = %d. Skipping!",
1884 run, startTime, endTime));
b0e53b15 1885
1886 Log("SHUTTLE", Form("Marking SHUTTLE done for run %d", run));
1887 fLogbookEntry = entry;
ee6f7523 1888 if (!UpdateShuttleLogbook("shuttle_ignored"))
b0e53b15 1889 {
1890 AliError(Form("Could not update logbook for run %d !", run));
1891 }
1892 fLogbookEntry = 0;
1893
1894 delete entry;
1895 delete aRow;
1896 delete aResult;
1897 return 0;
1898 }
1899
1abfbb60 1900 if (startTime && !endTime)
1901 {
1902 // TODO Here we don't mark SHUTTLE done, because this may mean
1903 //the run is still ongoing!!
1904 Log("SHUTTLE",
1905 Form("QueryRunParameters - Invalid parameters for Run %d: "
1906 "startTime = %d, endTime = %d. Skipping (Shuttle won't be marked as DONE)!",
1907 run, startTime, endTime));
1908
1909 //Log("SHUTTLE", Form("Marking SHUTTLE done for run %d", run));
1910 //fLogbookEntry = entry;
1911 //if (!UpdateShuttleLogbook("shuttle_done"))
1912 //{
1913 // AliError(Form("Could not update logbook for run %d !", run));
1914 //}
1915 //fLogbookEntry = 0;
1916
1917 delete entry;
1918 delete aRow;
1919 delete aResult;
1920 return 0;
1921 }
1922
1923 if (startTime && endTime && (startTime > endTime))
1924 {
1925 Log("SHUTTLE",
1926 Form("QueryRunParameters - Invalid parameters for Run %d: "
1927 "startTime = %d, endTime = %d. Skipping!",
1928 run, startTime, endTime));
1929
1930 Log("SHUTTLE", Form("Marking SHUTTLE done for run %d", run));
1931 fLogbookEntry = entry;
ee6f7523 1932 if (!UpdateShuttleLogbook("shuttle_ignored"))
1abfbb60 1933 {
1934 AliError(Form("Could not update logbook for run %d !", run));
1935 }
1936 fLogbookEntry = 0;
1937
1938 delete entry;
1939 delete aRow;
1940 delete aResult;
1941 return 0;
1942 }
1943
b0e53b15 1944 TString totEventsStr = entry->GetRunParameter("totalEvents");
1945 Int_t totEvents = totEventsStr.Atoi();
1946 if (totEvents < 1)
1947 {
eba76848 1948 Log("SHUTTLE",
b0e53b15 1949 Form("QueryRunParameters - Run %d has 0 events - Skipping!", run));
1950
1951 Log("SHUTTLE", Form("Marking SHUTTLE done for run %d", run));
1952 fLogbookEntry = entry;
97e3c167 1953 if (!UpdateShuttleLogbook("shuttle_ignored"))
b0e53b15 1954 {
1955 AliError(Form("Could not update logbook for run %d !", run));
1956 }
1957 fLogbookEntry = 0;
1958
eba76848 1959 delete entry;
2bb7b766 1960 delete aRow;
eba76848 1961 delete aResult;
1962 return 0;
2bb7b766 1963 }
1964
eba76848 1965 delete aRow;
2bb7b766 1966 delete aResult;
eba76848 1967
1968 return entry;
2bb7b766 1969}
1970
a038aa70 1971//______________________________________________________________________________________________
1972TMap* AliShuttle::GetValueSet(const char* host, Int_t port, const TSeqCollection* entries,
542b6cc8 1973 DCSType type, Int_t multiSplit)
a038aa70 1974{
1975 // Retrieve all "entry" data points from the DCS server
1976 // host, port: TSocket connection parameters
1977 // entries: list of name of the alias or data point
1978 // type: kAlias or kDP
1979 // returns TMap of values, 0 when failure
542b6cc8 1980
1981 AliDCSClient client(host, port, fTimeout, fRetries, multiSplit);
b41b252a 1982
a038aa70 1983 TMap* result = 0;
b41b252a 1984 if (type == kAlias)
a038aa70 1985 {
b41b252a 1986 result = client.GetAliasValues(entries, GetCurrentStartTime(),
1987 GetCurrentEndTime());
1988 }
1989 else if (type == kDP)
1990 {
1991 result = client.GetDPValues(entries, GetCurrentStartTime(),
1992 GetCurrentEndTime());
1993 }
a038aa70 1994
b41b252a 1995 if (result == 0)
1996 {
1997 Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get entries! Reason: %s",
1790d4b7 1998 client.GetErrorString(client.GetResultErrorCode())));
1999 if (client.GetResultErrorCode() == AliDCSClient::fgkServerError)
2000 Log(fCurrentDetector.Data(), Form("GetValueSet - Server error code: %s",
2001 client.GetServerError().Data()));
a038aa70 2002
b41b252a 2003 return 0;
a038aa70 2004 }
b41b252a 2005
a038aa70 2006 return result;
2007}
b41b252a 2008
b948db8d 2009//______________________________________________________________________________________________
57f50b3c 2010const char* AliShuttle::GetFile(Int_t system, const char* detector,
2011 const char* id, const char* source)
b948db8d 2012{
9827400b 2013 // Get calibration file from file exchange servers
2014 // First queris the FXS database for the file name, using the run, detector, id and source info
2015 // then calls RetrieveFile(filename) for actual copy to local disk
2016 // run: current run being processed (given by Logbook entry fLogbookEntry)
2017 // detector: the Preprocessor name
2018 // id: provided as a parameter by the Preprocessor
2019 // source: provided by the Preprocessor through GetFileSources function
2020
2021 // check if test mode should simulate a FXS error
2022 if (fTestMode & kErrorFXSFiles)
2023 {
2024 Log(detector, Form("GetFile - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
2025 return 0;
2026 }
2027
57f50b3c 2028 // check connection, in case connect
9d733021 2029 if (!Connect(system))
eba76848 2030 {
9d733021 2031 Log(detector, Form("GetFile - Couldn't connect to %s FXS database", GetSystemName(system)));
57f50b3c 2032 return 0;
2033 }
2034
2035 // Query preparation
9d733021 2036 TString sourceName(source);
d386d623 2037 Int_t nFields = 3;
2038 TString sqlQueryStart = Form("select filePath,size,fileChecksum from %s where",
2039 fConfig->GetFXSdbTable(system));
2040 TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
2041 GetCurrentRun(), detector, id);
2042
9d733021 2043 if (system == kDAQ)
2044 {
d386d623 2045 whereClause += Form(" and DAQsource=\"%s\"", source);
57f50b3c 2046 }
9d733021 2047 else if (system == kDCS)
eba76848 2048 {
9d733021 2049 sourceName="none";
57f50b3c 2050 }
9d733021 2051 else if (system == kHLT)
9e080f92 2052 {
d386d623 2053 whereClause += Form(" and DDLnumbers=\"%s\"", source);
9d733021 2054 nFields = 3;
9e080f92 2055 }
2056
9e080f92 2057 TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
2058
2059 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2060
2061 // Query execution
2062 TSQLResult* aResult = 0;
9d733021 2063 aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
9e080f92 2064 if (!aResult) {
9d733021 2065 Log(detector, Form("GetFileName - Can't execute SQL query to %s database for: id = %s, source = %s",
2066 GetSystemName(system), id, sourceName.Data()));
9e080f92 2067 return 0;
2068 }
2069
2070 if(aResult->GetRowCount() == 0)
2071 {
2072 Log(detector,
9d733021 2073 Form("GetFileName - No entry in %s FXS db for: id = %s, source = %s",
2074 GetSystemName(system), id, sourceName.Data()));
9e080f92 2075 delete aResult;
2076 return 0;
2077 }
2bb7b766 2078
9e080f92 2079 if (aResult->GetRowCount() > 1) {
2080 Log(detector,
9d733021 2081 Form("GetFileName - More than one entry in %s FXS db for: id = %s, source = %s",
2082 GetSystemName(system), id, sourceName.Data()));
9e080f92 2083 delete aResult;
2084 return 0;
2085 }
2086
9d733021 2087 if (aResult->GetFieldCount() != nFields) {
9e080f92 2088 Log(detector,
9d733021 2089 Form("GetFileName - Wrong field count in %s FXS db for: id = %s, source = %s",
2090 GetSystemName(system), id, sourceName.Data()));
9e080f92 2091 delete aResult;
2092 return 0;
2093 }
2094
2095 TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
2096
2097 if (!aRow){
9d733021 2098 Log(detector, Form("GetFileName - Empty set result in %s FXS db from query: id = %s, source = %s",
2099 GetSystemName(system), id, sourceName.Data()));
9e080f92 2100 delete aResult;
2101 return 0;
2102 }
2103
2104 TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
2105 TString fileSize(aRow->GetField(1), aRow->GetFieldLength(1));
d386d623 2106 TString fileChecksum(aRow->GetField(2), aRow->GetFieldLength(2));
9e080f92 2107
2108 delete aResult;
2109 delete aRow;
2110
d386d623 2111 AliDebug(2, Form("filePath = %s; size = %s, fileChecksum = %s",
2112 filePath.Data(), fileSize.Data(), fileChecksum.Data()));
9e080f92 2113
9e080f92 2114 // retrieved file is renamed to make it unique
675f64cd 2115 TString localFileName = Form("%s/%s_%d_process/%s_%s_%d_%s_%s.shuttle",
2116 GetShuttleTempDir(), detector, GetCurrentRun(),
d524ade6 2117 GetSystemName(system), detector, GetCurrentRun(),
2118 id, sourceName.Data());
9d733021 2119
9e080f92 2120
9d733021 2121 // file retrieval from FXS
4b95672b 2122 UInt_t nRetries = 0;
2123 UInt_t maxRetries = 3;
2124 Bool_t result = kFALSE;
2125
2126 // copy!! if successful TSystem::Exec returns 0
2127 while(nRetries++ < maxRetries) {
2128 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
2129 result = RetrieveFile(system, filePath.Data(), localFileName.Data());
2130 if(!result)
2131 {
2132 Log(detector, Form("GetFileName - Copy of file %s from %s FXS failed",
9d733021 2133 filePath.Data(), GetSystemName(system)));
4b95672b 2134 continue;
4f0749a8 2135 }
9e080f92 2136
c98417a1 2137 if (fileSize.Length()>0)
2138 {
2139 // compare filesize of local file with the one stored in the FXS DB
926cbc0e 2140 Long_t size = -1;
2141 Int_t sizeComp = gSystem->GetPathInfo(localFileName.Data(), 0, &size, 0, 0);
c98417a1 2142
926cbc0e 2143 if (sizeComp != 0 || size != fileSize.Atoi())
c98417a1 2144 {
2145 Log(detector, Form("GetFileName - size of file %s does not match with local copy!",
2146 filePath.Data()));
2147 result = kFALSE;
2148 continue;
2149 }
2150
2151 } else {
2152 Log(fCurrentDetector, Form("GetFile - size of file %s not set in %s database, skipping comparison",
2153 filePath.Data(), GetSystemName(system)));
2154 }
2155
d386d623 2156 if (fileChecksum.Length()>0)
4b95672b 2157 {
2158 // compare md5sum of local file with the one stored in the FXS DB
95d4aaef 2159 if(fileChecksum.Contains(' ')) fileChecksum.Resize(fileChecksum.First(' '));
d524ade6 2160 Int_t md5Comp = gSystem->Exec(Form("md5sum %s |grep %s 2>&1 > /dev/null",
2161 localFileName.Data(), fileChecksum.Data()));
9e080f92 2162
4b95672b 2163 if (md5Comp != 0)
2164 {
2165 Log(detector, Form("GetFileName - md5sum of file %s does not match with local copy!",
2166 filePath.Data()));
2167 result = kFALSE;
2168 continue;
2169 }
d386d623 2170 } else {
2171 Log(fCurrentDetector, Form("GetFile - md5sum of file %s not set in %s database, skipping comparison",
2172 filePath.Data(), GetSystemName(system)));
9d733021 2173 }
4b95672b 2174 if (result) break;
9e080f92 2175 }
2176
4b95672b 2177 if(!result) return 0;
2178
9d733021 2179 fFXSCalled[system]=kTRUE;
2180 TObjString *fileParams = new TObjString(Form("%s#!?!#%s", id, sourceName.Data()));
2181 fFXSlist[system].Add(fileParams);
9e080f92 2182
675f64cd 2183 static TString staticLocalFileName;
2184 staticLocalFileName.Form("%s", localFileName.Data());
2185
c88ad5db 2186 Log(fCurrentDetector, Form("GetFile - Retrieved file with id %s and "
2187 "source %s from %s to %s", id, source,
d524ade6 2188 GetSystemName(system), localFileName.Data()));
675f64cd 2189
d524ade6 2190 return staticLocalFileName.Data();
2bb7b766 2191}
2192
2193//______________________________________________________________________________________________
9d733021 2194Bool_t AliShuttle::RetrieveFile(UInt_t system, const char* fxsFileName, const char* localFileName)
9e080f92 2195{
9827400b 2196 //
2197 // Copies file from FXS to local Shuttle machine
2198 //
2bb7b766 2199
9e080f92 2200 // check temp directory: trying to cd to temp; if it does not exist, create it
d524ade6 2201 AliDebug(2, Form("Copy file %s from %s FXS into %s",
2202 GetSystemName(system), fxsFileName, localFileName));
2203
2204 TString tmpDir(localFileName);
2205
2206 tmpDir = tmpDir(0,tmpDir.Last('/'));
9e080f92 2207
d524ade6 2208 Int_t noDir = gSystem->GetPathInfo(tmpDir.Data(), 0, (Long64_t*) 0, 0, 0);
2209 if (noDir) // temp dir does not exists!
2210 {
2211 if (gSystem->mkdir(tmpDir.Data(), 1))
2212 {
2213 Log(fCurrentDetector.Data(), "RetrieveFile - could not make temp directory!!");
9e080f92 2214 return kFALSE;
2215 }
9e080f92 2216 }
2217
9d733021 2218 TString baseFXSFolder;
2219 if (system == kDAQ)
2220 {
2221 baseFXSFolder = "FES/";
2222 }
2223 else if (system == kDCS)
2224 {
2225 baseFXSFolder = "";
2226 }
2227 else if (system == kHLT)
2228 {
42fde080 2229 baseFXSFolder = "/opt/FXS/";
9d733021 2230 }
2231
2232
d524ade6 2233 TString command = Form("scp -oPort=%d -2 %s@%s:%s%s %s",
9d733021 2234 fConfig->GetFXSPort(system),
2235 fConfig->GetFXSUser(system),
2236 fConfig->GetFXSHost(system),
2237 baseFXSFolder.Data(),
2238 fxsFileName,
9e080f92 2239 localFileName);
2240
2241 AliDebug(2, Form("%s",command.Data()));
2242
4b95672b 2243 Bool_t result = (gSystem->Exec(command.Data()) == 0);
9e080f92 2244
4b95672b 2245 return result;
9e080f92 2246}
2247
2248//______________________________________________________________________________________________
9d733021 2249TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
2250{
9827400b 2251 //
2252 // Get sources producing the condition file Id from file exchange servers
4a33bdd9 2253 // if id is NULL all sources are returned (distinct)
9827400b 2254 //
1bcd28db 2255
8884d71e 2256 if (id)
2257 {
2258 Log(detector, Form("GetFileSources - Querying %s FXS for files with id %s produced by %s", GetSystemName(system), id, detector));
2259 } else {
2260 Log(detector, Form("GetFileSources - Querying %s FXS for files produced by %s", GetSystemName(system), detector));
2261 }
9827400b 2262
2263 // check if test mode should simulate a FXS error
2264 if (fTestMode & kErrorFXSSources)
2265 {
2266 Log(detector, Form("GetFileSources - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
2267 return 0;
2268 }
2269
9d733021 2270 if (system == kDCS)
2271 {
c88ad5db 2272 Log(detector, "GetFileSources - WARNING: DCS system has only one source of data!");
6297b37d 2273 TList *list = new TList();
2274 list->SetOwner(1);
2275 list->Add(new TObjString(" "));
2276 return list;
9d733021 2277 }
9e080f92 2278
2279 // check connection, in case connect
9d733021 2280 if (!Connect(system))
2281 {
4a33bdd9 2282 Log(detector, Form("GetFileSources - Couldn't connect to %s FXS database", GetSystemName(system)));
9d733021 2283 return NULL;
9e080f92 2284 }
2285
9d733021 2286 TString sourceName = 0;
2287 if (system == kDAQ)
2288 {
2289 sourceName = "DAQsource";
2290 } else if (system == kHLT)
2291 {
2292 sourceName = "DDLnumbers";
2293 }
2294
4a33bdd9 2295 TString sqlQueryStart = Form("select distinct %s from %s where", sourceName.Data(), fConfig->GetFXSdbTable(system));
2296 TString whereClause = Form("run=%d and detector=\"%s\"",
2297 GetCurrentRun(), detector);
2298 if (id)
2299 whereClause += Form(" and fileId=\"%s\"", id);
9e080f92 2300 TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
2301
2302 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2303
2304 // Query execution
2305 TSQLResult* aResult;
9d733021 2306 aResult = fServer[system]->Query(sqlQuery);
9e080f92 2307 if (!aResult) {
9d733021 2308 Log(detector, Form("GetFileSources - Can't execute SQL query to %s database for id: %s",
2309 GetSystemName(system), id));
9e080f92 2310 return 0;
2311 }
2312
86aa42c3 2313 TList *list = new TList();
2314 list->SetOwner(1);
2315
9d733021 2316 if (aResult->GetRowCount() == 0)
2317 {
9e080f92 2318 Log(detector,
9d733021 2319 Form("GetFileSources - No entry in %s FXS table for id: %s", GetSystemName(system), id));
9e080f92 2320 delete aResult;
86aa42c3 2321 return list;
9e080f92 2322 }
2323
1bcd28db 2324 Log(detector, Form("GetFileSources - Found %d sources", aResult->GetRowCount()));
9e080f92 2325
1bcd28db 2326 TSQLRow* aRow;
9d733021 2327 while ((aRow = aResult->Next()))
2328 {
9e080f92 2329
9d733021 2330 TString source(aRow->GetField(0), aRow->GetFieldLength(0));
2331 AliDebug(2, Form("%s = %s", sourceName.Data(), source.Data()));
2332 list->Add(new TObjString(source));
9e080f92 2333 delete aRow;
2334 }
9d733021 2335
9e080f92 2336 delete aResult;
2337
2338 return list;
2bb7b766 2339}
2340
4a33bdd9 2341//______________________________________________________________________________________________
2342TList* AliShuttle::GetFileIDs(Int_t system, const char* detector, const char* source)
2343{
2344 //
2345 // Get all ids of condition files produced by a given source from file exchange servers
2346 //
2347
1bcd28db 2348 Log(detector, Form("GetFileIDs - Retrieving ids with source %s with %s", source, GetSystemName(system)));
2349
4a33bdd9 2350 // check if test mode should simulate a FXS error
2351 if (fTestMode & kErrorFXSSources)
2352 {
2353 Log(detector, Form("GetFileIDs - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
2354 return 0;
2355 }
2356
2357 // check connection, in case connect
2358 if (!Connect(system))
2359 {
2360 Log(detector, Form("GetFileIDs - Couldn't connect to %s FXS database", GetSystemName(system)));
2361 return NULL;
2362 }
2363
2364 TString sourceName = 0;
2365 if (system == kDAQ)
2366 {
2367 sourceName = "DAQsource";
2368 } else if (system == kHLT)
2369 {
2370 sourceName = "DDLnumbers";
2371 }
2372
2373 TString sqlQueryStart = Form("select fileId from %s where", fConfig->GetFXSdbTable(system));
2374 TString whereClause = Form("run=%d and detector=\"%s\"",
2375 GetCurrentRun(), detector);
2376 if (sourceName.Length() > 0 && source)
2377 whereClause += Form(" and %s=\"%s\"", sourceName.Data(), source);
2378 TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
2379
2380 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2381
2382 // Query execution
2383 TSQLResult* aResult;
2384 aResult = fServer[system]->Query(sqlQuery);
2385 if (!aResult) {
2386 Log(detector, Form("GetFileIDs - Can't execute SQL query to %s database for source: %s",
2387 GetSystemName(system), source));
2388 return 0;
2389 }
2390
2391 TList *list = new TList();
2392 list->SetOwner(1);
2393
2394 if (aResult->GetRowCount() == 0)
2395 {
2396 Log(detector,
2397 Form("GetFileIDs - No entry in %s FXS table for source: %s", GetSystemName(system), source));
2398 delete aResult;
2399 return list;
2400 }
2401
1bcd28db 2402 Log(detector, Form("GetFileIDs - Found %d ids", aResult->GetRowCount()));
2403
4a33bdd9 2404 TSQLRow* aRow;
2405
2406 while ((aRow = aResult->Next()))
2407 {
2408
2409 TString id(aRow->GetField(0), aRow->GetFieldLength(0));
2410 AliDebug(2, Form("fileId = %s", id.Data()));
2411 list->Add(new TObjString(id));
2412 delete aRow;
2413 }
2414
2415 delete aResult;
2416
2417 return list;
2418}
2419
2bb7b766 2420//______________________________________________________________________________________________
9d733021 2421Bool_t AliShuttle::Connect(Int_t system)
2bb7b766 2422{
9827400b 2423 // Connect to MySQL Server of the system's FXS MySQL databases
2424 // DAQ Logbook, Shuttle Logbook and DAQ FXS db are on the same host
2425 //
57f50b3c 2426
9d733021 2427 // check connection: if already connected return
2428 if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
57f50b3c 2429
9d733021 2430 TString dbHost, dbUser, dbPass, dbName;
57f50b3c 2431
9d733021 2432 if (system < 3) // FXS db servers
2433 {
2434 dbHost = Form("mysql://%s:%d", fConfig->GetFXSdbHost(system), fConfig->GetFXSdbPort(system));
2435 dbUser = fConfig->GetFXSdbUser(system);
2436 dbPass = fConfig->GetFXSdbPass(system);
2437 dbName = fConfig->GetFXSdbName(system);
2438 } else { // Run & Shuttle logbook servers
2439 // TODO Will the Shuttle logbook server be the same as the Run logbook server ???
2440 dbHost = Form("mysql://%s:%d", fConfig->GetDAQlbHost(), fConfig->GetDAQlbPort());
2441 dbUser = fConfig->GetDAQlbUser();
2442 dbPass = fConfig->GetDAQlbPass();
2443 dbName = fConfig->GetDAQlbDB();
2444 }
57f50b3c 2445
9d733021 2446 fServer[system] = TSQLServer::Connect(dbHost.Data(), dbUser.Data(), dbPass.Data());
2447 if (!fServer[system] || !fServer[system]->IsConnected()) {
2448 if(system < 3)
2449 {
2450 AliError(Form("Can't establish connection to FXS database for %s",
2451 AliShuttleInterface::GetSystemName(system)));
2452 } else {
2453 AliError("Can't establish connection to Run logbook.");
57f50b3c 2454 }
9d733021 2455 if(fServer[system]) delete fServer[system];
2456 return kFALSE;
2bb7b766 2457 }
57f50b3c 2458
9d733021 2459 // Get tables
2460 TSQLResult* aResult=0;
2461 switch(system){
2462 case kDAQ:
2463 aResult = fServer[kDAQ]->GetTables(dbName.Data());
2464 break;
2465 case kDCS:
2466 aResult = fServer[kDCS]->GetTables(dbName.Data());
2467 break;
2468 case kHLT:
2469 aResult = fServer[kHLT]->GetTables(dbName.Data());
2470 break;
2471 default:
2472 aResult = fServer[3]->GetTables(dbName.Data());
2473 break;
2474 }
2475
2476 delete aResult;
2bb7b766 2477 return kTRUE;
2478}
57f50b3c 2479
9e080f92 2480//______________________________________________________________________________________________
9d733021 2481Bool_t AliShuttle::UpdateTable()
9e080f92 2482{
9827400b 2483 //
2484 // Update FXS table filling time_processed field in all rows corresponding to current run and detector
2485 //
9e080f92 2486
9d733021 2487 Bool_t result = kTRUE;
9e080f92 2488
9d733021 2489 for (UInt_t system=0; system<3; system++)
2490 {
2491 if(!fFXSCalled[system]) continue;
9e080f92 2492
9d733021 2493 // check connection, in case connect
2494 if (!Connect(system))
2495 {
2496 Log(fCurrentDetector, Form("UpdateTable - Couldn't connect to %s FXS database", GetSystemName(system)));
2497 result = kFALSE;
2498 continue;
9e080f92 2499 }
9e080f92 2500
9d733021 2501 TTimeStamp now; // now
2502
2503 // Loop on FXS list entries
2504 TIter iter(&fFXSlist[system]);
2505 TObjString *aFXSentry=0;
2506 while ((aFXSentry = dynamic_cast<TObjString*> (iter.Next())))
2507 {
2508 TString aFXSentrystr = aFXSentry->String();
2509 TObjArray *aFXSarray = aFXSentrystr.Tokenize("#!?!#");
2510 if (!aFXSarray || aFXSarray->GetEntries() != 2 )
2511 {
2512 Log(fCurrentDetector, Form("UpdateTable - error updating %s FXS entry. Check string: <%s>",
2513 GetSystemName(system), aFXSentrystr.Data()));
2514 if(aFXSarray) delete aFXSarray;
2515 result = kFALSE;
2516 continue;
2517 }
2518 const char* fileId = ((TObjString*) aFXSarray->At(0))->GetName();
2519 const char* source = ((TObjString*) aFXSarray->At(1))->GetName();
2520
2521 TString whereClause;
2522 if (system == kDAQ)
2523 {
2524 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
2525 GetCurrentRun(), fCurrentDetector.Data(), fileId, source);
2526 }
2527 else if (system == kDCS)
2528 {
2529 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\";",
2530 GetCurrentRun(), fCurrentDetector.Data(), fileId);
2531 }
2532 else if (system == kHLT)
2533 {
2534 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DDLnumbers=\"%s\";",
2535 GetCurrentRun(), fCurrentDetector.Data(), fileId, source);
2536 }
2537
2538 delete aFXSarray;
9e080f92 2539
9d733021 2540 TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(system),
2541 now.GetSec(), whereClause.Data());
9e080f92 2542
9d733021 2543 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
9e080f92 2544
9d733021 2545 // Query execution
2546 TSQLResult* aResult;
2547 aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
2548 if (!aResult)
2549 {
2550 Log(fCurrentDetector, Form("UpdateTable - %s db: can't execute SQL query <%s>",
2551 GetSystemName(system), sqlQuery.Data()));
2552 result = kFALSE;
2553 continue;
2554 }
2555 delete aResult;
9e080f92 2556 }
9e080f92 2557 }
2558
9d733021 2559 return result;
9e080f92 2560}
57f50b3c 2561
3301427a 2562//______________________________________________________________________________________________
2563Bool_t AliShuttle::UpdateTableFailCase()
2564{
9827400b 2565 // Update FXS table filling time_processed field in all rows corresponding to current run and detector
2566 // this is called in case the preprocessor is declared failed for the current run, because
2567 // the fields are updated only in case of success
3301427a 2568
2569 Bool_t result = kTRUE;
2570
2571 for (UInt_t system=0; system<3; system++)
2572 {
2573 // check connection, in case connect
2574 if (!Connect(system))
2575 {
2576 Log(fCurrentDetector, Form("UpdateTableFailCase - Couldn't connect to %s FXS database",
2577 GetSystemName(system)));
2578 result = kFALSE;
2579 continue;
2580 }
2581
2582 TTimeStamp now; // now
2583
2584 // Loop on FXS list entries
2585
2586 TString whereClause = Form("where run=%d and detector=\"%s\";",
2587 GetCurrentRun(), fCurrentDetector.Data());
2588
2589
2590 TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(system),
2591 now.GetSec(), whereClause.Data());
2592
2593 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2594
2595 // Query execution
2596 TSQLResult* aResult;
2597 aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
2598 if (!aResult)
2599 {
2600 Log(fCurrentDetector, Form("UpdateTableFailCase - %s db: can't execute SQL query <%s>",
2601 GetSystemName(system), sqlQuery.Data()));
2602 result = kFALSE;
2603 continue;
2604 }
2605 delete aResult;
2606 }
2607
2608 return result;
2609}
2610
2bb7b766 2611//______________________________________________________________________________________________
2612Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status)
2613{
e7f62f16 2614 //
2615 // Update Shuttle logbook filling detector or shuttle_done column
2616 // ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
2617 //
57f50b3c 2618
2bb7b766 2619 // check connection, in case connect
be48e3ea 2620 if(!Connect(3)){
2bb7b766 2621 Log("SHUTTLE", "UpdateShuttleLogbook - Couldn't connect to DAQ Logbook.");
2622 return kFALSE;
57f50b3c 2623 }
2624
2bb7b766 2625 TString detName(detector);
2626 TString setClause;
ee6f7523 2627 if (detName == "shuttle_done" || detName == "shuttle_ignored")
e7f62f16 2628 {
2bb7b766 2629 setClause = "set shuttle_done=1";
e7f62f16 2630
ee6f7523 2631 if (detName == "shuttle_done")
b0e53b15 2632 {
2633 // Send the information to ML
2634 TMonaLisaText mlStatus("SHUTTLE_status", "Done");
e7f62f16 2635
b0e53b15 2636 TList mlList;
2637 mlList.Add(&mlStatus);
2638
ee6f7523 2639 TString mlID;
2640 mlID.Form("%d", GetCurrentRun());
2641 fMonaLisa->SendParameters(&mlList, mlID);
b0e53b15 2642 }
2bb7b766 2643 } else {
2bb7b766 2644 TString statusStr(status);
2645 if(statusStr.Contains("done", TString::kIgnoreCase) ||
2646 statusStr.Contains("failed", TString::kIgnoreCase)){
eba76848 2647 setClause = Form("set %s=\"%s\"", detector, status);
2bb7b766 2648 } else {
2649 Log("SHUTTLE",
2650 Form("UpdateShuttleLogbook - Invalid status <%s> for detector %s",
2651 status, detector));
2652 return kFALSE;
2653 }
2654 }
57f50b3c 2655
2bb7b766 2656 TString whereClause = Form("where run=%d", GetCurrentRun());
2657
441b0e9c 2658 TString sqlQuery = Form("update %s %s %s",
2659 fConfig->GetShuttlelbTable(), setClause.Data(), whereClause.Data());
57f50b3c 2660
2bb7b766 2661 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2662
2663 // Query execution
2664 TSQLResult* aResult;
be48e3ea 2665 aResult = dynamic_cast<TSQLResult*> (fServer[3]->Query(sqlQuery));
2bb7b766 2666 if (!aResult) {
2667 Log("SHUTTLE", Form("UpdateShuttleLogbook - Can't execute query <%s>", sqlQuery.Data()));
2668 return kFALSE;
57f50b3c 2669 }
2bb7b766 2670 delete aResult;
57f50b3c 2671
2672 return kTRUE;
2673}
2674
2675//______________________________________________________________________________________________
2bb7b766 2676Int_t AliShuttle::GetCurrentRun() const
2677{
9827400b 2678 //
2679 // Get current run from logbook entry
2680 //
57f50b3c 2681
2bb7b766 2682 return fLogbookEntry ? fLogbookEntry->GetRun() : -1;
57f50b3c 2683}
2684
2685//______________________________________________________________________________________________
2bb7b766 2686UInt_t AliShuttle::GetCurrentStartTime() const
2687{
9827400b 2688 //
2689 // get current start time
2690 //
57f50b3c 2691
2bb7b766 2692 return fLogbookEntry ? fLogbookEntry->GetStartTime() : 0;
57f50b3c 2693}
2694
2695//______________________________________________________________________________________________
2bb7b766 2696UInt_t AliShuttle::GetCurrentEndTime() const
2697{
9827400b 2698 //
2699 // get current end time from logbook entry
2700 //
57f50b3c 2701
2bb7b766 2702 return fLogbookEntry ? fLogbookEntry->GetEndTime() : 0;
57f50b3c 2703}
2704
675f64cd 2705//______________________________________________________________________________________________
2706UInt_t AliShuttle::GetCurrentYear() const
2707{
2708 //
2709 // Get current year from logbook entry
2710 //
2711
2712 if (!fLogbookEntry) return 0;
2713
2714 TTimeStamp startTime(GetCurrentStartTime());
2715 TString year = Form("%d",startTime.GetDate());
2716 year = year(0,4);
2717
2718 return year.Atoi();
2719}
2720
2721//______________________________________________________________________________________________
2722const char* AliShuttle::GetLHCPeriod() const
2723{
2724 //
2725 // Get current LHC period from logbook entry
2726 //
2727
2728 if (!fLogbookEntry) return 0;
2729
2730 return fLogbookEntry->GetRunParameter("LHCperiod");
2731}
2732
b948db8d 2733//______________________________________________________________________________________________
2734void AliShuttle::Log(const char* detector, const char* message)
2735{
9827400b 2736 //
2737 // Fill log string with a message
2738 //
b948db8d 2739
7d4cf768 2740 TString logRunDir = GetShuttleLogDir();
2741 if (GetCurrentRun() >=0)
2742 logRunDir += Form("/%d", GetCurrentRun());
2743
2744 void* dir = gSystem->OpenDirectory(logRunDir.Data());
84090f85 2745 if (dir == NULL) {
7d4cf768 2746 if (gSystem->mkdir(logRunDir.Data(), kTRUE)) {
36c99a6a 2747 AliError(Form("Can't open directory <%s>", GetShuttleLogDir()));
84090f85 2748 return;
2749 }
b948db8d 2750
84090f85 2751 } else {
2752 gSystem->FreeDirectory(dir);
2753 }
b948db8d 2754
cb343cfd 2755 TString toLog = Form("%s (%d): %s - ", TTimeStamp(time(0)).AsString("s"), getpid(), detector);
e7f62f16 2756 if (GetCurrentRun() >= 0)
2757 toLog += Form("run %d - ", GetCurrentRun());
2bb7b766 2758 toLog += Form("%s", message);
2759
84090f85 2760 AliInfo(toLog.Data());
ffa29e93 2761
2762 // if we redirect the log output already to the file, leave here
2763 if (fOutputRedirected && strcmp(detector, "SHUTTLE") != 0)
2764 return;
b948db8d 2765
ffa29e93 2766 TString fileName = GetLogFileName(detector);
e7f62f16 2767
84090f85 2768 gSystem->ExpandPathName(fileName);
2769
2770 ofstream logFile;
2771 logFile.open(fileName, ofstream::out | ofstream::app);
2772
2773 if (!logFile.is_open()) {
2774 AliError(Form("Could not open file %s", fileName.Data()));
2775 return;
2776 }
7bfb2090 2777
84090f85 2778 logFile << toLog.Data() << "\n";
b948db8d 2779
84090f85 2780 logFile.close();
b948db8d 2781}
2bb7b766 2782
ffa29e93 2783//______________________________________________________________________________________________
2784TString AliShuttle::GetLogFileName(const char* detector) const
2785{
2786 //
2787 // returns the name of the log file for a given sub detector
2788 //
2789
2790 TString fileName;
2791
2792 if (GetCurrentRun() >= 0)
7d4cf768 2793 {
2794 fileName.Form("%s/%d/%s_%d.log", GetShuttleLogDir(), GetCurrentRun(),
2795 detector, GetCurrentRun());
2796 } else {
ffa29e93 2797 fileName.Form("%s/%s.log", GetShuttleLogDir(), detector);
7d4cf768 2798 }
ffa29e93 2799
2800 return fileName;
2801}
2802
ee6f7523 2803//______________________________________________________________________________________________
2804void AliShuttle::SendAlive()
2805{
2806 // sends alive message to ML
2807
2808 TMonaLisaText mlStatus("SHUTTLE_status", "Alive");
2809
2810 TList mlList;
2811 mlList.Add(&mlStatus);
2812
2813 fMonaLisa->SendParameters(&mlList, "__PROCESSINGINFO__");
2814}
2815
2bb7b766 2816//______________________________________________________________________________________________
2817Bool_t AliShuttle::Collect(Int_t run)
2818{
9827400b 2819 //
2820 // Collects conditions data for all UNPROCESSED run written to DAQ LogBook in case of run = -1 (default)
2821 // If a dedicated run is given this run is processed
2822 //
2823 // In operational mode, this is the Shuttle function triggered by the EOR signal.
2824 //
2bb7b766 2825
eba76848 2826 if (run == -1)
2827 Log("SHUTTLE","Collect - Shuttle called. Collecting conditions data for unprocessed runs");
2828 else
2829 Log("SHUTTLE", Form("Collect - Shuttle called. Collecting conditions data for run %d", run));
cb343cfd 2830
2831 SetLastAction("Starting");
2bb7b766 2832
ee6f7523 2833 // create ML instance
2834 if (!fMonaLisa)
2835 fMonaLisa = new TMonaLisaWriter(fConfig->GetMonitorHost(), fConfig->GetMonitorTable());
2836
ee6f7523 2837 SendAlive();
fbc112e3 2838 CountOpenRuns();
ee6f7523 2839
2bb7b766 2840 TString whereClause("where shuttle_done=0");
eba76848 2841 if (run != -1)
2842 whereClause += Form(" and run=%d", run);
2bb7b766 2843
2844 TObjArray shuttleLogbookEntries;
be48e3ea 2845 if (!QueryShuttleLogbook(whereClause, shuttleLogbookEntries))
2846 {
cb343cfd 2847 Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
2bb7b766 2848 return kFALSE;
2849 }
2850
9e080f92 2851 if (shuttleLogbookEntries.GetEntries() == 0)
2852 {
2853 if (run == -1)
2854 Log("SHUTTLE","Collect - Found no UNPROCESSED runs in Shuttle logbook");
2855 else
2856 Log("SHUTTLE", Form("Collect - Run %d is already DONE "
2857 "or it does not exist in Shuttle logbook", run));
2858 return kTRUE;
2859 }
2860
be48e3ea 2861 for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
2862 fFirstUnprocessed[iDet] = kTRUE;
2863
fc5a4708 2864 if (run != -1)
be48e3ea 2865 {
2866 // query Shuttle logbook for earlier runs, check if some detectors are unprocessed,
2867 // flag them into fFirstUnprocessed array
2868 TString whereClause(Form("where shuttle_done=0 and run < %d", run));
2869 TObjArray tmpLogbookEntries;
2870 if (!QueryShuttleLogbook(whereClause, tmpLogbookEntries))
2871 {
2872 Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
2873 return kFALSE;
2874 }
2875
2876 TIter iter(&tmpLogbookEntries);
2877 AliShuttleLogbookEntry* anEntry = 0;
2878 while ((anEntry = dynamic_cast<AliShuttleLogbookEntry*> (iter.Next())))
2879 {
2880 for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
2881 {
2882 if (anEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
2883 {
2884 AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
2885 anEntry->GetRun(), GetDetName(iDet)));
2886 fFirstUnprocessed[iDet] = kFALSE;
2887 }
2888 }
2889
2890 }
2891
2892 }
2893
2894 if (!RetrieveConditionsData(shuttleLogbookEntries))
2895 {
cb343cfd 2896 Log("SHUTTLE", "Collect - Process of at least one run failed");
a3ca69fe 2897 CountOpenRuns();
2bb7b766 2898 return kFALSE;
2899 }
2900
36c99a6a 2901 Log("SHUTTLE", "Collect - Requested run(s) successfully processed");
a3ca69fe 2902 CountOpenRuns();
eba76848 2903 return kTRUE;
2bb7b766 2904}
2905
2bb7b766 2906//______________________________________________________________________________________________
2907Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
2908{
9827400b 2909 //
2910 // Retrieve conditions data for all runs that aren't processed yet
2911 //
2bb7b766 2912
2913 Bool_t hasError = kFALSE;
2914
2915 TIter iter(&dateEntries);
2916 AliShuttleLogbookEntry* anEntry;
2917
2918 while ((anEntry = (AliShuttleLogbookEntry*) iter.Next())){
2919 if (!Process(anEntry)){
2920 hasError = kTRUE;
2921 }
4b95672b 2922
2923 // clean SHUTTLE temp directory
d524ade6 2924 //TString filename = Form("%s/*.shuttle", GetShuttleTempDir());
2925 //RemoveFile(filename.Data());
2bb7b766 2926 }
2927
2928 return hasError == kFALSE;
2929}
cb343cfd 2930
2931//______________________________________________________________________________________________
2932ULong_t AliShuttle::GetTimeOfLastAction() const
2933{
9827400b 2934 //
2935 // Gets time of last action
2936 //
2937
cb343cfd 2938 ULong_t tmp;
36c99a6a 2939
cb343cfd 2940 fMonitoringMutex->Lock();
be48e3ea 2941
cb343cfd 2942 tmp = fLastActionTime;
36c99a6a 2943
cb343cfd 2944 fMonitoringMutex->UnLock();
36c99a6a 2945
cb343cfd 2946 return tmp;
2947}
2948
2949//______________________________________________________________________________________________
2950const TString AliShuttle::GetLastAction() const
2951{
9827400b 2952 //
cb343cfd 2953 // returns a string description of the last action
9827400b 2954 //
cb343cfd 2955
2956 TString tmp;
36c99a6a 2957
cb343cfd 2958 fMonitoringMutex->Lock();
2959
2960 tmp = fLastAction;
2961
2962 fMonitoringMutex->UnLock();
2963
36c99a6a 2964 return tmp;
cb343cfd 2965}
2966
2967//______________________________________________________________________________________________
2968void AliShuttle::SetLastAction(const char* action)
2969{
9827400b 2970 //
cb343cfd 2971 // updates the monitoring variables
9827400b 2972 //
36c99a6a 2973
cb343cfd 2974 fMonitoringMutex->Lock();
36c99a6a 2975
cb343cfd 2976 fLastAction = action;
2977 fLastActionTime = time(0);
2978
2979 fMonitoringMutex->UnLock();
2980}
eba76848 2981
2982//______________________________________________________________________________________________
2983const char* AliShuttle::GetRunParameter(const char* param)
2984{
9827400b 2985 //
2986 // returns run parameter read from DAQ logbook
2987 //
eba76848 2988
2989 if(!fLogbookEntry) {
2990 AliError("No logbook entry!");
2991 return 0;
2992 }
2993
2994 return fLogbookEntry->GetRunParameter(param);
2995}
57c1a579 2996
d386d623 2997//______________________________________________________________________________________________
9827400b 2998AliCDBEntry* AliShuttle::GetFromOCDB(const char* detector, const AliCDBPath& path)
d386d623 2999{
9827400b 3000 //
3001 // returns object from OCDB valid for current run
3002 //
d386d623 3003
9827400b 3004 if (fTestMode & kErrorOCDB)
3005 {
3006 Log(detector, "GetFromOCDB - In TESTMODE - Simulating error with OCDB");
3007 return 0;
3008 }
3009
d386d623 3010 AliCDBStorage *sto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
3011 if (!sto)
3012 {
9827400b 3013 Log(detector, "GetFromOCDB - Cannot activate main OCDB for query!");
d386d623 3014 return 0;
3015 }
3016
3017 return dynamic_cast<AliCDBEntry*> (sto->Get(path, GetCurrentRun()));
3018}
3019
57c1a579 3020//______________________________________________________________________________________________
3021Bool_t AliShuttle::SendMail()
3022{
9827400b 3023 //
3024 // sends a mail to the subdetector expert in case of preprocessor error
3025 //
3026
3027 if (fTestMode != kNone)
3028 return kTRUE;
fb40ead4 3029
926cbc0e 3030 if (!fConfig->SendMail())
3031 return kTRUE;
57c1a579 3032
57c1a579 3033 TString to="";
3034 TIter iterExperts(fConfig->GetResponsibles(fCurrentDetector));
3035 TObjString *anExpert=0;
3036 while ((anExpert = (TObjString*) iterExperts.Next()))
3037 {
3038 to += Form("%s,", anExpert->GetName());
3039 }
4508ab8b 3040 if (to.Length() > 0)
3041 to.Remove(to.Length()-1);
909732f7 3042 AliDebug(2, Form("to: %s",to.Data()));
57c1a579 3043
86aa42c3 3044 if (to.IsNull()) {
fb40ead4 3045 Log("SHUTTLE", "List of detector responsibles not set!");
36c99a6a 3046 return kFALSE;
3047 }
3048
fbc112e3 3049 void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
3050 if (dir == NULL)
3051 {
3052 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE))
3053 {
3054 Log("SHUTTLE", Form("SendMail - Can't open directory <%s>", GetShuttleLogDir()));
3055 return kFALSE;
3056 }
3057
3058 } else {
3059 gSystem->FreeDirectory(dir);
3060 }
3061
4508ab8b 3062 TString bodyFileName;
3063 bodyFileName.Form("%s/mail.body", GetShuttleLogDir());
3064 gSystem->ExpandPathName(bodyFileName);
3065
3066 ofstream mailBody;
3067 mailBody.open(bodyFileName, ofstream::out);
3068
3069 if (!mailBody.is_open())
3070 {
3071 Log("SHUTTLE", Form("Could not open mail body file %s", bodyFileName.Data()));
3072 return kFALSE;
3073 }
3074
fb40ead4 3075 TString cc="";
3076 TIter iterAdmins(fConfig->GetAdmins(AliShuttleConfig::kGlobal));
3077 TObjString *anAdmin=0;
3078 while ((anAdmin = (TObjString*) iterAdmins.Next()))
3079 {
3080 cc += Form("%s,", anAdmin->GetName());
3081 }
3082 if (cc.Length() > 0)
c19963db 3083 cc.Remove(cc.Length()-1);
fb40ead4 3084 AliDebug(2, Form("cc: %s",to.Data()));
57c1a579 3085
6a1146c4 3086 TString subject = Form("%s Shuttle preprocessor FAILED in run %d (run type = %s)!",
3087 fCurrentDetector.Data(), GetCurrentRun(), GetRunType());
909732f7 3088 AliDebug(2, Form("subject: %s", subject.Data()));
57c1a579 3089
3090 TString body = Form("Dear %s expert(s), \n\n", fCurrentDetector.Data());
3091 body += Form("SHUTTLE just detected that your preprocessor "
6a1146c4 3092 "failed processing run %d (run type = %s)!!\n\n",
3093 GetCurrentRun(), GetRunType());
7d4cf768 3094 body += Form("Please check %s status on the SHUTTLE monitoring page: \n\n",
3095 fCurrentDetector.Data());
b0e53b15 3096 if (fConfig->GetRunMode() == AliShuttleConfig::kTest)
3097 {
3098 body += Form("\thttp://pcalimonitor.cern.ch:8889/shuttle.jsp?time=168 \n\n");
3099 } else {
4508ab8b 3100 body += Form("\thttp://pcalimonitor.cern.ch/shuttle.jsp?instance=PROD&time=168 \n\n");
b0e53b15 3101 }
3102
7d4cf768 3103
3104 TString logFolder = "logs";
3105 if (fConfig->GetRunMode() == AliShuttleConfig::kProd)
3106 logFolder += "_PROD";
3107
3108
546242fb 3109 body += Form("Find the %s log for the current run on \n\n"
7d4cf768 3110 "\thttp://pcalishuttle01.cern.ch:8880/%s/%d/%s_%d.log \n\n",
3111 fCurrentDetector.Data(), logFolder.Data(), GetCurrentRun(),
3112 fCurrentDetector.Data(), GetCurrentRun());
3113 body += Form("The last 10 lines of %s log file are following:\n\n", fCurrentDetector.Data());
57c1a579 3114
909732f7 3115 AliDebug(2, Form("Body begin: %s", body.Data()));
57c1a579 3116
3117 mailBody << body.Data();
3118 mailBody.close();
3119 mailBody.open(bodyFileName, ofstream::out | ofstream::app);
3120
7d4cf768 3121 TString logFileName = Form("%s/%d/%s_%d.log", GetShuttleLogDir(),
3122 GetCurrentRun(), fCurrentDetector.Data(), GetCurrentRun());
57c1a579 3123 TString tailCommand = Form("tail -n 10 %s >> %s", logFileName.Data(), bodyFileName.Data());
3124 if (gSystem->Exec(tailCommand.Data()))
3125 {
3126 mailBody << Form("%s log file not found ...\n\n", fCurrentDetector.Data());
3127 }
3128
3129 TString endBody = Form("------------------------------------------------------\n\n");
675f64cd 3130 endBody += Form("In case of problems please contact the SHUTTLE core team.\n\n");
3131 endBody += "Please do not answer this message directly, it is automatically generated.\n\n";
3132 endBody += "Greetings,\n\n \t\t\tthe SHUTTLE\n";
3133
3134 AliDebug(2, Form("Body end: %s", endBody.Data()));
3135
3136 mailBody << endBody.Data();
3137
3138 mailBody.close();
3139
3140 // send mail!
3141 TString mailCommand = Form("mail -s \"%s\" -c %s %s < %s",
3142 subject.Data(),
3143 cc.Data(),
3144 to.Data(),
3145 bodyFileName.Data());
3146 AliDebug(2, Form("mail command: %s", mailCommand.Data()));
3147
3148 Bool_t result = gSystem->Exec(mailCommand.Data());
3149
3150 return result == 0;
3151}
3152
3153//______________________________________________________________________________________________
3154Bool_t AliShuttle::SendMailToDCS()
3155{
3156 //
fb40ead4 3157 // sends a mail to the DCS Amanda experts in case of DCS data point retrieval error
675f64cd 3158 //
3159
3160 if (fTestMode != kNone)
3161 return kTRUE;
3162
926cbc0e 3163 if (!fConfig->SendMail())
3164 return kTRUE;
3165
3166 if (!fFirstProcessing)
3167 return kTRUE;
fb40ead4 3168
675f64cd 3169 void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
3170 if (dir == NULL)
3171 {
3172 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE))
3173 {
3174 Log("SHUTTLE", Form("SendMailToDCS - Can't open directory <%s>", GetShuttleLogDir()));
3175 return kFALSE;
3176 }
3177
3178 } else {
3179 gSystem->FreeDirectory(dir);
3180 }
3181
3182 TString bodyFileName;
3183 bodyFileName.Form("%s/mail.body", GetShuttleLogDir());
3184 gSystem->ExpandPathName(bodyFileName);
3185
3186 ofstream mailBody;
3187 mailBody.open(bodyFileName, ofstream::out);
3188
3189 if (!mailBody.is_open())
3190 {
3191 Log("SHUTTLE", Form("SendMailToDCS - Could not open mail body file %s", bodyFileName.Data()));
3192 return kFALSE;
3193 }
3194
fb40ead4 3195 TString to="";
3196 TIter iterExperts(fConfig->GetAdmins(AliShuttleConfig::kAmanda));
3197 TObjString *anExpert=0;
3198 while ((anExpert = (TObjString*) iterExperts.Next()))
3199 {
3200 to += Form("%s,", anExpert->GetName());
3201 }
3202 if (to.Length() > 0)
3203 to.Remove(to.Length()-1);
675f64cd 3204 AliDebug(2, Form("to: %s",to.Data()));
3205
3206 if (to.IsNull()) {
fb40ead4 3207 Log("SHUTTLE", "List of Amanda server administrators not set!");
675f64cd 3208 return kFALSE;
3209 }
3210
fb40ead4 3211 TString cc="";
3212 TIter iterAdmins(fConfig->GetAdmins(AliShuttleConfig::kGlobal));
3213 TObjString *anAdmin=0;
3214 while ((anAdmin = (TObjString*) iterAdmins.Next()))
3215 {
3216 cc += Form("%s,", anAdmin->GetName());
3217 }
3218 if (cc.Length() > 0)
c19963db 3219 cc.Remove(cc.Length()-1);
fb40ead4 3220 AliDebug(2, Form("cc: %s",to.Data()));
675f64cd 3221
3222 TString subject = Form("Retrieval of data points for %s FAILED in run %d !",
3223 fCurrentDetector.Data(), GetCurrentRun());
3224 AliDebug(2, Form("subject: %s", subject.Data()));
3225
3226 TString body = Form("Dear DCS experts, \n\n");
3227 body += Form("SHUTTLE couldn\'t retrieve the data points for detector %s "
3228 "in run %d!!\n\n", fCurrentDetector.Data(), GetCurrentRun());
7d4cf768 3229 body += Form("Please check %s status on the SHUTTLE monitoring page: \n\n",
3230 fCurrentDetector.Data());
b0e53b15 3231 if (fConfig->GetRunMode() == AliShuttleConfig::kTest)
3232 {
3233 body += Form("\thttp://pcalimonitor.cern.ch:8889/shuttle.jsp?time=168 \n\n");
3234 } else {
3235 body += Form("\thttp://pcalimonitor.cern.ch/shuttle.jsp?instance=PROD?time=168 \n\n");
3236 }
7d4cf768 3237
3238 TString logFolder = "logs";
3239 if (fConfig->GetRunMode() == AliShuttleConfig::kProd)
3240 logFolder += "_PROD";
3241
3242
675f64cd 3243 body += Form("Find the %s log for the current run on \n\n"
7d4cf768 3244 "\thttp://pcalishuttle01.cern.ch:8880/%s/%d/%s_%d.log \n\n",
3245 fCurrentDetector.Data(), logFolder.Data(), GetCurrentRun(),
3246 fCurrentDetector.Data(), GetCurrentRun());
3247 body += Form("The last 10 lines of %s log file are following:\n\n", fCurrentDetector.Data());
675f64cd 3248
3249 AliDebug(2, Form("Body begin: %s", body.Data()));
3250
3251 mailBody << body.Data();
3252 mailBody.close();
3253 mailBody.open(bodyFileName, ofstream::out | ofstream::app);
3254
7d4cf768 3255 TString logFileName = Form("%s/%d/%s_%d.log", GetShuttleLogDir(), GetCurrentRun(),
3256 fCurrentDetector.Data(), GetCurrentRun());
675f64cd 3257 TString tailCommand = Form("tail -n 10 %s >> %s", logFileName.Data(), bodyFileName.Data());
3258 if (gSystem->Exec(tailCommand.Data()))
3259 {
3260 mailBody << Form("%s log file not found ...\n\n", fCurrentDetector.Data());
3261 }
3262
3263 TString endBody = Form("------------------------------------------------------\n\n");
36c99a6a 3264 endBody += Form("In case of problems please contact the SHUTTLE core team.\n\n");
3265 endBody += "Please do not answer this message directly, it is automatically generated.\n\n";
546242fb 3266 endBody += "Greetings,\n\n \t\t\tthe SHUTTLE\n";
57c1a579 3267
909732f7 3268 AliDebug(2, Form("Body end: %s", endBody.Data()));
57c1a579 3269
3270 mailBody << endBody.Data();
3271
3272 mailBody.close();
3273
3274 // send mail!
3275 TString mailCommand = Form("mail -s \"%s\" -c %s %s < %s",
3276 subject.Data(),
3277 cc.Data(),
3278 to.Data(),
3279 bodyFileName.Data());
909732f7 3280 AliDebug(2, Form("mail command: %s", mailCommand.Data()));
57c1a579 3281
3282 Bool_t result = gSystem->Exec(mailCommand.Data());
3283
3284 return result == 0;
3285}
d386d623 3286
441b0e9c 3287//______________________________________________________________________________________________
9827400b 3288const char* AliShuttle::GetRunType()
441b0e9c 3289{
9827400b 3290 //
3291 // returns run type read from "run type" logbook
3292 //
441b0e9c 3293
3294 if(!fLogbookEntry) {
3295 AliError("No logbook entry!");
3296 return 0;
3297 }
3298
9827400b 3299 return fLogbookEntry->GetRunType();
441b0e9c 3300}
3301
4859271b 3302//______________________________________________________________________________________________
3303Bool_t AliShuttle::GetHLTStatus()
3304{
3305 // Return HLT status (ON=1 OFF=0)
3306 // Converts the HLT status from the status string read in the run logbook (not just a bool)
3307
3308 if(!fLogbookEntry) {
3309 AliError("No logbook entry!");
3310 return 0;
3311 }
3312
3313 // TODO implement when HLTStatus is inserted in run logbook
3314 //TString hltStatus = fLogbookEntry->GetRunParameter("HLTStatus");
3315 //if(hltStatus == "OFF") {return kFALSE};
3316
3317 return kTRUE;
3318}
3319
d386d623 3320//______________________________________________________________________________________________
3321void AliShuttle::SetShuttleTempDir(const char* tmpDir)
3322{
9827400b 3323 //
3324 // sets Shuttle temp directory
3325 //
d386d623 3326
3327 fgkShuttleTempDir = gSystem->ExpandPathName(tmpDir);
3328}
3329
3330//______________________________________________________________________________________________
3331void AliShuttle::SetShuttleLogDir(const char* logDir)
3332{
9827400b 3333 //
3334 // sets Shuttle log directory
3335 //
d386d623 3336
3337 fgkShuttleLogDir = gSystem->ExpandPathName(logDir);
3338}