git://git.uio.no / u/mrichter/AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
new lookup table for sim. data with numberof TRMs
[u/mrichter/AliRoot.git] / SHUTTLE / AliShuttle.cxx
CommitLineData
73abe331 1/**************************************************************************
2 * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16/*
17$Log$
3301427a 18Revision 1.32 2007/02/28 10:41:56 acolla
19Run type field added in SHUTTLE framework. Run type is read from "run type" logbook and retrieved by
20AliPreprocessor::GetRunType() function.
21Added some ldap definition files.
22
d386d623 23Revision 1.30 2007/02/13 11:23:21 acolla
24Moved getters and setters of Shuttle's main OCDB/Reference, local
25OCDB/Reference, temp and log folders to AliShuttleInterface
26
9d733021 27Revision 1.27 2007/01/30 17:52:42 jgrosseo
28adding monalisa monitoring
29
e7f62f16 30Revision 1.26 2007/01/23 19:20:03 acolla
31Removed old ldif files, added TOF, MCH ldif files. Added some options in
32AliShuttleConfig::Print. Added in Ali Shuttle: SetShuttleTempDir and
33SetShuttleLogDir
34
36c99a6a 35Revision 1.25 2007/01/15 19:13:52 acolla
36Moved some AliInfo to AliDebug in SendMail function
37
fc5a4708 38Revision 1.21 2006/12/07 08:51:26 jgrosseo
39update (alberto):
40table, db names in ldap configuration
41added GRP preprocessor
42DCS data can also be retrieved by data point
43
2c15234c 44Revision 1.20 2006/11/16 16:16:48 jgrosseo
45introducing strict run ordering flag
46removed giving preprocessor name to preprocessor, they have to know their name themselves ;-)
47
be48e3ea 48Revision 1.19 2006/11/06 14:23:04 jgrosseo
49major update (Alberto)
50o) reading of run parameters from the logbook
51o) online offline naming conversion
52o) standalone DCSclient package
53
eba76848 54Revision 1.18 2006/10/20 15:22:59 jgrosseo
55o) Adding time out to the execution of the preprocessors: The Shuttle forks and the parent process monitors the child
56o) Merging Collect, CollectAll, CollectNew function
57o) Removing implementation of empty copy constructors (declaration still there!)
58
cb343cfd 59Revision 1.17 2006/10/05 16:20:55 jgrosseo
60adapting to new CDB classes
61
6ec0e06c 62Revision 1.16 2006/10/05 15:46:26 jgrosseo
63applying to the new interface
64
481441a2 65Revision 1.15 2006/10/02 16:38:39 jgrosseo
66update (alberto):
67fixed memory leaks
68storing of objects that failed to be stored to the grid before
69interfacing of shuttle status table in daq system
70
2bb7b766 71Revision 1.14 2006/08/29 09:16:05 jgrosseo
72small update
73
85a80aa9 74Revision 1.13 2006/08/15 10:50:00 jgrosseo
75effc++ corrections (alberto)
76
4f0ab988 77Revision 1.12 2006/08/08 14:19:29 jgrosseo
78Update to shuttle classes (Alberto)
79
80- Possibility to set the full object's path in the Preprocessor's and
81Shuttle's Store functions
82- Possibility to extend the object's run validity in the same classes
83("startValidity" and "validityInfinite" parameters)
84- Implementation of the StoreReferenceData function to store reference
85data in a dedicated CDB storage.
86
84090f85 87Revision 1.11 2006/07/21 07:37:20 jgrosseo
88last run is stored after each run
89
7bfb2090 90Revision 1.10 2006/07/20 09:54:40 jgrosseo
91introducing status management: The processing per subdetector is divided into several steps,
92after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
93can keep track of the number of failures and skips further processing after a certain threshold is
94exceeded. These thresholds can be configured in LDAP.
95
5164a766 96Revision 1.9 2006/07/19 10:09:55 jgrosseo
97new configuration, accesst to DAQ FES (Alberto)
98
57f50b3c 99Revision 1.8 2006/07/11 12:44:36 jgrosseo
100adding parameters for extended validity range of data produced by preprocessor
101
17111222 102Revision 1.7 2006/07/10 14:37:09 jgrosseo
103small fix + todo comment
104
e090413b 105Revision 1.6 2006/07/10 13:01:41 jgrosseo
106enhanced storing of last sucessfully processed run (alberto)
107
a7160fe9 108Revision 1.5 2006/07/04 14:59:57 jgrosseo
109revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
110
45a493ce 111Revision 1.4 2006/06/12 09:11:16 jgrosseo
112coding conventions (Alberto)
113
58bc3020 114Revision 1.3 2006/06/06 14:26:40 jgrosseo
115o) removed files that were moved to STEER
116o) shuttle updated to follow the new interface (Alberto)
117
b948db8d 118Revision 1.2 2006/03/07 07:52:34 hristov
119New version (B.Yordanov)
120
d477ad88 121Revision 1.6 2005/11/19 17:19:14 byordano
122RetrieveDATEEntries and RetrieveConditionsData added
123
124Revision 1.5 2005/11/19 11:09:27 byordano
125AliShuttle declaration added
126
127Revision 1.4 2005/11/17 17:47:34 byordano
128TList changed to TObjArray
129
130Revision 1.3 2005/11/17 14:43:23 byordano
131import to local CVS
132
133Revision 1.1.1.1 2005/10/28 07:33:58 hristov
134Initial import as subdirectory in AliRoot
135
73abe331 136Revision 1.2 2005/09/13 08:41:15 byordano
137default startTime endTime added
138
139Revision 1.4 2005/08/30 09:13:02 byordano
140some docs added
141
142Revision 1.3 2005/08/29 21:15:47 byordano
143some docs added
144
145*/
146
147//
148// This class is the main manager for AliShuttle.
149// It organizes the data retrieval from DCS and call the
b948db8d 150// interface methods of AliPreprocessor.
73abe331 151// For every detector in AliShuttleConfgi (see AliShuttleConfig),
152// data for its set of aliases is retrieved. If there is registered
b948db8d 153// AliPreprocessor for this detector then it will be used
154// accroding to the schema (see AliPreprocessor).
155// If there isn't registered AliPreprocessor than the retrieved
73abe331 156// data is stored automatically to the undelying AliCDBStorage.
157// For detSpec is used the alias name.
158//
159
160#include "AliShuttle.h"
161
162#include "AliCDBManager.h"
163#include "AliCDBStorage.h"
164#include "AliCDBId.h"
84090f85 165#include "AliCDBRunRange.h"
166#include "AliCDBPath.h"
5164a766 167#include "AliCDBEntry.h"
73abe331 168#include "AliShuttleConfig.h"
eba76848 169#include "DCSClient/AliDCSClient.h"
73abe331 170#include "AliLog.h"
b948db8d 171#include "AliPreprocessor.h"
5164a766 172#include "AliShuttleStatus.h"
2bb7b766 173#include "AliShuttleLogbookEntry.h"
73abe331 174
57f50b3c 175#include <TSystem.h>
58bc3020 176#include <TObject.h>
b948db8d 177#include <TString.h>
57f50b3c 178#include <TTimeStamp.h>
73abe331 179#include <TObjString.h>
57f50b3c 180#include <TSQLServer.h>
181#include <TSQLResult.h>
182#include <TSQLRow.h>
cb343cfd 183#include <TMutex.h>
73abe331 184
e7f62f16 185#include <TMonaLisaWriter.h>
186
5164a766 187#include <fstream>
188
cb343cfd 189#include <sys/types.h>
190#include <sys/wait.h>
191
73abe331 192ClassImp(AliShuttle)
193
10a5a932 194Bool_t AliShuttle::fgkProcessDCS(kTRUE);
57f50b3c 195
b948db8d 196//______________________________________________________________________________________________
197AliShuttle::AliShuttle(const AliShuttleConfig* config,
198 UInt_t timeout, Int_t retries):
4f0ab988 199fConfig(config),
200fTimeout(timeout), fRetries(retries),
201fPreprocessorMap(),
2bb7b766 202fLogbookEntry(0),
eba76848 203fCurrentDetector(),
85a80aa9 204fStatusEntry(0),
cb343cfd 205fMonitoringMutex(0),
eba76848 206fLastActionTime(0),
e7f62f16 207fLastAction(),
208fMonaLisa(0)
73abe331 209{
210 //
211 // config: AliShuttleConfig used
73abe331 212 // timeout: timeout used for AliDCSClient connection
213 // retries: the number of retries in case of connection error.
214 //
215
57f50b3c 216 if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
be48e3ea 217 for(int iSys=0;iSys<4;iSys++) {
57f50b3c 218 fServer[iSys]=0;
be48e3ea 219 if (iSys < 3)
2c15234c 220 fFXSlist[iSys].SetOwner(kTRUE);
57f50b3c 221 }
2bb7b766 222 fPreprocessorMap.SetOwner(kTRUE);
be48e3ea 223
224 for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
225 fFirstUnprocessed[iDet] = kFALSE;
226
cb343cfd 227 fMonitoringMutex = new TMutex();
58bc3020 228}
229
b948db8d 230//______________________________________________________________________________________________
57f50b3c 231AliShuttle::~AliShuttle()
58bc3020 232{
233// destructor
234
b948db8d 235 fPreprocessorMap.DeleteAll();
be48e3ea 236 for(int iSys=0;iSys<4;iSys++)
57f50b3c 237 if(fServer[iSys]) {
238 fServer[iSys]->Close();
239 delete fServer[iSys];
eba76848 240 fServer[iSys] = 0;
57f50b3c 241 }
2bb7b766 242
243 if (fStatusEntry){
244 delete fStatusEntry;
245 fStatusEntry = 0;
246 }
cb343cfd 247
248 if (fMonitoringMutex)
249 {
250 delete fMonitoringMutex;
251 fMonitoringMutex = 0;
252 }
73abe331 253}
254
b948db8d 255//______________________________________________________________________________________________
57f50b3c 256void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
58bc3020 257{
73abe331 258 //
b948db8d 259 // Registers new AliPreprocessor.
73abe331 260 // It uses GetName() for indentificator of the pre processor.
261 // The pre processor is registered it there isn't any other
262 // with the same identificator (GetName()).
263 //
264
eba76848 265 const char* detName = preprocessor->GetName();
266 if(GetDetPos(detName) < 0)
267 AliFatal(Form("********** !!!!! Invalid detector name: %s !!!!! **********", detName));
268
269 if (fPreprocessorMap.GetValue(detName)) {
270 AliWarning(Form("AliPreprocessor %s is already registered!", detName));
73abe331 271 return;
272 }
273
eba76848 274 fPreprocessorMap.Add(new TObjString(detName), preprocessor);
73abe331 275}
b948db8d 276//______________________________________________________________________________________________
3301427a 277Bool_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
84090f85 278 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
73abe331 279{
84090f85 280 // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
281 // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
282 // using this function. Use StoreReferenceData instead!
3301427a 283 // It calls StoreLocally function which temporarily stores the data locally; when the preprocessor
284 // finishes the data are transferred to the main storage (Grid).
b948db8d 285
3301427a 286 return StoreLocally(fgkLocalCDB, path, object, metaData, validityStart, validityInfinite);
84090f85 287
288}
289
290//______________________________________________________________________________________________
3301427a 291Bool_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData)
84090f85 292{
293 // Stores a CDB object in the storage for reference data. This objects will not be available during
294 // offline reconstrunction. Use this function for reference data only!
3301427a 295 // It calls StoreLocally function which temporarily stores the data locally; when the preprocessor
296 // finishes the data are transferred to the main storage (Grid).
85a80aa9 297
3301427a 298 return StoreLocally(fgkLocalRefStorage, path, object, metaData);
84090f85 299
85a80aa9 300}
301
302//______________________________________________________________________________________________
3301427a 303Bool_t AliShuttle::StoreLocally(const TString& localUri,
85a80aa9 304 const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
305 Int_t validityStart, Bool_t validityInfinite)
306{
3301427a 307 // Store object temporarily in local storage. Parameters are passed by Store and StoreReferenceData functions.
308 // when the preprocessor finishes the data are transferred to the main storage (Grid).
85a80aa9 309 // The parameters are:
3301427a 310 // 1) Uri of the backup storage (Local)
311 // 2) the object's path.
312 // 3) the object to be stored
313 // 4) the metaData to be associated with the object
314 // 5) the validity start run number w.r.t. the current run,
84090f85 315 // if the data is valid only for this run leave the default 0
3301427a 316 // 6) specifies if the calibration data is valid for infinity (this means until updated),
84090f85 317 // typical for calibration runs, the default is kFALSE
318 //
3301427a 319 // returns 0 if fail, 1 otherwise
84090f85 320
3301427a 321 const char* cdbType = (localUri == fgkLocalCDB) ? "CDB" : "Reference";
2bb7b766 322
85a80aa9 323 Int_t firstRun = GetCurrentRun() - validityStart;
84090f85 324 if(firstRun < 0) {
2bb7b766 325 AliError("First valid run happens to be less than 0! Setting it to 0.");
84090f85 326 firstRun=0;
327 }
328
329 Int_t lastRun = -1;
330 if(validityInfinite) {
331 lastRun = AliCDBRunRange::Infinity();
332 } else {
333 lastRun = GetCurrentRun();
334 }
335
3301427a 336 // Version is set to current run, it will be used later to transfer data to Grid
337 AliCDBId id(path, firstRun, lastRun, GetCurrentRun(), -1);
2bb7b766 338
339 if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
340 TObjString runUsed = Form("%d", GetCurrentRun());
9e080f92 341 metaData->SetProperty("RunUsed(TObjString)", runUsed.Clone());
2bb7b766 342 }
84090f85 343
3301427a 344 Bool_t result = kFALSE;
84090f85 345
3301427a 346 if (!(AliCDBManager::Instance()->GetStorage(localUri))) {
347 Log("SHUTTLE", Form("StoreLocally - Cannot activate local %s storage", cdbType));
84090f85 348 } else {
3301427a 349 result = AliCDBManager::Instance()->GetStorage(localUri)
84090f85 350 ->Put(object, id, metaData);
351 }
352
353 if(!result) {
354
3301427a 355 Log("SHUTTLE", Form("StoreLocally - Can't store %s data!", fCurrentDetector.Data()));
356 }
2bb7b766 357
3301427a 358 return result;
359}
84090f85 360
3301427a 361//______________________________________________________________________________________________
362Bool_t AliShuttle::StoreOCDB()
363{
364 // Called when preprocessor ends successfully or when previous storage attempt failed (kStoreError status)
365 // Calls underlying StoreOCDB(const char*) function twice, for OCDB and Reference storage.
366
367 AliInfo("Storing OCDB data ...");
368 Bool_t resultCDB = StoreOCDB(fgkMainCDB);
369
370 AliInfo("Storing reference data ...");
371 Bool_t resultRef = StoreOCDB(fgkMainRefStorage);
372
373 return resultCDB && resultRef;
374}
375
376//______________________________________________________________________________________________
377Bool_t AliShuttle::StoreOCDB(const TString& gridURI)
378{
379 //
380 // Called by StoreOCDB(), performs actual storage to the main OCDB and reference storages (Grid)
381 //
382
383 TObjArray* gridIds=0;
384
385 Bool_t result = kTRUE;
386
387 const char* type = 0;
388 TString localURI;
389 if(gridURI == fgkMainCDB) {
390 type = "OCDB";
391 localURI = fgkLocalCDB;
392 } else if(gridURI == fgkMainRefStorage) {
393 type = "reference";
394 localURI = fgkLocalRefStorage;
395 } else {
396 AliError(Form("Invalid storage URI: %s", gridURI.Data()));
397 return kFALSE;
398 }
399
400 AliCDBManager* man = AliCDBManager::Instance();
401
402 AliCDBStorage *gridSto = man->GetStorage(gridURI);
403 if(!gridSto) {
404 Log("SHUTTLE",
405 Form("StoreOCDB - cannot activate main %s storage", type));
406 return kFALSE;
407 }
408
409 gridIds = gridSto->GetQueryCDBList();
410
411 // get objects previously stored in local CDB
412 AliCDBStorage *localSto = man->GetStorage(localURI);
413 if(!localSto) {
414 Log("SHUTTLE",
415 Form("StoreOCDB - cannot activate local %s storage", type));
416 return kFALSE;
417 }
418 AliCDBPath aPath(GetOfflineDetName(fCurrentDetector.Data()),"*","*");
419 // Local objects were stored with current run as Grid version!
420 TList* localEntries = localSto->GetAll(aPath.GetPath(), GetCurrentRun(), GetCurrentRun());
421 localEntries->SetOwner(1);
422
423 // loop on local stored objects
424 TIter localIter(localEntries);
425 AliCDBEntry *aLocEntry = 0;
426 while((aLocEntry = dynamic_cast<AliCDBEntry*> (localIter.Next()))){
427 aLocEntry->SetOwner(1);
428 AliCDBId aLocId = aLocEntry->GetId();
429 aLocEntry->SetVersion(-1);
430 aLocEntry->SetSubVersion(-1);
431
432 // If local object is valid up to infinity we store it only if it is
433 // the first unprocessed run!
434 if (aLocId.GetLastRun() == AliCDBRunRange::Infinity() &&
435 !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
436 {
437 Log("SHUTTLE", Form("StoreOCDB - %s: object %s has validity infinite but "
438 "there are previous unprocessed runs!",
439 fCurrentDetector.Data(), aLocId.GetPath().Data()));
440 continue;
441 }
442
443 // loop on Grid valid Id's
444 Bool_t store = kTRUE;
445 TIter gridIter(gridIds);
446 AliCDBId* aGridId = 0;
447 while((aGridId = dynamic_cast<AliCDBId*> (gridIter.Next()))){
448 if(aGridId->GetPath() != aLocId.GetPath()) continue;
449 // skip all objects valid up to infinity
450 if(aGridId->GetLastRun() == AliCDBRunRange::Infinity()) continue;
451 // if we get here, it means there's already some more recent object stored on Grid!
452 store = kFALSE;
453 break;
454 }
455
456 // If we get here, the file can be stored!
457 Bool_t storeOk = gridSto->Put(aLocEntry);
458 if(!store || storeOk){
459
460 if (!store)
461 {
462 Log(fCurrentDetector.Data(),
463 Form("StoreOCDB - A more recent object already exists in %s storage: <%s>",
464 type, aGridId->ToString().Data()));
465 } else {
466 Log("SHUTTLE",
467 Form("StoreOCDB - Object <%s> successfully put into %s storage",
468 aLocId.ToString().Data(), type));
469 }
84090f85 470
3301427a 471 // removing local filename...
472 TString filename;
473 localSto->IdToFilename(aLocId, filename);
474 AliInfo(Form("Removing local file %s", filename.Data()));
475 RemoveFile(filename.Data());
476 continue;
477 } else {
478 Log("SHUTTLE",
479 Form("StoreOCDB - Grid %s storage of object <%s> failed",
480 type, aLocId.ToString().Data()));
481 result = kFALSE;
b948db8d 482 }
483 }
3301427a 484 localEntries->Clear();
2bb7b766 485
b948db8d 486 return result;
3301427a 487}
488
489//______________________________________________________________________________________________
490void AliShuttle::CleanLocalStorage(const TString& uri)
491{
492// Called in case the preprocessor is declared failed. Remove remaining objects from the local storages.
493
494 const char* type = 0;
495 if(uri == fgkLocalCDB) {
496 type = "OCDB";
497 } else if(uri == fgkLocalRefStorage) {
498 type = "reference";
499 } else {
500 AliError(Form("Invalid storage URI: %s", uri.Data()));
501 return;
502 }
503
504 AliCDBManager* man = AliCDBManager::Instance();
b948db8d 505
3301427a 506 // open local storage
507 AliCDBStorage *localSto = man->GetStorage(uri);
508 if(!localSto) {
509 Log("SHUTTLE",
510 Form("CleanLocalStorage - cannot activate local %s storage", type));
511 return;
512 }
513
514 TString filename(Form("%s/%s/*/Run*_v%d_s*.root",
515 localSto->GetBaseFolder().Data(), fCurrentDetector.Data(), GetCurrentRun()));
516
517 AliInfo(Form("filename = %s", filename.Data()));
518
519 AliInfo(Form("Removing remaining local files from run %d and detector %s ...",
520 GetCurrentRun(), fCurrentDetector.Data()));
521
522 RemoveFile(filename.Data());
523
524}
525
526//______________________________________________________________________________________________
527void AliShuttle::RemoveFile(const char* filename)
528{
529// removes local file
530
531 TString command(Form("rm -f %s", filename));
532
533 Int_t result = gSystem->Exec(command.Data());
534 if(result != 0)
535 {
536 Log("SHUTTLE", Form("RemoveFile - %s: Cannot remove file %s!",
537 fCurrentDetector.Data(), filename));
538 }
73abe331 539}
540
b948db8d 541//______________________________________________________________________________________________
5164a766 542AliShuttleStatus* AliShuttle::ReadShuttleStatus()
543{
2bb7b766 544// Reads the AliShuttleStatus from the CDB
5164a766 545
2bb7b766 546 if (fStatusEntry){
547 delete fStatusEntry;
548 fStatusEntry = 0;
549 }
5164a766 550
10a5a932 551 fStatusEntry = AliCDBManager::Instance()->GetStorage(GetLocalCDB())
2bb7b766 552 ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), GetCurrentRun());
5164a766 553
2bb7b766 554 if (!fStatusEntry) return 0;
555 fStatusEntry->SetOwner(1);
5164a766 556
2bb7b766 557 AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
558 if (!status) {
559 AliError("Invalid object stored to CDB!");
560 return 0;
561 }
5164a766 562
2bb7b766 563 return status;
5164a766 564}
565
566//______________________________________________________________________________________________
7bfb2090 567Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
5164a766 568{
2bb7b766 569// writes the status for one subdetector
570
571 if (fStatusEntry){
572 delete fStatusEntry;
573 fStatusEntry = 0;
574 }
5164a766 575
2bb7b766 576 Int_t run = GetCurrentRun();
5164a766 577
2bb7b766 578 AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), run, run);
5164a766 579
2bb7b766 580 fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
581 fStatusEntry->SetOwner(1);
5164a766 582
2bb7b766 583 UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
7bfb2090 584
2bb7b766 585 if (!result) {
3301427a 586 Log("SHUTTLE", Form("WriteShuttleStatus - Failed for %s, run %d",
587 fCurrentDetector.Data(), run));
2bb7b766 588 return kFALSE;
589 }
e7f62f16 590
591 SendMLInfo();
7bfb2090 592
2bb7b766 593 return kTRUE;
5164a766 594}
595
596//______________________________________________________________________________________________
597void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
598{
599 // changes the AliShuttleStatus for the given detector and run to the given status
600
2bb7b766 601 if (!fStatusEntry){
602 AliError("UNEXPECTED: fStatusEntry empty");
603 return;
604 }
5164a766 605
2bb7b766 606 AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
5164a766 607
2bb7b766 608 if (!status){
3301427a 609 Log("SHUTTLE", "UNEXPECTED: status could not be read from current CDB entry");
2bb7b766 610 return;
611 }
5164a766 612
2c15234c 613 TString actionStr = Form("UpdateShuttleStatus - %s: Changing state from %s to %s",
eba76848 614 fCurrentDetector.Data(),
36c99a6a 615 status->GetStatusName(),
eba76848 616 status->GetStatusName(newStatus));
cb343cfd 617 Log("SHUTTLE", actionStr);
618 SetLastAction(actionStr);
5164a766 619
2bb7b766 620 status->SetStatus(newStatus);
621 if (increaseCount) status->IncreaseCount();
5164a766 622
2bb7b766 623 AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
e7f62f16 624
625 SendMLInfo();
5164a766 626}
e7f62f16 627
628//______________________________________________________________________________________________
629void AliShuttle::SendMLInfo()
630{
631 //
632 // sends ML information about the current status of the current detector being processed
633 //
634
635 AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
636
637 if (!status){
3301427a 638 Log("SHUTTLE", "SendMLInfo - UNEXPECTED: status could not be read from current CDB entry");
e7f62f16 639 return;
640 }
641
642 TMonaLisaText mlStatus(Form("%s_status", fCurrentDetector.Data()), status->GetStatusName());
643 TMonaLisaValue mlRetryCount(Form("%s_count", fCurrentDetector.Data()), status->GetCount());
644
645 TList mlList;
646 mlList.Add(&mlStatus);
647 mlList.Add(&mlRetryCount);
648
649 fMonaLisa->SendParameters(&mlList);
650}
651
5164a766 652//______________________________________________________________________________________________
653Bool_t AliShuttle::ContinueProcessing()
654{
2bb7b766 655// this function reads the AliShuttleStatus information from CDB and
656// checks if the processing should be continued
657// if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
658
57c1a579 659 if (!fConfig->HostProcessDetector(fCurrentDetector)) return kFALSE;
660
661 AliPreprocessor* aPreprocessor =
662 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
663 if (!aPreprocessor)
664 {
665 AliInfo(Form("%s: no preprocessor registered", fCurrentDetector.Data()));
666 return kFALSE;
667 }
668
2bb7b766 669 AliShuttleLogbookEntry::Status entryStatus =
eba76848 670 fLogbookEntry->GetDetectorStatus(fCurrentDetector);
2bb7b766 671
672 if(entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
9e080f92 673 AliInfo(Form("ContinueProcessing - %s is %s",
2bb7b766 674 fCurrentDetector.Data(),
675 fLogbookEntry->GetDetectorStatusName(entryStatus)));
676 return kFALSE;
677 }
678
679 // if we get here, according to Shuttle logbook subdetector is in UNPROCESSED state
be48e3ea 680
681 // check if current run is first unprocessed run for current detector
682 if (fConfig->StrictRunOrder(fCurrentDetector) &&
683 !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
684 {
685 Log("SHUTTLE", Form("ContinueProcessing - %s requires strict run ordering but this is not the first unprocessed run!"));
686 return kFALSE;
687 }
688
2bb7b766 689 AliShuttleStatus* status = ReadShuttleStatus();
690 if (!status) {
691 // first time
692 Log("SHUTTLE", Form("ContinueProcessing - %s: Processing first time",
693 fCurrentDetector.Data()));
694 status = new AliShuttleStatus(AliShuttleStatus::kStarted);
695 return WriteShuttleStatus(status);
696 }
697
698 // The following two cases shouldn't happen if Shuttle Logbook was correctly updated.
699 // If it happens it may mean Logbook updating failed... let's do it now!
700 if (status->GetStatus() == AliShuttleStatus::kDone ||
701 status->GetStatus() == AliShuttleStatus::kFailed){
702 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s. Updating Shuttle Logbook",
703 fCurrentDetector.Data(),
704 status->GetStatusName(status->GetStatus())));
705 UpdateShuttleLogbook(fCurrentDetector.Data(),
706 status->GetStatusName(status->GetStatus()));
707 return kFALSE;
708 }
709
3301427a 710 if (status->GetStatus() == AliShuttleStatus::kStoreError) {
2bb7b766 711 Log("SHUTTLE",
712 Form("ContinueProcessing - %s: Grid storage of one or more objects failed. Trying again now",
713 fCurrentDetector.Data()));
3301427a 714 if(StoreOCDB()){
715 Log("SHUTTLE", Form("ContinueProcessing - %s: all objects successfully stored into main storage",
716 fCurrentDetector.Data()));
2bb7b766 717 UpdateShuttleStatus(AliShuttleStatus::kDone);
718 UpdateShuttleLogbook(fCurrentDetector.Data(), "DONE");
719 } else {
720 Log("SHUTTLE",
721 Form("ContinueProcessing - %s: Grid storage failed again",
722 fCurrentDetector.Data()));
e7f62f16 723 // trigger ML information manually because we do not had a status change
724 SendMLInfo();
2bb7b766 725 }
726 return kFALSE;
727 }
728
729 // if we get here, there is a restart
57c1a579 730 Bool_t cont = kFALSE;
2bb7b766 731
732 // abort conditions
cb343cfd 733 if (status->GetCount() >= fConfig->GetMaxRetries()) {
57c1a579 734 Log("SHUTTLE", Form("ContinueProcessing - %s failed %d times in status %s - "
735 "Updating Shuttle Logbook", fCurrentDetector.Data(),
2bb7b766 736 status->GetCount(), status->GetStatusName()));
737 UpdateShuttleLogbook(fCurrentDetector.Data(), "FAILED");
e7f62f16 738 UpdateShuttleStatus(AliShuttleStatus::kFailed);
3301427a 739
740 // there may still be objects in local OCDB and reference storage
741 // and FXS databases may be not updated: do it now!
742 CleanLocalStorage(fgkLocalCDB);
743 CleanLocalStorage(fgkLocalRefStorage);
744 UpdateTableFailCase();
745
57c1a579 746 } else {
747 Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. "
748 "Aborted before with %s. Retry number %d.", fCurrentDetector.Data(),
749 status->GetStatusName(), status->GetCount()));
750 UpdateShuttleStatus(AliShuttleStatus::kStarted, kTRUE);
751 cont = kTRUE;
2bb7b766 752 }
753
57c1a579 754 // Send mail to detector expert!
755 AliInfo(Form("Sending mail to %s expert...", fCurrentDetector.Data()));
756 if (!SendMail())
757 Log("SHUTTLE", Form("ContinueProcessing - Could not send mail to %s expert",
758 fCurrentDetector.Data()));
2bb7b766 759
57c1a579 760 return cont;
5164a766 761}
762
763//______________________________________________________________________________________________
2bb7b766 764Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
58bc3020 765{
73abe331 766 //
b948db8d 767 // Makes data retrieval for all detectors in the configuration.
2bb7b766 768 // entry: Shuttle logbook entry, contains run paramenters and status of detectors
769 // (Unprocessed, Inactive, Failed or Done).
d477ad88 770 // Returns kFALSE in case of error occured and kTRUE otherwise
73abe331 771 //
772
2bb7b766 773 if(!entry) return kFALSE;
774
775 fLogbookEntry = entry;
776
e7f62f16 777 if (fLogbookEntry->IsDone())
778 {
2bb7b766 779 Log("SHUTTLE","Process - Shuttle is already DONE. Updating logbook");
780 UpdateShuttleLogbook("shuttle_done");
781 fLogbookEntry = 0;
782 return kTRUE;
783 }
784
e7f62f16 785 // create ML instance that monitors this run
786 fMonaLisa = new TMonaLisaWriter(Form("%d", GetCurrentRun()), "SHUTTLE", "aliendb1.cern.ch");
787 // disable monitoring of other parameters that come e.g. from TFile
788 gMonitoringWriter = 0;
2bb7b766 789
790 AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^* \n",
791 GetCurrentRun()));
792
441b0e9c 793 // Set run type from run type logbook into current fLogbookEntry
794 SetRunType();
e7f62f16 795
796 // Send the information to ML
797 TMonaLisaText mlStatus("SHUTTLE_status", "Processing");
798
799 TList mlList;
800 mlList.Add(&mlStatus);
801
802 fMonaLisa->SendParameters(&mlList);
3301427a 803
eba76848 804 fLogbookEntry->Print("all");
57f50b3c 805
806 // Initialization
d477ad88 807 Bool_t hasError = kFALSE;
5164a766 808
2bb7b766 809 AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
810 if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
811 AliCDBStorage *mainRefSto = AliCDBManager::Instance()->GetStorage(fgkMainRefStorage);
812 if(mainRefSto) mainRefSto->QueryCDB(GetCurrentRun());
d477ad88 813
57f50b3c 814 // Loop on detectors in the configuration
b948db8d 815 TIter iter(fConfig->GetDetectors());
2bb7b766 816 TObjString* aDetector = 0;
b948db8d 817
be48e3ea 818 while ((aDetector = (TObjString*) iter.Next()))
819 {
7bfb2090 820 fCurrentDetector = aDetector->String();
5164a766 821
9e080f92 822 if (ContinueProcessing() == kFALSE) continue;
823
2bb7b766 824 AliInfo(Form("\n\n \t\t\t****** run %d - %s: START ******",
825 GetCurrentRun(), aDetector->GetName()));
826
9d733021 827 for(Int_t iSys=0;iSys<3;iSys++) fFXSCalled[iSys]=kFALSE;
828
e7f62f16 829 Log(fCurrentDetector.Data(), "Starting processing");
85a80aa9 830
be48e3ea 831 Int_t pid = fork();
832
833 if (pid < 0)
834 {
835 Log("SHUTTLE", "ERROR: Forking failed");
836 }
837 else if (pid > 0)
838 {
839 // parent
840 AliInfo(Form("In parent process of %d - %s: Starting monitoring",
841 GetCurrentRun(), aDetector->GetName()));
842
843 Long_t begin = time(0);
844
845 int status; // to be used with waitpid, on purpose an int (not Int_t)!
846 while (waitpid(pid, &status, WNOHANG) == 0)
847 {
848 Long_t expiredTime = time(0) - begin;
849
850 if (expiredTime > fConfig->GetPPTimeOut())
851 {
3301427a 852 Log("SHUTTLE", Form("%s: Process time out. Run time: %d seconds. Killing...",
853 fCurrentDetector.Data(), expiredTime));
be48e3ea 854
855 kill(pid, 9);
856
3301427a 857 UpdateShuttleStatus(AliShuttleStatus::kPPTimeOut);
be48e3ea 858 hasError = kTRUE;
859
860 gSystem->Sleep(1000);
861 }
862 else
863 {
864 if (expiredTime % 60 == 0)
865 Log("SHUTTLE", Form("Checked process. Run time: %d seconds.",
866 expiredTime));
867 gSystem->Sleep(1000);
868 }
869 }
870
871 AliInfo(Form("In parent process of %d - %s: Client has terminated.",
872 GetCurrentRun(), aDetector->GetName()));
873
874 if (WIFEXITED(status))
875 {
876 Int_t returnCode = WEXITSTATUS(status);
877
3301427a 878 Log("SHUTTLE", Form("%s: the return code is %d", fCurrentDetector.Data(),
879 returnCode));
be48e3ea 880
3301427a 881 if (returnCode != 0) hasError = kTRUE;
be48e3ea 882 }
883 }
884 else if (pid == 0)
885 {
886 // client
887 AliInfo(Form("In client process of %d - %s", GetCurrentRun(), aDetector->GetName()));
888
3301427a 889 UInt_t returnCode = ProcessCurrentDetector(); // 0 means success
be48e3ea 890
be48e3ea 891
3301427a 892 if (returnCode > 0) // TODO: returnCode>0 means Preprocessor ERROR! Preprocessors should follow this!
be48e3ea 893 {
3301427a 894 if (returnCode == kDCSErrCode)
895 {
896 AliInfo(Form("\n \t\t\t****** run %d - %s: DCS ERROR ****** \n\n",
be48e3ea 897 GetCurrentRun(), aDetector->GetName()));
3301427a 898 } else {
899
900 AliInfo(Form("\n \t\t\t****** run %d - %s: PREPROCESSOR ERROR ****** \n\n",
be48e3ea 901 GetCurrentRun(), aDetector->GetName()));
3301427a 902 }
903 } else { // Preprocessor finished successfully!
be48e3ea 904
3301427a 905 // Update time_processed field in FXS DB
906 if (UpdateTable() == kFALSE)
907 Log("SHUTTLE", Form("Process - %s: Could not update FXS databases!"));
908
909 // Transfer the data from local storage to main storage (Grid)
910 UpdateShuttleStatus(AliShuttleStatus::kStoreStarted);
911 if (StoreOCDB() == kFALSE)
912 {
913 AliInfo(Form("\n \t\t\t****** run %d - %s: STORAGE ERROR ****** \n\n",
914 GetCurrentRun(), aDetector->GetName()));
915 UpdateShuttleStatus(AliShuttleStatus::kStoreError);
916 returnCode = kStorErrCode;
917 } else {
918 AliInfo(Form("\n \t\t\t****** run %d - %s: DONE ****** \n\n",
919 GetCurrentRun(), aDetector->GetName()));
920 UpdateShuttleStatus(AliShuttleStatus::kDone);
921 }
be48e3ea 922 }
923
4b95672b 924 for (UInt_t iSys=0; iSys<3; iSys++)
925 {
926 if (fFXSCalled[iSys]) fFXSlist[iSys].Clear();
927 }
928
be48e3ea 929 AliInfo(Form("Client process of %d - %s is exiting now with %d.",
930 GetCurrentRun(), aDetector->GetName(), returnCode));
931
932 // the client exits here
933 gSystem->Exit(returnCode);
934
935 AliError("We should never get here!!!");
936 }
7bfb2090 937 }
5164a766 938
2bb7b766 939 AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^* \n",
940 GetCurrentRun()));
941
942 //check if shuttle is done for this run, if so update logbook
943 TObjArray checkEntryArray;
944 checkEntryArray.SetOwner(1);
9e080f92 945 TString whereClause = Form("where run=%d", GetCurrentRun());
946 if (!QueryShuttleLogbook(whereClause.Data(), checkEntryArray) || checkEntryArray.GetEntries() == 0) {
947 Log("SHUTTLE", Form("Process - Warning: Cannot check status of run %d on Shuttle logbook!",
948 GetCurrentRun()));
949 return hasError == kFALSE;
950 }
b948db8d 951
9e080f92 952 AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
953 (checkEntryArray.At(0));
2bb7b766 954
9e080f92 955 if (checkEntry)
956 {
957 if (checkEntry->IsDone())
be48e3ea 958 {
9e080f92 959 Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
960 UpdateShuttleLogbook("shuttle_done");
961 }
962 else
963 {
964 for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
be48e3ea 965 {
9e080f92 966 if (checkEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
be48e3ea 967 {
9e080f92 968 AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
969 checkEntry->GetRun(), GetDetName(iDet)));
970 fFirstUnprocessed[iDet] = kFALSE;
be48e3ea 971 }
972 }
2bb7b766 973 }
974 }
975
e7f62f16 976 // remove ML instance
977 delete fMonaLisa;
978 fMonaLisa = 0;
979
2bb7b766 980 fLogbookEntry = 0;
85a80aa9 981
a7160fe9 982 return hasError == kFALSE;
73abe331 983}
984
b948db8d 985//______________________________________________________________________________________________
2bb7b766 986UInt_t AliShuttle::ProcessCurrentDetector()
73abe331 987{
988 //
2bb7b766 989 // Makes data retrieval just for a specific detector (fCurrentDetector).
73abe331 990 // Threre should be a configuration for this detector.
73abe331 991
2bb7b766 992 AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), GetCurrentRun()));
73abe331 993
2c15234c 994 TMap dcsMap;
995 dcsMap.SetOwner(1);
73abe331 996
85a80aa9 997 Bool_t aDCSError = kFALSE;
3301427a 998
999 // call preprocessor
1000 AliPreprocessor* aPreprocessor =
1001 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
1002
1003 aPreprocessor->Initialize(GetCurrentRun(), GetCurrentStartTime(), GetCurrentEndTime());
1004
1005 Bool_t processDCS = aPreprocessor->ProcessDCS();
d477ad88 1006
2c15234c 1007 // TODO Test only... I've added a flag that allows to
1008 // exclude DCS archive DB query
3301427a 1009 if (!processDCS || !fgkProcessDCS)
2c15234c 1010 {
1011 AliInfo("Skipping DCS processing!");
1012 aDCSError = kFALSE;
1013 } else {
3301427a 1014
1015 UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
1016
2c15234c 1017 TString host(fConfig->GetDCSHost(fCurrentDetector));
1018 Int_t port = fConfig->GetDCSPort(fCurrentDetector);
1019
1020 // Retrieval of Aliases
1021 TObjString* anAlias = 0;
36c99a6a 1022 Int_t iAlias = 1;
1023 Int_t nTotAliases= ((TMap*)fConfig->GetDCSAliases(fCurrentDetector))->GetEntries();
2c15234c 1024 TIter iterAliases(fConfig->GetDCSAliases(fCurrentDetector));
1025 while ((anAlias = (TObjString*) iterAliases.Next()))
1026 {
1027 TObjArray *valueSet = new TObjArray();
1028 valueSet->SetOwner(1);
1029
36c99a6a 1030 if (((iAlias-1) % 500) == 0 || iAlias == nTotAliases)
1031 AliInfo(Form("Querying DCS archive: alias %s (%d of %d)",
1032 anAlias->GetName(), iAlias++, nTotAliases));
2c15234c 1033 aDCSError = (GetValueSet(host, port, anAlias->String(), valueSet, kAlias) == 0);
1034
1035 if(!aDCSError)
1036 {
1037 dcsMap.Add(anAlias->Clone(), valueSet);
1038 } else {
1039 Log(fCurrentDetector,
1040 Form("ProcessCurrentDetector - Error while retrieving alias %s",
1041 anAlias->GetName()));
1042 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
1043 dcsMap.DeleteAll();
3301427a 1044 return kDCSErrCode;
2c15234c 1045 }
4f0ab988 1046 }
2c15234c 1047
1048 // Retrieval of Data Points
1049 TObjString* aDP = 0;
36c99a6a 1050 Int_t iDP = 0;
1051 Int_t nTotDPs= ((TMap*)fConfig->GetDCSDataPoints(fCurrentDetector))->GetEntries();
2c15234c 1052 TIter iterDP(fConfig->GetDCSDataPoints(fCurrentDetector));
1053 while ((aDP = (TObjString*) iterDP.Next()))
1054 {
1055 TObjArray *valueSet = new TObjArray();
1056 valueSet->SetOwner(1);
36c99a6a 1057 if (((iDP-1) % 500) == 0 || iDP == nTotDPs)
1058 AliInfo(Form("Querying DCS archive: DP %s (%d of %d)",
1059 aDP->GetName(), iDP++, nTotDPs));
2c15234c 1060 aDCSError = (GetValueSet(host, port, aDP->String(), valueSet, kDP) == 0);
1061
1062 if(!aDCSError)
1063 {
1064 dcsMap.Add(aDP->Clone(), valueSet);
1065 } else {
1066 Log(fCurrentDetector,
1067 Form("ProcessCurrentDetector - Error while retrieving data point %s",
1068 aDP->GetName()));
1069 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
1070 dcsMap.DeleteAll();
3301427a 1071 return kDCSErrCode;
2c15234c 1072 }
73abe331 1073 }
1074 }
b948db8d 1075
2bb7b766 1076 // DCS Archive DB processing successful. Call Preprocessor!
85a80aa9 1077 UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
a7160fe9 1078
3301427a 1079 UInt_t returnValue = aPreprocessor->Process(&dcsMap);
b948db8d 1080
3301427a 1081 if (returnValue > 0) // Preprocessor error!
1082 {
cb343cfd 1083 UpdateShuttleStatus(AliShuttleStatus::kPPError);
3301427a 1084
1085 } else { // preprocessor ok!
1086 UpdateShuttleStatus(AliShuttleStatus::kPPDone);
2bb7b766 1087 UpdateShuttleLogbook(fCurrentDetector, "DONE");
3301427a 1088 Log("SHUTTLE", Form("ProcessCurrentDetector - %s preprocessor returned success",
1089 fCurrentDetector.Data()));
1090 }
b948db8d 1091
2c15234c 1092 dcsMap.DeleteAll();
b948db8d 1093
2bb7b766 1094 return returnValue;
1095}
1096
1097//______________________________________________________________________________________________
1098Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
1099 TObjArray& entries)
1100{
1101// Query DAQ's Shuttle logbook and fills detector status object.
1102// Call QueryRunParameters to query DAQ logbook for run parameters.
1103
fc5a4708 1104 entries.SetOwner(1);
1105
2bb7b766 1106 // check connection, in case connect
be48e3ea 1107 if(!Connect(3)) return kFALSE;
2bb7b766 1108
1109 TString sqlQuery;
441b0e9c 1110 sqlQuery = Form("select * from %s %s order by run", fConfig->GetShuttlelbTable(), whereClause);
2bb7b766 1111
be48e3ea 1112 TSQLResult* aResult = fServer[3]->Query(sqlQuery);
2bb7b766 1113 if (!aResult) {
1114 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
1115 return kFALSE;
1116 }
1117
fc5a4708 1118 AliDebug(2,Form("Query = %s", sqlQuery.Data()));
1119
2bb7b766 1120 if(aResult->GetRowCount() == 0) {
9e080f92 1121// if(sqlQuery.EndsWith("where shuttle_done=0 order by run")){
1122// Log("SHUTTLE", "QueryShuttleLogbook - All runs in Shuttle Logbook are already DONE");
1123// delete aResult;
1124// return kTRUE;
1125// } else {
1126 AliInfo("No entries in Shuttle Logbook match request");
2bb7b766 1127 delete aResult;
1128 return kTRUE;
9e080f92 1129// }
2bb7b766 1130 }
1131
1132 // TODO Check field count!
fc5a4708 1133 const UInt_t nCols = 22;
2bb7b766 1134 if (aResult->GetFieldCount() != (Int_t) nCols) {
1135 AliError("Invalid SQL result field number!");
1136 delete aResult;
1137 return kFALSE;
1138 }
1139
2bb7b766 1140 TSQLRow* aRow;
1141 while ((aRow = aResult->Next())) {
1142 TString runString(aRow->GetField(0), aRow->GetFieldLength(0));
1143 Int_t run = runString.Atoi();
1144
eba76848 1145 AliShuttleLogbookEntry *entry = QueryRunParameters(run);
1146 if (!entry)
1147 continue;
2bb7b766 1148
1149 // loop on detectors
eba76848 1150 for(UInt_t ii = 0; ii < nCols; ii++)
1151 entry->SetDetectorStatus(aResult->GetFieldName(ii), aRow->GetField(ii));
2bb7b766 1152
eba76848 1153 entries.AddLast(entry);
2bb7b766 1154 delete aRow;
1155 }
1156
9e080f92 1157// if(sqlQuery.EndsWith("where shuttle_done=0 order by run"))
1158// Log("SHUTTLE", Form("QueryShuttleLogbook - Found %d unprocessed runs in Shuttle Logbook",
1159// entries.GetEntriesFast()));
2bb7b766 1160 delete aResult;
1161 return kTRUE;
1162}
1163
1164//______________________________________________________________________________________________
eba76848 1165AliShuttleLogbookEntry* AliShuttle::QueryRunParameters(Int_t run)
2bb7b766 1166{
eba76848 1167 //
1168 // Retrieve run parameters written in the DAQ logbook and sets them into AliShuttleLogbookEntry object
1169 //
2bb7b766 1170
1171 // check connection, in case connect
be48e3ea 1172 if (!Connect(3))
eba76848 1173 return 0;
2bb7b766 1174
1175 TString sqlQuery;
2c15234c 1176 sqlQuery.Form("select * from %s where run=%d", fConfig->GetDAQlbTable(), run);
2bb7b766 1177
be48e3ea 1178 TSQLResult* aResult = fServer[3]->Query(sqlQuery);
2bb7b766 1179 if (!aResult) {
1180 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
eba76848 1181 return 0;
2bb7b766 1182 }
1183
eba76848 1184 if (aResult->GetRowCount() == 0) {
2bb7b766 1185 Log("SHUTTLE", Form("QueryRunParameters - No entry in DAQ Logbook for run %d. Skipping", run));
1186 delete aResult;
eba76848 1187 return 0;
2bb7b766 1188 }
1189
eba76848 1190 if (aResult->GetRowCount() > 1) {
2bb7b766 1191 AliError(Form("More than one entry in DAQ Logbook for run %d. Skipping", run));
1192 delete aResult;
eba76848 1193 return 0;
2bb7b766 1194 }
1195
eba76848 1196 TSQLRow* aRow = aResult->Next();
1197 if (!aRow)
1198 {
1199 AliError(Form("Could not retrieve row for run %d. Skipping", run));
1200 delete aResult;
1201 return 0;
1202 }
2bb7b766 1203
eba76848 1204 AliShuttleLogbookEntry* entry = new AliShuttleLogbookEntry(run);
2bb7b766 1205
eba76848 1206 for (Int_t ii = 0; ii < aResult->GetFieldCount(); ii++)
1207 entry->SetRunParameter(aResult->GetFieldName(ii), aRow->GetField(ii));
2bb7b766 1208
eba76848 1209 UInt_t startTime = entry->GetStartTime();
1210 UInt_t endTime = entry->GetEndTime();
1211
1212 if (!startTime || !endTime || startTime > endTime) {
1213 Log("SHUTTLE",
1214 Form("QueryRunParameters - Invalid parameters for Run %d: startTime = %d, endTime = %d",
1215 run, startTime, endTime));
1216 delete entry;
2bb7b766 1217 delete aRow;
eba76848 1218 delete aResult;
1219 return 0;
2bb7b766 1220 }
1221
eba76848 1222 delete aRow;
2bb7b766 1223 delete aResult;
eba76848 1224
1225 return entry;
2bb7b766 1226}
1227
1228//______________________________________________________________________________________________
2c15234c 1229Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* entry,
1230 TObjArray* valueSet, DCSType type)
73abe331 1231{
2c15234c 1232// Retrieve all "entry" data points from the DCS server
58bc3020 1233// host, port: TSocket connection parameters
2c15234c 1234// entry: name of the alias or data point
2bb7b766 1235// valueSet: array of retrieved AliDCSValue's
2c15234c 1236// type: kAlias or kDP
58bc3020 1237
73abe331 1238 AliDCSClient client(host, port, fTimeout, fRetries);
2c15234c 1239 if (!client.IsConnected())
1240 {
b948db8d 1241 return kFALSE;
73abe331 1242 }
1243
2c15234c 1244 Int_t result=0;
73abe331 1245
2c15234c 1246 if (type == kAlias)
1247 {
1248 result = client.GetAliasValues(entry,
1249 GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
1250 } else
1251 if (type == kDP)
1252 {
1253 result = client.GetDPValues(entry,
1254 GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
1255 }
1256
1257 if (result < 0)
1258 {
2bb7b766 1259 Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get '%s'! Reason: %s",
2c15234c 1260 entry, AliDCSClient::GetErrorString(result)));
73abe331 1261
2c15234c 1262 if (result == AliDCSClient::fgkServerError)
1263 {
2bb7b766 1264 Log(fCurrentDetector.Data(), Form("GetValueSet - Server error: %s",
73abe331 1265 client.GetServerError().Data()));
1266 }
1267
1268 return kFALSE;
1269 }
1270
1271 return kTRUE;
1272}
b948db8d 1273
1274//______________________________________________________________________________________________
57f50b3c 1275const char* AliShuttle::GetFile(Int_t system, const char* detector,
1276 const char* id, const char* source)
b948db8d 1277{
57f50b3c 1278// Get calibration file from file exchange servers
9d733021 1279// First queris the FXS database for the file name, using the run, detector, id and source info
1280// then calls RetrieveFile(filename) for actual copy to local disk
2bb7b766 1281// run: current run being processed (given by Logbook entry fLogbookEntry)
eba76848 1282// detector: the Preprocessor name
57f50b3c 1283// id: provided as a parameter by the Preprocessor
1284// source: provided by the Preprocessor through GetFileSources function
1285
1286 // check connection, in case connect
9d733021 1287 if (!Connect(system))
eba76848 1288 {
9d733021 1289 Log(detector, Form("GetFile - Couldn't connect to %s FXS database", GetSystemName(system)));
57f50b3c 1290 return 0;
1291 }
1292
1293 // Query preparation
9d733021 1294 TString sourceName(source);
d386d623 1295 Int_t nFields = 3;
1296 TString sqlQueryStart = Form("select filePath,size,fileChecksum from %s where",
1297 fConfig->GetFXSdbTable(system));
1298 TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
1299 GetCurrentRun(), detector, id);
1300
9d733021 1301 if (system == kDAQ)
1302 {
d386d623 1303 whereClause += Form(" and DAQsource=\"%s\"", source);
57f50b3c 1304 }
9d733021 1305 else if (system == kDCS)
eba76848 1306 {
9d733021 1307 sourceName="none";
57f50b3c 1308 }
9d733021 1309 else if (system == kHLT)
9e080f92 1310 {
d386d623 1311 whereClause += Form(" and DDLnumbers=\"%s\"", source);
9d733021 1312 nFields = 3;
9e080f92 1313 }
1314
9e080f92 1315 TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1316
1317 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1318
1319 // Query execution
1320 TSQLResult* aResult = 0;
9d733021 1321 aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
9e080f92 1322 if (!aResult) {
9d733021 1323 Log(detector, Form("GetFileName - Can't execute SQL query to %s database for: id = %s, source = %s",
1324 GetSystemName(system), id, sourceName.Data()));
9e080f92 1325 return 0;
1326 }
1327
1328 if(aResult->GetRowCount() == 0)
1329 {
1330 Log(detector,
9d733021 1331 Form("GetFileName - No entry in %s FXS db for: id = %s, source = %s",
1332 GetSystemName(system), id, sourceName.Data()));
9e080f92 1333 delete aResult;
1334 return 0;
1335 }
2bb7b766 1336
9e080f92 1337 if (aResult->GetRowCount() > 1) {
1338 Log(detector,
9d733021 1339 Form("GetFileName - More than one entry in %s FXS db for: id = %s, source = %s",
1340 GetSystemName(system), id, sourceName.Data()));
9e080f92 1341 delete aResult;
1342 return 0;
1343 }
1344
9d733021 1345 if (aResult->GetFieldCount() != nFields) {
9e080f92 1346 Log(detector,
9d733021 1347 Form("GetFileName - Wrong field count in %s FXS db for: id = %s, source = %s",
1348 GetSystemName(system), id, sourceName.Data()));
9e080f92 1349 delete aResult;
1350 return 0;
1351 }
1352
1353 TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
1354
1355 if (!aRow){
9d733021 1356 Log(detector, Form("GetFileName - Empty set result in %s FXS db from query: id = %s, source = %s",
1357 GetSystemName(system), id, sourceName.Data()));
9e080f92 1358 delete aResult;
1359 return 0;
1360 }
1361
1362 TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
1363 TString fileSize(aRow->GetField(1), aRow->GetFieldLength(1));
d386d623 1364 TString fileChecksum(aRow->GetField(2), aRow->GetFieldLength(2));
9e080f92 1365
1366 delete aResult;
1367 delete aRow;
1368
d386d623 1369 AliDebug(2, Form("filePath = %s; size = %s, fileChecksum = %s",
1370 filePath.Data(), fileSize.Data(), fileChecksum.Data()));
9e080f92 1371
9e080f92 1372 // retrieved file is renamed to make it unique
9d733021 1373 TString localFileName = Form("%s_%s_%d_%s_%s.shuttle",
1374 GetSystemName(system), detector, GetCurrentRun(), id, sourceName.Data());
1375
9e080f92 1376
9d733021 1377 // file retrieval from FXS
4b95672b 1378 UInt_t nRetries = 0;
1379 UInt_t maxRetries = 3;
1380 Bool_t result = kFALSE;
1381
1382 // copy!! if successful TSystem::Exec returns 0
1383 while(nRetries++ < maxRetries) {
1384 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
1385 result = RetrieveFile(system, filePath.Data(), localFileName.Data());
1386 if(!result)
1387 {
1388 Log(detector, Form("GetFileName - Copy of file %s from %s FXS failed",
9d733021 1389 filePath.Data(), GetSystemName(system)));
4b95672b 1390 continue;
1391 } else {
1392 AliInfo(Form("File %s copied from %s FXS into %s/%s",
1393 filePath.Data(), GetSystemName(system),
1394 GetShuttleTempDir(), localFileName.Data()));
1395 }
9e080f92 1396
d386d623 1397 if (fileChecksum.Length()>0)
4b95672b 1398 {
1399 // compare md5sum of local file with the one stored in the FXS DB
1400 Int_t md5Comp = gSystem->Exec(Form("md5sum %s/%s |grep %s 2>&1 > /dev/null",
d386d623 1401 GetShuttleTempDir(), localFileName.Data(), fileChecksum.Data()));
9e080f92 1402
4b95672b 1403 if (md5Comp != 0)
1404 {
1405 Log(detector, Form("GetFileName - md5sum of file %s does not match with local copy!",
1406 filePath.Data()));
1407 result = kFALSE;
1408 continue;
1409 }
d386d623 1410 } else {
1411 Log(fCurrentDetector, Form("GetFile - md5sum of file %s not set in %s database, skipping comparison",
1412 filePath.Data(), GetSystemName(system)));
9d733021 1413 }
4b95672b 1414 if (result) break;
9e080f92 1415 }
1416
4b95672b 1417 if(!result) return 0;
1418
9d733021 1419 fFXSCalled[system]=kTRUE;
1420 TObjString *fileParams = new TObjString(Form("%s#!?!#%s", id, sourceName.Data()));
1421 fFXSlist[system].Add(fileParams);
9e080f92 1422
1423 static TString fullLocalFileName;
36c99a6a 1424 fullLocalFileName = TString::Format("%s/%s", GetShuttleTempDir(), localFileName.Data());
1425
9e080f92 1426 AliInfo(Form("fullLocalFileName = %s", fullLocalFileName.Data()));
1427
1428 return fullLocalFileName.Data();
2bb7b766 1429
1430}
1431
1432//______________________________________________________________________________________________
9d733021 1433Bool_t AliShuttle::RetrieveFile(UInt_t system, const char* fxsFileName, const char* localFileName)
9e080f92 1434{
9d733021 1435// Copies file from FXS to local Shuttle machine
2bb7b766 1436
9e080f92 1437 // check temp directory: trying to cd to temp; if it does not exist, create it
9d733021 1438 AliDebug(2, Form("Copy file %s from %s FXS into %s/%s",
1439 GetSystemName(system), fxsFileName, GetShuttleTempDir(), localFileName));
9e080f92 1440
36c99a6a 1441 void* dir = gSystem->OpenDirectory(GetShuttleTempDir());
9e080f92 1442 if (dir == NULL) {
36c99a6a 1443 if (gSystem->mkdir(GetShuttleTempDir(), kTRUE)) {
1444 AliError(Form("Can't open directory <%s>", GetShuttleTempDir()));
9e080f92 1445 return kFALSE;
1446 }
1447
1448 } else {
1449 gSystem->FreeDirectory(dir);
1450 }
1451
9d733021 1452 TString baseFXSFolder;
1453 if (system == kDAQ)
1454 {
1455 baseFXSFolder = "FES/";
1456 }
1457 else if (system == kDCS)
1458 {
1459 baseFXSFolder = "";
1460 }
1461 else if (system == kHLT)
1462 {
1463 baseFXSFolder = "~/";
1464 }
1465
1466
1467 TString command = Form("scp -oPort=%d -2 %s@%s:%s%s %s/%s",
1468 fConfig->GetFXSPort(system),
1469 fConfig->GetFXSUser(system),
1470 fConfig->GetFXSHost(system),
1471 baseFXSFolder.Data(),
1472 fxsFileName,
36c99a6a 1473 GetShuttleTempDir(),
9e080f92 1474 localFileName);
1475
1476 AliDebug(2, Form("%s",command.Data()));
1477
4b95672b 1478 Bool_t result = (gSystem->Exec(command.Data()) == 0);
9e080f92 1479
4b95672b 1480 return result;
9e080f92 1481}
1482
1483//______________________________________________________________________________________________
9d733021 1484TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
1485{
1486// Get sources producing the condition file Id from file exchange servers
1487
1488 if (system == kDCS)
1489 {
1490 AliError("DCS system has only one source of data!");
1491 return NULL;
1492
1493 }
9e080f92 1494
1495 // check connection, in case connect
9d733021 1496 if (!Connect(system))
1497 {
1498 Log(detector, Form("GetFile - Couldn't connect to %s FXS database", GetSystemName(system)));
1499 return NULL;
9e080f92 1500 }
1501
9d733021 1502 TString sourceName = 0;
1503 if (system == kDAQ)
1504 {
1505 sourceName = "DAQsource";
1506 } else if (system == kHLT)
1507 {
1508 sourceName = "DDLnumbers";
1509 }
1510
d386d623 1511 TString sqlQueryStart = Form("select %s from %s where", sourceName.Data(), fConfig->GetFXSdbTable(system));
9e080f92 1512 TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
1513 GetCurrentRun(), detector, id);
1514 TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1515
1516 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1517
1518 // Query execution
1519 TSQLResult* aResult;
9d733021 1520 aResult = fServer[system]->Query(sqlQuery);
9e080f92 1521 if (!aResult) {
9d733021 1522 Log(detector, Form("GetFileSources - Can't execute SQL query to %s database for id: %s",
1523 GetSystemName(system), id));
9e080f92 1524 return 0;
1525 }
1526
9d733021 1527 if (aResult->GetRowCount() == 0)
1528 {
9e080f92 1529 Log(detector,
9d733021 1530 Form("GetFileSources - No entry in %s FXS table for id: %s", GetSystemName(system), id));
9e080f92 1531 delete aResult;
1532 return 0;
1533 }
1534
1535 TSQLRow* aRow;
1536 TList *list = new TList();
1537 list->SetOwner(1);
1538
9d733021 1539 while ((aRow = aResult->Next()))
1540 {
9e080f92 1541
9d733021 1542 TString source(aRow->GetField(0), aRow->GetFieldLength(0));
1543 AliDebug(2, Form("%s = %s", sourceName.Data(), source.Data()));
1544 list->Add(new TObjString(source));
9e080f92 1545 delete aRow;
1546 }
9d733021 1547
9e080f92 1548 delete aResult;
1549
1550 return list;
2bb7b766 1551}
1552
1553//______________________________________________________________________________________________
9d733021 1554Bool_t AliShuttle::Connect(Int_t system)
2bb7b766 1555{
9d733021 1556// Connect to MySQL Server of the system's FXS MySQL databases
1557// DAQ Logbook, Shuttle Logbook and DAQ FXS db are on the same host
57f50b3c 1558
9d733021 1559 // check connection: if already connected return
1560 if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
57f50b3c 1561
9d733021 1562 TString dbHost, dbUser, dbPass, dbName;
57f50b3c 1563
9d733021 1564 if (system < 3) // FXS db servers
1565 {
1566 dbHost = Form("mysql://%s:%d", fConfig->GetFXSdbHost(system), fConfig->GetFXSdbPort(system));
1567 dbUser = fConfig->GetFXSdbUser(system);
1568 dbPass = fConfig->GetFXSdbPass(system);
1569 dbName = fConfig->GetFXSdbName(system);
1570 } else { // Run & Shuttle logbook servers
1571 // TODO Will the Shuttle logbook server be the same as the Run logbook server ???
1572 dbHost = Form("mysql://%s:%d", fConfig->GetDAQlbHost(), fConfig->GetDAQlbPort());
1573 dbUser = fConfig->GetDAQlbUser();
1574 dbPass = fConfig->GetDAQlbPass();
1575 dbName = fConfig->GetDAQlbDB();
1576 }
57f50b3c 1577
9d733021 1578 fServer[system] = TSQLServer::Connect(dbHost.Data(), dbUser.Data(), dbPass.Data());
1579 if (!fServer[system] || !fServer[system]->IsConnected()) {
1580 if(system < 3)
1581 {
1582 AliError(Form("Can't establish connection to FXS database for %s",
1583 AliShuttleInterface::GetSystemName(system)));
1584 } else {
1585 AliError("Can't establish connection to Run logbook.");
57f50b3c 1586 }
9d733021 1587 if(fServer[system]) delete fServer[system];
1588 return kFALSE;
2bb7b766 1589 }
57f50b3c 1590
9d733021 1591 // Get tables
1592 TSQLResult* aResult=0;
1593 switch(system){
1594 case kDAQ:
1595 aResult = fServer[kDAQ]->GetTables(dbName.Data());
1596 break;
1597 case kDCS:
1598 aResult = fServer[kDCS]->GetTables(dbName.Data());
1599 break;
1600 case kHLT:
1601 aResult = fServer[kHLT]->GetTables(dbName.Data());
1602 break;
1603 default:
1604 aResult = fServer[3]->GetTables(dbName.Data());
1605 break;
1606 }
1607
1608 delete aResult;
2bb7b766 1609 return kTRUE;
1610}
57f50b3c 1611
9e080f92 1612//______________________________________________________________________________________________
9d733021 1613Bool_t AliShuttle::UpdateTable()
9e080f92 1614{
9d733021 1615// Update FXS table filling time_processed field in all rows corresponding to current run and detector
9e080f92 1616
9d733021 1617 Bool_t result = kTRUE;
9e080f92 1618
9d733021 1619 for (UInt_t system=0; system<3; system++)
1620 {
1621 if(!fFXSCalled[system]) continue;
9e080f92 1622
9d733021 1623 // check connection, in case connect
1624 if (!Connect(system))
1625 {
1626 Log(fCurrentDetector, Form("UpdateTable - Couldn't connect to %s FXS database", GetSystemName(system)));
1627 result = kFALSE;
1628 continue;
9e080f92 1629 }
9e080f92 1630
9d733021 1631 TTimeStamp now; // now
1632
1633 // Loop on FXS list entries
1634 TIter iter(&fFXSlist[system]);
1635 TObjString *aFXSentry=0;
1636 while ((aFXSentry = dynamic_cast<TObjString*> (iter.Next())))
1637 {
1638 TString aFXSentrystr = aFXSentry->String();
1639 TObjArray *aFXSarray = aFXSentrystr.Tokenize("#!?!#");
1640 if (!aFXSarray || aFXSarray->GetEntries() != 2 )
1641 {
1642 Log(fCurrentDetector, Form("UpdateTable - error updating %s FXS entry. Check string: <%s>",
1643 GetSystemName(system), aFXSentrystr.Data()));
1644 if(aFXSarray) delete aFXSarray;
1645 result = kFALSE;
1646 continue;
1647 }
1648 const char* fileId = ((TObjString*) aFXSarray->At(0))->GetName();
1649 const char* source = ((TObjString*) aFXSarray->At(1))->GetName();
1650
1651 TString whereClause;
1652 if (system == kDAQ)
1653 {
1654 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
1655 GetCurrentRun(), fCurrentDetector.Data(), fileId, source);
1656 }
1657 else if (system == kDCS)
1658 {
1659 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\";",
1660 GetCurrentRun(), fCurrentDetector.Data(), fileId);
1661 }
1662 else if (system == kHLT)
1663 {
1664 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DDLnumbers=\"%s\";",
1665 GetCurrentRun(), fCurrentDetector.Data(), fileId, source);
1666 }
1667
1668 delete aFXSarray;
9e080f92 1669
9d733021 1670 TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(system),
1671 now.GetSec(), whereClause.Data());
9e080f92 1672
9d733021 1673 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
9e080f92 1674
9d733021 1675 // Query execution
1676 TSQLResult* aResult;
1677 aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
1678 if (!aResult)
1679 {
1680 Log(fCurrentDetector, Form("UpdateTable - %s db: can't execute SQL query <%s>",
1681 GetSystemName(system), sqlQuery.Data()));
1682 result = kFALSE;
1683 continue;
1684 }
1685 delete aResult;
9e080f92 1686 }
9e080f92 1687 }
1688
9d733021 1689 return result;
9e080f92 1690}
57f50b3c 1691
2bb7b766 1692//______________________________________________________________________________________________
3301427a 1693Bool_t AliShuttle::UpdateTableFailCase()
1694{
1695// Update FXS table filling time_processed field in all rows corresponding to current run and detector
1696// this is called in case the preprocessor is declared failed for the current run, because
1697// the fields are updated only in case of success
1698
1699 Bool_t result = kTRUE;
1700
1701 for (UInt_t system=0; system<3; system++)
1702 {
1703 // check connection, in case connect
1704 if (!Connect(system))
1705 {
1706 Log(fCurrentDetector, Form("UpdateTableFailCase - Couldn't connect to %s FXS database",
1707 GetSystemName(system)));
1708 result = kFALSE;
1709 continue;
1710 }
1711
1712 TTimeStamp now; // now
1713
1714 // Loop on FXS list entries
1715
1716 TString whereClause = Form("where run=%d and detector=\"%s\";",
1717 GetCurrentRun(), fCurrentDetector.Data());
1718
1719
1720 TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(system),
1721 now.GetSec(), whereClause.Data());
1722
1723 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1724
1725 // Query execution
1726 TSQLResult* aResult;
1727 aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
1728 if (!aResult)
1729 {
1730 Log(fCurrentDetector, Form("UpdateTableFailCase - %s db: can't execute SQL query <%s>",
1731 GetSystemName(system), sqlQuery.Data()));
1732 result = kFALSE;
1733 continue;
1734 }
1735 delete aResult;
1736 }
1737
1738 return result;
1739}
1740
1741//______________________________________________________________________________________________
2bb7b766 1742Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status)
1743{
e7f62f16 1744 //
1745 // Update Shuttle logbook filling detector or shuttle_done column
1746 // ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
1747 //
57f50b3c 1748
2bb7b766 1749 // check connection, in case connect
be48e3ea 1750 if(!Connect(3)){
2bb7b766 1751 Log("SHUTTLE", "UpdateShuttleLogbook - Couldn't connect to DAQ Logbook.");
1752 return kFALSE;
57f50b3c 1753 }
1754
2bb7b766 1755 TString detName(detector);
1756 TString setClause;
e7f62f16 1757 if(detName == "shuttle_done")
1758 {
2bb7b766 1759 setClause = "set shuttle_done=1";
e7f62f16 1760
1761 // Send the information to ML
1762 TMonaLisaText mlStatus("SHUTTLE_status", "Done");
1763
1764 TList mlList;
1765 mlList.Add(&mlStatus);
1766
1767 fMonaLisa->SendParameters(&mlList);
2bb7b766 1768 } else {
2bb7b766 1769 TString statusStr(status);
1770 if(statusStr.Contains("done", TString::kIgnoreCase) ||
1771 statusStr.Contains("failed", TString::kIgnoreCase)){
eba76848 1772 setClause = Form("set %s=\"%s\"", detector, status);
2bb7b766 1773 } else {
1774 Log("SHUTTLE",
1775 Form("UpdateShuttleLogbook - Invalid status <%s> for detector %s",
1776 status, detector));
1777 return kFALSE;
1778 }
1779 }
57f50b3c 1780
2bb7b766 1781 TString whereClause = Form("where run=%d", GetCurrentRun());
1782
441b0e9c 1783 TString sqlQuery = Form("update %s %s %s",
1784 fConfig->GetShuttlelbTable(), setClause.Data(), whereClause.Data());
57f50b3c 1785
2bb7b766 1786 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1787
1788 // Query execution
1789 TSQLResult* aResult;
be48e3ea 1790 aResult = dynamic_cast<TSQLResult*> (fServer[3]->Query(sqlQuery));
2bb7b766 1791 if (!aResult) {
1792 Log("SHUTTLE", Form("UpdateShuttleLogbook - Can't execute query <%s>", sqlQuery.Data()));
1793 return kFALSE;
57f50b3c 1794 }
2bb7b766 1795 delete aResult;
57f50b3c 1796
1797 return kTRUE;
1798}
1799
1800//______________________________________________________________________________________________
2bb7b766 1801Int_t AliShuttle::GetCurrentRun() const
1802{
1803// Get current run from logbook entry
57f50b3c 1804
2bb7b766 1805 return fLogbookEntry ? fLogbookEntry->GetRun() : -1;
57f50b3c 1806}
1807
1808//______________________________________________________________________________________________
2bb7b766 1809UInt_t AliShuttle::GetCurrentStartTime() const
1810{
1811// get current start time
57f50b3c 1812
2bb7b766 1813 return fLogbookEntry ? fLogbookEntry->GetStartTime() : 0;
57f50b3c 1814}
1815
1816//______________________________________________________________________________________________
2bb7b766 1817UInt_t AliShuttle::GetCurrentEndTime() const
1818{
1819// get current end time from logbook entry
57f50b3c 1820
2bb7b766 1821 return fLogbookEntry ? fLogbookEntry->GetEndTime() : 0;
57f50b3c 1822}
1823
1824//______________________________________________________________________________________________
b948db8d 1825void AliShuttle::Log(const char* detector, const char* message)
1826{
58bc3020 1827// Fill log string with a message
b948db8d 1828
36c99a6a 1829 void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
84090f85 1830 if (dir == NULL) {
36c99a6a 1831 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE)) {
1832 AliError(Form("Can't open directory <%s>", GetShuttleLogDir()));
84090f85 1833 return;
1834 }
b948db8d 1835
84090f85 1836 } else {
1837 gSystem->FreeDirectory(dir);
1838 }
b948db8d 1839
cb343cfd 1840 TString toLog = Form("%s (%d): %s - ", TTimeStamp(time(0)).AsString("s"), getpid(), detector);
e7f62f16 1841 if (GetCurrentRun() >= 0)
1842 toLog += Form("run %d - ", GetCurrentRun());
2bb7b766 1843 toLog += Form("%s", message);
1844
84090f85 1845 AliInfo(toLog.Data());
b948db8d 1846
84090f85 1847 TString fileName;
e7f62f16 1848 if (GetCurrentRun() >= 0)
1849 fileName.Form("%s/%s_%d.log", GetShuttleLogDir(), detector, GetCurrentRun());
1850 else
1851 fileName.Form("%s/%s.log", GetShuttleLogDir(), detector);
1852
84090f85 1853 gSystem->ExpandPathName(fileName);
1854
1855 ofstream logFile;
1856 logFile.open(fileName, ofstream::out | ofstream::app);
1857
1858 if (!logFile.is_open()) {
1859 AliError(Form("Could not open file %s", fileName.Data()));
1860 return;
1861 }
7bfb2090 1862
84090f85 1863 logFile << toLog.Data() << "\n";
b948db8d 1864
84090f85 1865 logFile.close();
b948db8d 1866}
2bb7b766 1867
2bb7b766 1868//______________________________________________________________________________________________
1869Bool_t AliShuttle::Collect(Int_t run)
1870{
eba76848 1871//
1872// Collects conditions data for all UNPROCESSED run written to DAQ LogBook in case of run = -1 (default)
1873// If a dedicated run is given this run is processed
1874//
1875// In operational mode, this is the Shuttle function triggered by the EOR signal.
1876//
2bb7b766 1877
eba76848 1878 if (run == -1)
1879 Log("SHUTTLE","Collect - Shuttle called. Collecting conditions data for unprocessed runs");
1880 else
1881 Log("SHUTTLE", Form("Collect - Shuttle called. Collecting conditions data for run %d", run));
cb343cfd 1882
1883 SetLastAction("Starting");
2bb7b766 1884
1885 TString whereClause("where shuttle_done=0");
eba76848 1886 if (run != -1)
1887 whereClause += Form(" and run=%d", run);
2bb7b766 1888
1889 TObjArray shuttleLogbookEntries;
be48e3ea 1890 if (!QueryShuttleLogbook(whereClause, shuttleLogbookEntries))
1891 {
cb343cfd 1892 Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
2bb7b766 1893 return kFALSE;
1894 }
1895
9e080f92 1896 if (shuttleLogbookEntries.GetEntries() == 0)
1897 {
1898 if (run == -1)
1899 Log("SHUTTLE","Collect - Found no UNPROCESSED runs in Shuttle logbook");
1900 else
1901 Log("SHUTTLE", Form("Collect - Run %d is already DONE "
1902 "or it does not exist in Shuttle logbook", run));
1903 return kTRUE;
1904 }
1905
be48e3ea 1906 for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
1907 fFirstUnprocessed[iDet] = kTRUE;
1908
fc5a4708 1909 if (run != -1)
be48e3ea 1910 {
1911 // query Shuttle logbook for earlier runs, check if some detectors are unprocessed,
1912 // flag them into fFirstUnprocessed array
1913 TString whereClause(Form("where shuttle_done=0 and run < %d", run));
1914 TObjArray tmpLogbookEntries;
1915 if (!QueryShuttleLogbook(whereClause, tmpLogbookEntries))
1916 {
1917 Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
1918 return kFALSE;
1919 }
1920
1921 TIter iter(&tmpLogbookEntries);
1922 AliShuttleLogbookEntry* anEntry = 0;
1923 while ((anEntry = dynamic_cast<AliShuttleLogbookEntry*> (iter.Next())))
1924 {
1925 for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
1926 {
1927 if (anEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
1928 {
1929 AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
1930 anEntry->GetRun(), GetDetName(iDet)));
1931 fFirstUnprocessed[iDet] = kFALSE;
1932 }
1933 }
1934
1935 }
1936
1937 }
1938
1939 if (!RetrieveConditionsData(shuttleLogbookEntries))
1940 {
cb343cfd 1941 Log("SHUTTLE", "Collect - Process of at least one run failed");
2bb7b766 1942 return kFALSE;
1943 }
1944
36c99a6a 1945 Log("SHUTTLE", "Collect - Requested run(s) successfully processed");
eba76848 1946 return kTRUE;
2bb7b766 1947}
1948
2bb7b766 1949//______________________________________________________________________________________________
1950Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
1951{
1952// Retrieve conditions data for all runs that aren't processed yet
1953
1954 Bool_t hasError = kFALSE;
1955
1956 TIter iter(&dateEntries);
1957 AliShuttleLogbookEntry* anEntry;
1958
1959 while ((anEntry = (AliShuttleLogbookEntry*) iter.Next())){
1960 if (!Process(anEntry)){
1961 hasError = kTRUE;
1962 }
4b95672b 1963
1964 // clean SHUTTLE temp directory
3301427a 1965 TString filename = Form("%s/*.shuttle", GetShuttleTempDir());
1966 RemoveFile(filename.Data());
2bb7b766 1967 }
1968
1969 return hasError == kFALSE;
1970}
cb343cfd 1971
1972//______________________________________________________________________________________________
1973ULong_t AliShuttle::GetTimeOfLastAction() const
1974{
1975 ULong_t tmp;
36c99a6a 1976
cb343cfd 1977 fMonitoringMutex->Lock();
be48e3ea 1978
cb343cfd 1979 tmp = fLastActionTime;
36c99a6a 1980
cb343cfd 1981 fMonitoringMutex->UnLock();
36c99a6a 1982
cb343cfd 1983 return tmp;
1984}
1985
1986//______________________________________________________________________________________________
1987const TString AliShuttle::GetLastAction() const
1988{
1989 // returns a string description of the last action
1990
1991 TString tmp;
36c99a6a 1992
cb343cfd 1993 fMonitoringMutex->Lock();
1994
1995 tmp = fLastAction;
1996
1997 fMonitoringMutex->UnLock();
1998
36c99a6a 1999 return tmp;
cb343cfd 2000}
2001
2002//______________________________________________________________________________________________
2003void AliShuttle::SetLastAction(const char* action)
2004{
2005 // updates the monitoring variables
36c99a6a 2006
cb343cfd 2007 fMonitoringMutex->Lock();
36c99a6a 2008
cb343cfd 2009 fLastAction = action;
2010 fLastActionTime = time(0);
2011
2012 fMonitoringMutex->UnLock();
2013}
eba76848 2014
2015//______________________________________________________________________________________________
2016const char* AliShuttle::GetRunParameter(const char* param)
2017{
2018// returns run parameter read from DAQ logbook
2019
2020 if(!fLogbookEntry) {
2021 AliError("No logbook entry!");
2022 return 0;
2023 }
2024
2025 return fLogbookEntry->GetRunParameter(param);
2026}
57c1a579 2027
2028//______________________________________________________________________________________________
d386d623 2029AliCDBEntry* AliShuttle::GetFromOCDB(const AliCDBPath& path)
2030{
2031// returns obiect from OCDB valid for current run
2032
2033 AliCDBStorage *sto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
2034 if (!sto)
2035 {
2036 Log("SHUTTLE", "GetFromOCDB - Cannot activate main OCDB for query!");
2037 return 0;
2038 }
2039
2040 return dynamic_cast<AliCDBEntry*> (sto->Get(path, GetCurrentRun()));
2041}
2042
2043//______________________________________________________________________________________________
57c1a579 2044Bool_t AliShuttle::SendMail()
2045{
2046// sends a mail to the subdetector expert in case of preprocessor error
2047
36c99a6a 2048 void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
57c1a579 2049 if (dir == NULL)
2050 {
36c99a6a 2051 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE))
57c1a579 2052 {
36c99a6a 2053 AliError(Form("Can't open directory <%s>", GetShuttleLogDir()));
57c1a579 2054 return kFALSE;
2055 }
2056
2057 } else {
2058 gSystem->FreeDirectory(dir);
2059 }
2060
2061 TString bodyFileName;
36c99a6a 2062 bodyFileName.Form("%s/mail.body", GetShuttleLogDir());
57c1a579 2063 gSystem->ExpandPathName(bodyFileName);
2064
2065 ofstream mailBody;
2066 mailBody.open(bodyFileName, ofstream::out);
2067
2068 if (!mailBody.is_open())
2069 {
2070 AliError(Form("Could not open mail body file %s", bodyFileName.Data()));
2071 return kFALSE;
2072 }
2073
2074 TString to="";
2075 TIter iterExperts(fConfig->GetResponsibles(fCurrentDetector));
2076 TObjString *anExpert=0;
2077 while ((anExpert = (TObjString*) iterExperts.Next()))
2078 {
2079 to += Form("%s,", anExpert->GetName());
2080 }
2081 to.Remove(to.Length()-1);
909732f7 2082 AliDebug(2, Form("to: %s",to.Data()));
57c1a579 2083
36c99a6a 2084 // TODO this will be removed...
2085 if (to.Contains("not_yet_set")) {
2086 AliInfo("List of detector responsibles not yet set!");
2087 return kFALSE;
2088 }
2089
57c1a579 2090 TString cc="alberto.colla@cern.ch";
2091
2092 TString subject = Form("%s Shuttle preprocessor error in run %d !",
2093 fCurrentDetector.Data(), GetCurrentRun());
909732f7 2094 AliDebug(2, Form("subject: %s", subject.Data()));
57c1a579 2095
2096 TString body = Form("Dear %s expert(s), \n\n", fCurrentDetector.Data());
2097 body += Form("SHUTTLE just detected that your preprocessor "
36c99a6a 2098 "exited with ERROR state in run %d!!\n\n", GetCurrentRun());
57c1a579 2099 body += Form("Please check %s status on the web page asap!\n\n", fCurrentDetector.Data());
2100 body += Form("The last 10 lines of %s log file are following:\n\n");
2101
909732f7 2102 AliDebug(2, Form("Body begin: %s", body.Data()));
57c1a579 2103
2104 mailBody << body.Data();
2105 mailBody.close();
2106 mailBody.open(bodyFileName, ofstream::out | ofstream::app);
2107
9d733021 2108 TString logFileName = Form("%s/%s_%d.log", GetShuttleLogDir(), fCurrentDetector.Data(), GetCurrentRun());
57c1a579 2109 TString tailCommand = Form("tail -n 10 %s >> %s", logFileName.Data(), bodyFileName.Data());
2110 if (gSystem->Exec(tailCommand.Data()))
2111 {
2112 mailBody << Form("%s log file not found ...\n\n", fCurrentDetector.Data());
2113 }
2114
2115 TString endBody = Form("------------------------------------------------------\n\n");
36c99a6a 2116 endBody += Form("In case of problems please contact the SHUTTLE core team.\n\n");
2117 endBody += "Please do not answer this message directly, it is automatically generated.\n\n";
57c1a579 2118 endBody += "Sincerely yours,\n\n \t\t\tthe SHUTTLE\n";
2119
909732f7 2120 AliDebug(2, Form("Body end: %s", endBody.Data()));
57c1a579 2121
2122 mailBody << endBody.Data();
2123
2124 mailBody.close();
2125
2126 // send mail!
2127 TString mailCommand = Form("mail -s \"%s\" -c %s %s < %s",
2128 subject.Data(),
2129 cc.Data(),
2130 to.Data(),
2131 bodyFileName.Data());
909732f7 2132 AliDebug(2, Form("mail command: %s", mailCommand.Data()));
57c1a579 2133
2134 Bool_t result = gSystem->Exec(mailCommand.Data());
2135
2136 return result == 0;
2137}
d386d623 2138
2139//______________________________________________________________________________________________
441b0e9c 2140void AliShuttle::SetRunType()
2141{
2142// Gets run type from logbook and fills current Shuttle logbook entry
2143
2144 // check connection, in case connect
2145 if(!Connect(3)){
3301427a 2146 Log("SHUTTLE", "SetRunType - Couldn't connect to DAQ Logbook.");
441b0e9c 2147 return;
2148 }
2149
2150 TString sqlQuery = Form("select detector,run_type from %s where run_number=%d",
2151 fConfig->GetRunTypelbTable(), GetCurrentRun());
2152
2153 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2154
2155 // Query execution
2156 TSQLResult* aResult;
2157 aResult = dynamic_cast<TSQLResult*> (fServer[3]->Query(sqlQuery));
2158 if (!aResult) {
3301427a 2159 Log("SHUTTLE", Form("SetRunType - Can't execute query <%s>", sqlQuery.Data()));
441b0e9c 2160 return;
2161 }
2162
2163 TSQLRow* aRow;
2164 while ((aRow = aResult->Next())) {
2165 TString det(aRow->GetField(0), aRow->GetFieldLength(0));
2166 TString runType(aRow->GetField(1), aRow->GetFieldLength(1));
2167
2168 fLogbookEntry->SetRunType(det, runType);
2169 delete aRow;
2170 }
2171
2172 delete aResult;
2173
2174 return;
2175
2176}
2177
2178//______________________________________________________________________________________________
2179const char* AliShuttle::GetRunType(const char* detCode)
2180{
2181// returns run type read from "run type" logbook
2182
2183 if(!fLogbookEntry) {
2184 AliError("No logbook entry!");
2185 return 0;
2186 }
2187
2188 return fLogbookEntry->GetRunType(detCode);
2189}
2190
2191//______________________________________________________________________________________________
d386d623 2192void AliShuttle::SetShuttleTempDir(const char* tmpDir)
2193{
2194// sets Shuttle temp directory
2195
2196 fgkShuttleTempDir = gSystem->ExpandPathName(tmpDir);
2197}
2198
2199//______________________________________________________________________________________________
2200void AliShuttle::SetShuttleLogDir(const char* logDir)
2201{
2202// sets Shuttle log directory
2203
2204 fgkShuttleLogDir = gSystem->ExpandPathName(logDir);
2205}