368d3a3cb401d110322b2a04f0851f1d93a72ce9
[u/mrichter/AliRoot.git] / SHUTTLE / AliShuttle.cxx
1 /**************************************************************************
2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
3  *                                                                        *
4  * Author: The ALICE Off-line Project.                                    *
5  * Contributors are mentioned in the code where appropriate.              *
6  *                                                                        *
7  * Permission to use, copy, modify and distribute this software and its   *
8  * documentation strictly for non-commercial purposes is hereby granted   *
9  * without fee, provided that the above copyright notice appears in all   *
10  * copies and that both the copyright notice and this permission notice   *
11  * appear in the supporting documentation. The authors make no claims     *
12  * about the suitability of this software for any purpose. It is          *
13  * provided "as is" without express or implied warranty.                  *
14  **************************************************************************/
15
16 /*
17 $Log$
18 Revision 1.12  2006/08/08 14:19:29  jgrosseo
19 Update to shuttle classes (Alberto)
20
21 - Possibility to set the full object's path in the Preprocessor's and
22 Shuttle's  Store functions
23 - Possibility to extend the object's run validity in the same classes
24 ("startValidity" and "validityInfinite" parameters)
25 - Implementation of the StoreReferenceData function to store reference
26 data in a dedicated CDB storage.
27
28 Revision 1.11  2006/07/21 07:37:20  jgrosseo
29 last run is stored after each run
30
31 Revision 1.10  2006/07/20 09:54:40  jgrosseo
32 introducing status management: The processing per subdetector is divided into several steps,
33 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
34 can keep track of the number of failures and skips further processing after a certain threshold is
35 exceeded. These thresholds can be configured in LDAP.
36
37 Revision 1.9  2006/07/19 10:09:55  jgrosseo
38 new configuration, accesst to DAQ FES (Alberto)
39
40 Revision 1.8  2006/07/11 12:44:36  jgrosseo
41 adding parameters for extended validity range of data produced by preprocessor
42
43 Revision 1.7  2006/07/10 14:37:09  jgrosseo
44 small fix + todo comment
45
46 Revision 1.6  2006/07/10 13:01:41  jgrosseo
47 enhanced storing of last sucessfully processed run (alberto)
48
49 Revision 1.5  2006/07/04 14:59:57  jgrosseo
50 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
51
52 Revision 1.4  2006/06/12 09:11:16  jgrosseo
53 coding conventions (Alberto)
54
55 Revision 1.3  2006/06/06 14:26:40  jgrosseo
56 o) removed files that were moved to STEER
57 o) shuttle updated to follow the new interface (Alberto)
58
59 Revision 1.2  2006/03/07 07:52:34  hristov
60 New version (B.Yordanov)
61
62 Revision 1.6  2005/11/19 17:19:14  byordano
63 RetrieveDATEEntries and RetrieveConditionsData added
64
65 Revision 1.5  2005/11/19 11:09:27  byordano
66 AliShuttle declaration added
67
68 Revision 1.4  2005/11/17 17:47:34  byordano
69 TList changed to TObjArray
70
71 Revision 1.3  2005/11/17 14:43:23  byordano
72 import to local CVS
73
74 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
75 Initial import as subdirectory in AliRoot
76
77 Revision 1.2  2005/09/13 08:41:15  byordano
78 default startTime endTime added
79
80 Revision 1.4  2005/08/30 09:13:02  byordano
81 some docs added
82
83 Revision 1.3  2005/08/29 21:15:47  byordano
84 some docs added
85
86 */
87
88 //
89 // This class is the main manager for AliShuttle. 
90 // It organizes the data retrieval from DCS and call the 
91 // interface methods of AliPreprocessor.
92 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
93 // data for its set of aliases is retrieved. If there is registered
94 // AliPreprocessor for this detector then it will be used
95 // accroding to the schema (see AliPreprocessor).
96 // If there isn't registered AliPreprocessor than the retrieved
97 // data is stored automatically to the undelying AliCDBStorage.
98 // For detSpec is used the alias name.
99 //
100
101 #include "AliShuttle.h"
102
103 #include "AliCDBManager.h"
104 #include "AliCDBStorage.h"
105 #include "AliCDBId.h"
106 #include "AliCDBRunRange.h"
107 #include "AliCDBPath.h"
108 #include "AliCDBEntry.h"
109 #include "AliShuttleConfig.h"
110 #include "AliDCSClient.h"
111 #include "AliLog.h"
112 #include "AliPreprocessor.h"
113 #include "AliShuttleStatus.h"
114
115 #include <TSystem.h>
116 #include <TObject.h>
117 #include <TString.h>
118 #include <TTimeStamp.h>
119 #include <TObjString.h>
120 #include <TSQLServer.h>
121 #include <TSQLResult.h>
122 #include <TSQLRow.h>
123
124 #include <fstream>
125
126 ClassImp(AliShuttle)
127
128 TString AliShuttle::fgkMainCDB("alien://DBFolder=ShuttleCDB");
129 TString AliShuttle::fgkLocalCDB("local://LocalShuttleCDB");
130 TString AliShuttle::fgkMainRefStorage("alien://DBFolder=ShuttleReference");
131 TString AliShuttle::fgkLocalRefStorage("local://LocalReferenceStorage");
132
133 Bool_t AliShuttle::fgkProcessDCS(kTRUE); 
134
135
136 const char* AliShuttle::fgkShuttleTempDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/temp");
137 const char* AliShuttle::fgkShuttleLogDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/log");
138
139 const char* AliShuttle::fgkDetectorName[AliShuttle::fgkNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
140         "PHOS", "CPV", "RICH", "EMCAL", "MUON_TRK", "MUON_TRG", "FMD", "ZDC", "PMD", "START", "VZERO"};
141
142 const char* AliShuttle::fgkDetectorCode[AliShuttle::fgkNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
143         "PHS", "CPV", "HMP", "EMC", "MCH", "MTR", "FMD", "ZDC", "PMD", "T00", "V00"};
144
145 //______________________________________________________________________________________________
146 AliShuttle::AliShuttle(const AliShuttleConfig* config,
147                 UInt_t timeout, Int_t retries):
148 fConfig(config),
149 fTimeout(timeout), fRetries(retries),
150 fPreprocessorMap(),
151 fCurrentRun(-1),
152 fCurrentStartTime(0), fCurrentEndTime(0),
153 fCurrentDetector(""),
154 fStatusEntry(0)
155 {
156         //
157         // config: AliShuttleConfig used
158         // timeout: timeout used for AliDCSClient connection
159         // retries: the number of retries in case of connection error.
160         //
161
162         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
163         for(int iSys=0;iSys<3;iSys++) {
164                 fServer[iSys]=0;
165                 fFESlist[iSys].SetOwner(kTRUE);
166         }
167 }
168
169 //______________________________________________________________________
170 AliShuttle::AliShuttle(const AliShuttle& /*other*/):
171 AliShuttleInterface(),
172 fConfig(0),
173 fTimeout(0), fRetries(0),
174 fPreprocessorMap(),
175 fCurrentRun(-1),
176 fCurrentStartTime(0), fCurrentEndTime(0),
177 fCurrentDetector(""),
178 fStatusEntry(0)
179
180 {
181 // copy constructor (not implemented)
182
183 }
184
185 //______________________________________________________________________
186 AliShuttle &AliShuttle::operator=(const AliShuttle& /*other*/)
187 {
188 // assignment operator (not implemented)
189
190 return *this;
191 }
192
193 //______________________________________________________________________________________________
194 AliShuttle::~AliShuttle()
195 {
196 // destructor
197
198         fPreprocessorMap.DeleteAll();
199         for(int iSys=0;iSys<3;iSys++)
200                 if(fServer[iSys]) {
201                         fServer[iSys]->Close();
202                         delete fServer[iSys];
203                 }
204 }
205
206 //______________________________________________________________________________________________
207 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
208 {
209         //
210         // Registers new AliPreprocessor.
211         // It uses GetName() for indentificator of the pre processor.
212         // The pre processor is registered it there isn't any other
213         // with the same identificator (GetName()).
214         //
215
216         if (fPreprocessorMap.GetValue(preprocessor->GetName())) {
217                 AliWarning(Form("AliPreprocessor %s is already registered!",
218                         preprocessor->GetName()));
219                 return;
220         }
221
222         fPreprocessorMap.Add(new TObjString(preprocessor->GetName()), preprocessor);
223 }
224
225 //______________________________________________________________________________________________
226 UInt_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
227                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
228 {
229   // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
230   // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
231   // using this function. Use StoreReferenceData instead!
232   //
233
234   // The parameters are
235   //   1) the object's path.
236   //   2) the object to be stored
237   //   3) the metaData to be associated with the object
238   //   4) the validity start run number w.r.t. the current run,
239   //      if the data is valid only for this run leave the default 0
240   //   5) specifies if the calibration data is valid for infinity (this means until updated),
241   //      typical for calibration runs, the default is kFALSE
242   //
243   //
244   // returns 0 if fail
245   //         1 if stored in main (Grid) CDB storage
246   //         2 if stored in backup (Local) CDB storage
247
248
249         Int_t firstRun = GetCurrentRun() - validityStart;
250         if(firstRun < 0) {
251                 AliError("First valid run happens to be less than 0! Setting it to 0...");
252                 firstRun=0;
253         }
254
255         Int_t lastRun = -1;
256         if(validityInfinite) {
257                 lastRun = AliCDBRunRange::Infinity();
258         } else {
259                 lastRun = GetCurrentRun();
260         }
261
262         AliCDBId id(path, firstRun, lastRun);
263
264         UInt_t result = 0;
265
266         if (!(AliCDBManager::Instance()->GetStorage(fgkMainCDB))) {
267                 Log(fCurrentDetector, "Cannot activate main CDB storage!");
268         } else {
269                 result = (UInt_t) AliCDBManager::Instance()->GetStorage(fgkMainCDB)
270                                         ->Put(object, id, metaData);
271         }
272
273         if(!result) {
274
275                 Log(fCurrentDetector,
276                         "Error while storing object in main storage: it will go to local storage!");
277
278                 result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)
279                                         ->Put(object, id, metaData);
280
281                 if(result) {
282                         result = 2;
283                 }else{
284                         Log(fCurrentDetector, "Can't store data!");
285                 }
286         }
287         return result;
288
289 }
290
291 //______________________________________________________________________________________________
292 UInt_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object,
293                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
294 {
295   // Stores a CDB object in the storage for reference data. This objects will not be available during
296   // offline reconstrunction. Use this function for reference data only!
297   //
298
299   // The parameters are
300   //   1) the object's path.
301   //   2) the object to be stored
302   //   3) the metaData to be associated with the object
303   //   4) the validity start run number w.r.t. the current run,
304   //      if the data is valid only for this run leave the default 0
305   //   5) specifies if the calibration data is valid for infinity (this means until updated),
306   //      typical for calibration runs, the default is kFALSE
307   //
308   //
309   // returns 0 if fail
310   //         1 if stored in main (Grid) reference storage
311   //         2 if stored in backup (Local) reference storage
312
313         Int_t firstRun = GetCurrentRun() - validityStart;
314         if(firstRun < 0) {
315                 AliError("First valid run happens to be less than 0! Setting it to 0...");
316                 firstRun=0;
317         }
318
319         Int_t lastRun = -1;
320         if(validityInfinite) {
321                 lastRun = AliCDBRunRange::Infinity();
322         } else {
323                 lastRun = GetCurrentRun();
324         }
325
326         AliCDBId id(path, firstRun, lastRun);
327
328         UInt_t result = 0;
329
330         if (!(AliCDBManager::Instance()->GetStorage(fgkMainRefStorage))) {
331                 Log(fCurrentDetector, "Cannot activate main reference storage!");
332         } else {
333                 result = (UInt_t) AliCDBManager::Instance()->GetStorage(fgkMainRefStorage)
334                                         ->Put(object, id, metaData);
335         }
336
337         if(!result) {
338
339                 Log(fCurrentDetector,
340                         "Error while storing object in main reference storage: it will go to local ref storage!");
341
342                 result = AliCDBManager::Instance()->GetStorage(fgkLocalRefStorage)
343                                         ->Put(object, id, metaData);
344
345                 if(result) {
346                         result = 2;
347                 }else{
348                         Log(fCurrentDetector, "Can't store reference data!");
349                 }
350         }
351         return result;
352
353 }
354
355 //______________________________________________________________________________________________
356 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
357 {
358   // Reads the AliShuttleStatus from the CDB
359
360   if (fStatusEntry)
361   {
362     delete fStatusEntry;
363     fStatusEntry = 0;
364   }
365
366   fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalCDB())
367       ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), fCurrentRun);
368
369   if (!fStatusEntry)
370     return 0;
371
372   TObject* anObject = fStatusEntry->GetObject();
373   if (anObject == NULL || anObject->IsA() != AliShuttleStatus::Class())
374   {
375     AliError("Invalid object stored to CDB!");
376     return 0;
377   }
378
379   AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (anObject);
380   return status;
381 }
382
383 //______________________________________________________________________________________________
384 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
385 {
386   // writes the status for one subdetector
387
388   if (fStatusEntry)
389   {
390     delete fStatusEntry;
391     fStatusEntry = 0;
392   }
393
394   AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), fCurrentRun, fCurrentRun);
395
396   fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
397
398   UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
399
400   if (!result)
401   {
402     AliError(Form("WriteShuttleStatus for %s, run %d failed", fCurrentDetector.Data(), fCurrentRun));
403     return kFALSE;
404   }
405
406   return kTRUE;
407 }
408
409 //______________________________________________________________________________________________
410 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
411 {
412   // changes the AliShuttleStatus for the given detector and run to the given status
413
414   if (!fStatusEntry)
415   {
416     AliError("UNEXPECTED: fStatusEntry empty");
417     return;
418   }
419
420   TObject* anObject = fStatusEntry->GetObject();
421   AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (anObject);
422
423   if (!status)
424   {
425     AliError("UNEXPECTED: status could not be read from current CDB entry");
426     return;
427   }
428
429   Log("SHUTTLE", Form("%s: Changing state from %s to %s", fCurrentDetector.Data(),
430                                 status->GetStatusName(), status->GetStatusName(newStatus)));
431
432   status->SetStatus(newStatus);
433   if (increaseCount)
434     status->IncreaseCount();
435
436   AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
437 }
438
439 //______________________________________________________________________________________________
440 Bool_t AliShuttle::ContinueProcessing()
441 {
442   // this function reads the AliShuttleStatus information from CDB and
443   // checks if the processing should be continued
444   // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
445
446   AliShuttleStatus* status = ReadShuttleStatus();
447   if (!status)
448   {
449     // first time
450
451     Log("SHUTTLE", Form("%s: Processing first time.", fCurrentDetector.Data()));
452     status = new AliShuttleStatus(AliShuttleStatus::kStarted);
453     return WriteShuttleStatus(status);
454   }
455
456   if (status->GetStatus() == AliShuttleStatus::kDone)
457   {
458     Log("SHUTTLE", Form("%s already done for run %d", fCurrentDetector.Data(), fCurrentRun));
459     return kFALSE;
460   }
461
462   if (status->GetStatus() == AliShuttleStatus::kFailed)
463   {
464     Log("SHUTTLE", Form("%s already in failed state for run %d", fCurrentDetector.Data(), fCurrentRun));
465     return kFALSE;
466   }
467
468   // if we get here, there is a restart
469
470   // abort conditions
471   if (status->GetStatus() == AliShuttleStatus::kPPStarted && status->GetCount() >= fConfig->GetMaxPPRetries() ||
472       status->GetCount() >= fConfig->GetMaxRetries())
473   {
474     Log("SHUTTLE", Form("%s, run %d failed too often, %d times, status %s. Skipping processing.",
475                 fCurrentDetector.Data(), fCurrentRun, status->GetCount(), status->GetStatusName()));
476
477     return kFALSE;
478   }
479
480   Log("SHUTTLE", Form("Restart of %s, run %d. Got stuck before in %s, count %d",
481                 fCurrentDetector.Data(), fCurrentRun, status->GetStatusName(), status->GetCount()));
482
483   UpdateShuttleStatus(AliShuttleStatus::kStarted, kTRUE);
484
485   return kTRUE;
486 }
487
488 //______________________________________________________________________________________________
489 Bool_t AliShuttle::Process(Int_t run, UInt_t startTime, UInt_t endTime)
490 {
491         //
492         // Makes data retrieval for all detectors in the configuration.
493         // run: is the run number used
494         // startTime: is the run start time
495         // endTime: is the run end time
496         // Returns kFALSE in case of error occured and kTRUE otherwise
497         //
498
499         AliInfo(Form("\n\n ^*^*^*^*^*^* Processing run %d ^*^*^*^*^*^*", run));
500
501         // Initialization
502         Bool_t hasError = kFALSE;
503         for(Int_t iSys=0;iSys<3;iSys++) fFESCalled[iSys]=kFALSE;
504
505         fCurrentRun = run;
506         fCurrentStartTime = startTime;
507         fCurrentEndTime = endTime;
508
509         // Loop on detectors in the configuration
510         TIter iter(fConfig->GetDetectors());
511         TObjString* aDetector;
512
513         while ((aDetector = (TObjString*) iter.Next())) {
514                 fCurrentDetector = aDetector->String();
515
516                 Bool_t detectorError=kFALSE;
517                 if (!fConfig->HostProcessDetector(fCurrentDetector)) continue;
518
519                 if (ContinueProcessing() == kFALSE) continue;
520
521                 if(!Process()) {
522                         hasError = kTRUE;
523                         detectorError=kTRUE;
524                         continue;
525                 }
526                 AliInfo(Form("Process ended successfully for detector %s!",aDetector->GetName()));
527
528                 // Process successful: Update time_processed field in FES logbooks!
529                 if(fFESCalled[kDAQ]) {
530                         hasError = (UpdateDAQTable() == kFALSE);
531                         fFESlist[kDAQ].Clear();
532                 }
533                 //if(fFESCalled[kDCS]) {
534                 //      hasError = UpdateDCSTable(aDetector->GetName());
535                 //      fFESlist[kDCS].Clear();
536                 //}
537                 //if(fFESCalled[kHLT]) {
538                 //      hasError = UpdateHLTTable(aDetector->GetName());
539                 //      fFESlist[kHLT].Clear();
540                 //}
541
542                 UpdateShuttleStatus(AliShuttleStatus::kDone);
543         }
544
545         fCurrentRun = -1;
546         fCurrentStartTime = 0;
547         fCurrentEndTime = 0;
548
549         return hasError == kFALSE;
550 }
551
552 //______________________________________________________________________________________________
553 Bool_t AliShuttle::Process()
554 {
555         //
556         // Makes data retrieval just for one specific detector.
557         // Threre should be a configuration for this detector.
558         // run: is the run number used
559         // startTime: is the run start time
560         // endTime: is the run end time
561         // detector: detector for which the retrieval will be made
562         // Returns kFALSE in case of error occured and kTRUE otherwise
563         //
564
565         AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), fCurrentRun));
566
567         if (!fConfig->HasDetector(fCurrentDetector)) {
568                 Log(fCurrentDetector, "There isn't any configuration for %s !");
569                 UpdateShuttleStatus(AliShuttleStatus::kFailed);
570                 return kFALSE;
571         }
572
573         UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
574
575         TString host(fConfig->GetDCSHost(fCurrentDetector));
576         Int_t port = fConfig->GetDCSPort(fCurrentDetector);
577
578         TIter iter(fConfig->GetDCSAliases(fCurrentDetector));
579         TObjString* anAlias;
580         TMap aliasMap;
581
582         Bool_t hasError = kFALSE;
583         Bool_t result=kFALSE;
584
585         while ((anAlias = (TObjString*) iter.Next())) {
586                 TObjArray valueSet;
587                 // TODO Test only... I've added a flag that allows to
588                 // exclude DCS archive DB query
589                 if(fgkProcessDCS){
590                         AliInfo("Querying DCS archive DB data...");
591                         result = GetValueSet(host, port, anAlias->String(), valueSet);
592                 } else {
593                         AliInfo(Form("Skipping DCS processing. Port = %d",port));
594                         result = kTRUE;
595                 }
596                 if(result) {
597                         aliasMap.Add(anAlias->Clone(), valueSet.Clone());
598                 }else{
599                         TString message = Form("Error while retrieving alias %s !",
600                                         anAlias->GetName());
601                         Log(fCurrentDetector, message.Data());
602                         hasError = kTRUE;
603                         break;
604                 }
605         }
606
607         if (hasError)
608         {
609                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
610                 return kFALSE;
611         }
612
613   UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
614
615   AliPreprocessor* aPreprocessor =
616                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
617         if(aPreprocessor)
618         {
619                 aPreprocessor->Initialize(fCurrentRun, fCurrentStartTime, fCurrentEndTime);
620
621                 // TODO Think about what to do in case of "Grid storage error"
622                 // (-> object put in local backup storage, return 2)
623                 hasError = (aPreprocessor->Process(&aliasMap) == 0);
624         }else{
625     // TODO default behaviour?
626                 AliInfo(Form("No Preprocessor for %s: storing TMap of DP arrays into CDB!", fCurrentDetector.Data()));
627                 AliCDBMetaData metaData;
628                 AliDCSValue dcsValue(fCurrentStartTime, fCurrentEndTime);
629                 metaData.SetResponsible(Form("Duck, Donald"));
630                 metaData.SetProperty("StartEndTime", &dcsValue);
631                 metaData.SetComment("Automatically stored by Shuttle!");
632                 AliCDBPath path(fCurrentDetector,"DCS","Data");
633                 hasError = (Store(path, &aliasMap, &metaData) == 0);
634         }
635
636   if (hasError)
637     UpdateShuttleStatus(AliShuttleStatus::kPPError);
638   else
639     UpdateShuttleStatus(AliShuttleStatus::kPPDone);
640
641         aliasMap.Delete();
642
643         return hasError == kFALSE;
644 }
645
646 //______________________________________________________________________________________________
647 Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* alias,
648                                 TObjArray& valueSet)
649 {
650 // Retrieve all "alias" data points from the DCS server
651 // host, port: TSocket connection parameters
652 // alias: name of the alias
653 // valueSet: array of retrieved AliDCSValue's 
654
655         AliDCSClient client(host, port, fTimeout, fRetries);
656         if (!client.IsConnected()) {
657                 return kFALSE;
658         }
659
660         Int_t result = client.GetAliasValues(alias,
661                 GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
662
663         if (result < 0) {
664                 AliError(Form("Can't get '%s'! Reason: %s",
665                         alias, AliDCSClient::GetErrorString(result)));
666
667                 if (result == AliDCSClient::fgkServerError) {
668                         AliError(Form("Server error: %s",
669                                 client.GetServerError().Data()));
670                 }
671
672                 return kFALSE;
673         }
674
675         return kTRUE;
676 }
677
678 //______________________________________________________________________________________________
679 const char* AliShuttle::GetFile(Int_t system, const char* detector,
680                 const char* id, const char* source)
681 {
682 // Get calibration file from file exchange servers
683 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
684
685         switch(system){
686                 case kDAQ:
687                         return GetDAQFileName(detector, id, source);
688                         break;
689                 case kDCS:
690                         return GetDCSFileName(detector, id, source);
691                         break;
692                 case kHLT:
693                         return GetHLTFileName(detector, id, source);
694                         break;
695                 default:
696                         AliError(Form("No valid system index: %d",system));
697         }
698
699         return 0;
700 }
701
702 //______________________________________________________________________________________________
703 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
704 {
705 // Get sources producing the condition file Id from file exchange servers
706 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
707
708         switch(system){
709                 case kDAQ:
710                         return GetDAQFileSources(detector, id);
711                         break;
712                 case kDCS:
713                         return GetDCSFileSources(detector, id);
714                         break;
715                 case kHLT:
716                         return GetHLTFileSources(detector, id);
717                         break;
718                 default:
719                         AliError(Form("No valid system index: %d",system));
720         }
721
722         return NULL;
723 }
724
725 //______________________________________________________________________________________________
726 Bool_t AliShuttle::Connect(Int_t system){
727 // Connect to MySQL Server of the system's FES logbook
728
729         // check connection: if already connected return
730         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
731
732         TString aFESlbHost= Form("mysql://%s", fConfig->GetFESlbHost(system));
733
734         fServer[system] = TSQLServer::Connect(aFESlbHost,
735                         fConfig->GetFESlbUser(system),
736                         fConfig->GetFESlbPass(system));
737         if (!fServer[system] || !fServer[system]->IsConnected()) {
738                 AliError(Form("Can't establish connection to FES logbook for %s !",fkSystemNames[system]));
739                 return kFALSE;
740         }
741
742         // Get tables
743         // TODO in the configuration should the table name be there too?
744         switch(system){
745                 case kDAQ:
746                         fServer[kDAQ]->GetTables("REFSYSLOG");
747                         break;
748                 case kDCS:
749                         //fServer[kDCS]->GetTables("REFSYSLOG");
750                         break;
751                 case kHLT:
752                         //fServer[kHLT]->GetTables("REFSYSLOG");
753                         break;
754                 default:
755                         break;
756         }
757
758         return kTRUE;
759 }
760
761 //______________________________________________________________________________________________
762 const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source){
763 // Retrieves a file from the DAQ FES.
764 // First queris the DAQ logbook_fs for the DAQ file name, using the run, detector, id and source info
765 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
766 // run: current run being processed (fCurrentRun)
767 // detector: comes from the Preprocessor name (must be converted into detector code with GetDetCode)
768 // id: provided as a parameter by the Preprocessor
769 // source: provided by the Preprocessor through GetFileSources function
770
771         // check connection, in case connect
772         if(!Connect(kDAQ)){
773                 Log(detector, "GetDAQFileName: Couldn't connect to DAQ Logbook !");
774                 return 0;
775         }
776
777         // Query preparation
778         TString sqlQueryStart = "select filePath from logbook_fs where";
779         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
780                                 fCurrentRun, GetDetCode(detector), id, source);
781         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
782
783         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
784
785         // Query execution
786         TSQLResult* aResult;
787         aResult = fServer[kDAQ]->Query(sqlQuery);
788         if (!aResult) {
789                 Log(detector, Form("Can't execute query <%s>!", sqlQuery.Data()));
790                 return 0;
791         }
792
793         if (aResult->GetRowCount() == 0) {
794                 Log(detector,
795                         Form("GetDAQFileName: No result from SQL query <%s>!", sqlQuery.Data()));
796                 delete aResult;
797                 return 0;
798         }
799
800         if (aResult->GetRowCount() >1) {
801                 Log(detector,
802                         Form("GetDAQFileName: More than one row resulting from SQL query <%s>!", sqlQuery.Data()));
803                 delete aResult;
804                 return 0;
805         }
806
807         TSQLRow* aRow = aResult->Next();
808
809         if(!aRow){
810                 Log(detector, Form("GetDAQFileName: Empty set result from query <%s>!", sqlQuery.Data()));
811                 delete aResult;
812                 return 0;
813         }
814
815         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
816
817         delete aResult;
818
819         AliDebug(2, Form("filePath = %s",filePath.Data()));
820
821         // retrieved file is renamed to make it unique
822         TString localFileName = Form("%s_%d_%s_%s.shuttle",
823                                         detector, fCurrentRun, id, source);
824
825         // file retrieval from DAQ FES
826         Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
827         if(!result) {
828                 Log(detector, Form("copying file %s from DAQ FES failed!", filePath.Data()));
829                 return 0;
830         } else {
831                 AliInfo(Form("File %s copied from DAQ FES into %s/%s !",
832                         filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
833         }
834
835
836         fFESCalled[kDAQ]=kTRUE;
837         TObjString *fileParams = new TObjString(Form("%s_!?!_%s", id, source));
838         fFESlist[kDAQ].Add(fileParams);
839
840         return localFileName.Data();
841
842 }
843
844 //______________________________________________________________________________________________
845 Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName){
846
847         // check temp directory: trying to cd to temp; if it does not exist, create it
848         AliDebug(2, Form("Copy file %s from DAQ FES into folder %s and rename it as %s",
849                         daqFileName,fgkShuttleTempDir, localFileName));
850
851         void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
852         if (dir == NULL) {
853                 if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
854                         AliError(Form("Can't open directory <%s>!", fgkShuttleTempDir));
855                         return kFALSE;
856                 }
857
858         } else {
859                 gSystem->FreeDirectory(dir);
860         }
861
862         TString baseDAQFESFolder = "DAQ";
863         TString command = Form("scp %s@%s:%s/%s %s/%s",
864                 fConfig->GetFESUser(kDAQ),
865                 fConfig->GetFESHost(kDAQ),
866                 baseDAQFESFolder.Data(),
867                 daqFileName,
868                 fgkShuttleTempDir,
869                 localFileName);
870
871         AliDebug(2, Form("%s",command.Data()));
872
873         UInt_t nRetries = 0;
874         UInt_t maxRetries = 3;
875
876         // copy!! if successful TSystem::Exec returns 0
877         while(nRetries++ < maxRetries) {
878                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
879                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
880         }
881
882         return kFALSE;
883
884 }
885
886 //______________________________________________________________________________________________
887 TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id){
888 // Retrieves a file from the DCS FES.
889
890         // check connection, in case connect
891         if(!Connect(kDAQ)){
892                 Log(detector, "GetDAQFileName: Couldn't connect to DAQ Logbook !");
893                 return 0;
894         }
895
896         // Query preparation
897         TString sqlQueryStart = "select DAQsource from logbook_fs where";
898         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
899                                 fCurrentRun, GetDetCode(detector), id);
900         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
901
902         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
903
904         // Query execution
905         TSQLResult* aResult;
906         aResult = fServer[kDAQ]->Query(sqlQuery);
907         if (!aResult) {
908                 Log(detector, Form("GetDAQFileSources: Can't execute query <%s>!", sqlQuery.Data()));
909                 return 0;
910         }
911
912         if (aResult->GetRowCount() == 0) {
913                 Log(detector,
914                         Form("GetDAQFileSources: No result from SQL query <%s>!", sqlQuery.Data()));
915                 delete aResult;
916                 return 0;
917         }
918
919         TSQLRow* aRow;
920         TList *list = new TList();
921         list->SetOwner(1);
922
923         while((aRow = aResult->Next())){
924
925                 TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
926                 AliDebug(2, Form("daqSource = %s", daqSource.Data()));
927                 list->Add(new TObjString(daqSource));
928         }
929         delete aResult;
930
931         return list;
932
933 }
934
935 //______________________________________________________________________________________________
936 Bool_t AliShuttle::UpdateDAQTable(){
937 // Update DAQ table filling time_processed field in all rows corresponding to current run and detector
938
939         // check connection, in case connect
940         if(!Connect(kDAQ)){
941                 Log(fCurrentDetector, "UpdateDAQTable: Couldn't connect to DAQ Logbook !");
942                 return kFALSE;
943         }
944
945         TTimeStamp now; // now
946
947         // Loop on FES list entries
948         TIter iter(&fFESlist[kDAQ]);
949         TObjString *aFESentry=0;
950         while((aFESentry = dynamic_cast<TObjString*> (iter.Next()))){
951                 TString aFESentrystr = aFESentry->String();
952                 TObjArray *aFESarray = aFESentrystr.Tokenize("_!?!_");
953                 if(!aFESarray || aFESarray->GetEntries() != 2 ) {
954                         Log(fCurrentDetector,Form("UpdateDAQTable: error updating FES entry! string = %s",
955                                 aFESentrystr.Data()));
956                         if(aFESarray) delete aFESarray;
957                         return kFALSE;
958                 }
959                 const char* fileId = ((TObjString*) aFESarray->At(0))->GetName();
960                 const char* daqSource = ((TObjString*) aFESarray->At(1))->GetName();
961                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
962                         fCurrentRun,GetDetCode(fCurrentDetector), fileId, daqSource);
963
964                 delete aFESarray;
965
966                 TString sqlQuery = Form("update logbook_fs set time_processed=%d %s", now.GetSec(), whereClause.Data());
967
968                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
969
970                 // Query execution
971                 TSQLResult* aResult;
972                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
973                 if (!aResult) {
974                         Log(fCurrentDetector, Form("UpdateDAQTable: Can't execute query <%s>!", sqlQuery.Data()));
975                         return kFALSE;
976                 }
977                 delete aResult;
978
979                 // check result - TODO Is it necessary?
980                 sqlQuery = Form("select time_processed from logbook_fs %s", whereClause.Data());
981                 AliDebug(2, Form(" CHECK - SQL query: \n%s",sqlQuery.Data()));
982
983                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
984                 if (!aResult) {
985                         AliWarning("Can't check result!");
986                         continue;
987                 }
988
989         if (aResult->GetRowCount() == 0) {
990                 Log(fCurrentDetector,
991                         Form("GetDAQFileName: No result from SQL query <%s>!", sqlQuery.Data()));
992                 delete aResult;
993                 //return 0;
994         }
995
996         if (aResult->GetRowCount() >1) {
997                 Log(fCurrentDetector,
998                         Form("GetDAQFileName: More than one row resulting from SQL query <%s>!", sqlQuery.Data()));
999                 delete aResult;
1000                 //return 0;
1001         }
1002
1003                 TSQLRow *row = dynamic_cast<TSQLRow*> (aResult->Next());
1004                 TString processedTimeString(row->GetField(0), row->GetFieldLength(0));
1005                 Int_t processedTime = processedTimeString.Atoi();
1006                 if(processedTime != now.GetSec()){
1007                         Log(fCurrentDetector, Form("UpdateDAQTable: Update table error: processed_time=%d, now=%d !",
1008                                 processedTime, now.GetSec()));
1009                         delete aResult;
1010                         return kFALSE;
1011                 }
1012
1013                 delete aResult;
1014
1015         }
1016
1017         return kTRUE;
1018 }
1019
1020 //______________________________________________________________________________________________
1021 const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1022 // Retrieves a file from the DCS FES.
1023
1024 return "You're in DCS";
1025
1026 }
1027
1028 //______________________________________________________________________________________________
1029 TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
1030 // Retrieves a file from the DCS FES.
1031
1032 return NULL;
1033
1034 }
1035
1036 //______________________________________________________________________________________________
1037 const char* AliShuttle::GetHLTFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1038 // Retrieves a file from the HLT FES.
1039
1040 return "You're in HLT";
1041
1042 }
1043
1044 //______________________________________________________________________________________________
1045 TList* AliShuttle::GetHLTFileSources(const char* /*detector*/, const char* /*id*/){
1046 // Retrieves a file from the HLT FES.
1047
1048 return NULL;
1049
1050 }
1051
1052 //______________________________________________________________________________________________
1053 const char* AliShuttle::GetDetCode(const char* detector){
1054 // Return detector code
1055
1056         for(int iDet=0; iDet < fgkNDetectors; iDet++){
1057                 if(!strcmp(fgkDetectorName[iDet], detector)) return fgkDetectorCode[iDet];
1058         }
1059
1060         return 0;
1061 }
1062
1063 //______________________________________________________________________________________________
1064 void AliShuttle::Log(const char* detector, const char* message)
1065 {
1066 // Fill log string with a message
1067
1068         void* dir = gSystem->OpenDirectory(fgkShuttleLogDir);
1069         if (dir == NULL) {
1070                 if (gSystem->mkdir(fgkShuttleLogDir, kTRUE)) {
1071                         AliError(Form("Can't open directory <%s>!", fgkShuttleTempDir));
1072                         return;
1073                 }
1074
1075         } else {
1076                 gSystem->FreeDirectory(dir);
1077         }
1078
1079         TString toLog = Form("%s: %s, run %d - %s", TTimeStamp(time(0)).AsString(),
1080         detector, GetCurrentRun(), message);
1081         AliInfo(toLog.Data());
1082
1083         TString fileName;
1084         fileName.Form("%s/%s.log", fgkShuttleLogDir, detector);
1085         gSystem->ExpandPathName(fileName);
1086
1087         ofstream logFile;
1088         logFile.open(fileName, ofstream::out | ofstream::app);
1089
1090         if (!logFile.is_open()) {
1091                 AliError(Form("Could not open file %s", fileName.Data()));
1092                 return;
1093         }
1094
1095         logFile << toLog.Data() << "\n";
1096
1097         logFile.close();
1098 }