small update
[u/mrichter/AliRoot.git] / SHUTTLE / AliShuttle.cxx
1 /**************************************************************************
2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
3  *                                                                        *
4  * Author: The ALICE Off-line Project.                                    *
5  * Contributors are mentioned in the code where appropriate.              *
6  *                                                                        *
7  * Permission to use, copy, modify and distribute this software and its   *
8  * documentation strictly for non-commercial purposes is hereby granted   *
9  * without fee, provided that the above copyright notice appears in all   *
10  * copies and that both the copyright notice and this permission notice   *
11  * appear in the supporting documentation. The authors make no claims     *
12  * about the suitability of this software for any purpose. It is          *
13  * provided "as is" without express or implied warranty.                  *
14  **************************************************************************/
15
16 /*
17 $Log$
18 Revision 1.13  2006/08/15 10:50:00  jgrosseo
19 effc++ corrections (alberto)
20
21 Revision 1.12  2006/08/08 14:19:29  jgrosseo
22 Update to shuttle classes (Alberto)
23
24 - Possibility to set the full object's path in the Preprocessor's and
25 Shuttle's  Store functions
26 - Possibility to extend the object's run validity in the same classes
27 ("startValidity" and "validityInfinite" parameters)
28 - Implementation of the StoreReferenceData function to store reference
29 data in a dedicated CDB storage.
30
31 Revision 1.11  2006/07/21 07:37:20  jgrosseo
32 last run is stored after each run
33
34 Revision 1.10  2006/07/20 09:54:40  jgrosseo
35 introducing status management: The processing per subdetector is divided into several steps,
36 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
37 can keep track of the number of failures and skips further processing after a certain threshold is
38 exceeded. These thresholds can be configured in LDAP.
39
40 Revision 1.9  2006/07/19 10:09:55  jgrosseo
41 new configuration, accesst to DAQ FES (Alberto)
42
43 Revision 1.8  2006/07/11 12:44:36  jgrosseo
44 adding parameters for extended validity range of data produced by preprocessor
45
46 Revision 1.7  2006/07/10 14:37:09  jgrosseo
47 small fix + todo comment
48
49 Revision 1.6  2006/07/10 13:01:41  jgrosseo
50 enhanced storing of last sucessfully processed run (alberto)
51
52 Revision 1.5  2006/07/04 14:59:57  jgrosseo
53 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
54
55 Revision 1.4  2006/06/12 09:11:16  jgrosseo
56 coding conventions (Alberto)
57
58 Revision 1.3  2006/06/06 14:26:40  jgrosseo
59 o) removed files that were moved to STEER
60 o) shuttle updated to follow the new interface (Alberto)
61
62 Revision 1.2  2006/03/07 07:52:34  hristov
63 New version (B.Yordanov)
64
65 Revision 1.6  2005/11/19 17:19:14  byordano
66 RetrieveDATEEntries and RetrieveConditionsData added
67
68 Revision 1.5  2005/11/19 11:09:27  byordano
69 AliShuttle declaration added
70
71 Revision 1.4  2005/11/17 17:47:34  byordano
72 TList changed to TObjArray
73
74 Revision 1.3  2005/11/17 14:43:23  byordano
75 import to local CVS
76
77 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
78 Initial import as subdirectory in AliRoot
79
80 Revision 1.2  2005/09/13 08:41:15  byordano
81 default startTime endTime added
82
83 Revision 1.4  2005/08/30 09:13:02  byordano
84 some docs added
85
86 Revision 1.3  2005/08/29 21:15:47  byordano
87 some docs added
88
89 */
90
91 //
92 // This class is the main manager for AliShuttle. 
93 // It organizes the data retrieval from DCS and call the 
94 // interface methods of AliPreprocessor.
95 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
96 // data for its set of aliases is retrieved. If there is registered
97 // AliPreprocessor for this detector then it will be used
98 // accroding to the schema (see AliPreprocessor).
99 // If there isn't registered AliPreprocessor than the retrieved
100 // data is stored automatically to the undelying AliCDBStorage.
101 // For detSpec is used the alias name.
102 //
103
104 #include "AliShuttle.h"
105
106 #include "AliCDBManager.h"
107 #include "AliCDBStorage.h"
108 #include "AliCDBId.h"
109 #include "AliCDBRunRange.h"
110 #include "AliCDBPath.h"
111 #include "AliCDBEntry.h"
112 #include "AliShuttleConfig.h"
113 #include "AliDCSClient.h"
114 #include "AliLog.h"
115 #include "AliPreprocessor.h"
116 #include "AliShuttleStatus.h"
117
118 #include <TSystem.h>
119 #include <TObject.h>
120 #include <TString.h>
121 #include <TTimeStamp.h>
122 #include <TObjString.h>
123 #include <TSQLServer.h>
124 #include <TSQLResult.h>
125 #include <TSQLRow.h>
126
127 #include <fstream>
128
129 ClassImp(AliShuttle)
130
131 TString AliShuttle::fgkMainCDB("alien://DBFolder=ShuttleCDB");
132 TString AliShuttle::fgkLocalCDB("local://LocalShuttleCDB");
133 TString AliShuttle::fgkMainRefStorage("alien://DBFolder=ShuttleReference");
134 TString AliShuttle::fgkLocalRefStorage("local://LocalReferenceStorage");
135
136 Bool_t AliShuttle::fgkProcessDCS(kTRUE); 
137
138
139 const char* AliShuttle::fgkShuttleTempDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/temp");
140 const char* AliShuttle::fgkShuttleLogDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/log");
141
142 const char* AliShuttle::fgkDetectorName[AliShuttle::fgkNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
143         "PHOS", "CPV", "RICH", "EMCAL", "MUON_TRK", "MUON_TRG", "FMD", "ZDC", "PMD", "START", "VZERO"};
144
145 const char* AliShuttle::fgkDetectorCode[AliShuttle::fgkNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
146         "PHS", "CPV", "HMP", "EMC", "MCH", "MTR", "FMD", "ZDC", "PMD", "T00", "V00"};
147
148 //______________________________________________________________________________________________
149 AliShuttle::AliShuttle(const AliShuttleConfig* config,
150                 UInt_t timeout, Int_t retries):
151 fConfig(config),
152 fTimeout(timeout), fRetries(retries),
153 fPreprocessorMap(),
154 fCurrentRun(-1),
155 fCurrentStartTime(0), fCurrentEndTime(0),
156 fCurrentDetector(""),
157 fStatusEntry(0),
158 fGridError(kFALSE)
159 {
160         //
161         // config: AliShuttleConfig used
162         // timeout: timeout used for AliDCSClient connection
163         // retries: the number of retries in case of connection error.
164         //
165
166         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
167         for(int iSys=0;iSys<3;iSys++) {
168                 fServer[iSys]=0;
169                 fFESlist[iSys].SetOwner(kTRUE);
170         }
171 }
172
173 //______________________________________________________________________
174 AliShuttle::AliShuttle(const AliShuttle& /*other*/):
175 AliShuttleInterface(),
176 fConfig(0),
177 fTimeout(0), fRetries(0),
178 fPreprocessorMap(),
179 fCurrentRun(-1),
180 fCurrentStartTime(0), fCurrentEndTime(0),
181 fCurrentDetector(""),
182 fStatusEntry(0),
183 fGridError(kFALSE)
184 {
185 // copy constructor (not implemented)
186
187 }
188
189 //______________________________________________________________________
190 AliShuttle &AliShuttle::operator=(const AliShuttle& /*other*/)
191 {
192 // assignment operator (not implemented)
193
194 return *this;
195 }
196
197 //______________________________________________________________________________________________
198 AliShuttle::~AliShuttle()
199 {
200 // destructor
201
202         fPreprocessorMap.DeleteAll();
203         for(int iSys=0;iSys<3;iSys++)
204                 if(fServer[iSys]) {
205                         fServer[iSys]->Close();
206                         delete fServer[iSys];
207                 }
208 }
209
210 //______________________________________________________________________________________________
211 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
212 {
213         //
214         // Registers new AliPreprocessor.
215         // It uses GetName() for indentificator of the pre processor.
216         // The pre processor is registered it there isn't any other
217         // with the same identificator (GetName()).
218         //
219
220         if (fPreprocessorMap.GetValue(preprocessor->GetName())) {
221                 AliWarning(Form("AliPreprocessor %s is already registered!",
222                         preprocessor->GetName()));
223                 return;
224         }
225
226         fPreprocessorMap.Add(new TObjString(preprocessor->GetName()), preprocessor);
227 }
228
229 //______________________________________________________________________________________________
230 UInt_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
231                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
232 {
233   // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
234   // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
235   // using this function. Use StoreReferenceData instead!
236   // It calls WriteToCDB function which perform actual storage
237
238         return WriteToCDB(fgkMainCDB, fgkLocalCDB, path, object,
239                                 metaData, validityStart, validityInfinite);
240
241 }
242
243 //______________________________________________________________________________________________
244 UInt_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object,
245                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
246 {
247   // Stores a CDB object in the storage for reference data. This objects will not be available during
248   // offline reconstrunction. Use this function for reference data only!
249   // It calls WriteToCDB function which perform actual storage
250
251         return WriteToCDB(fgkMainRefStorage, fgkLocalRefStorage, path, object,
252                                 metaData, validityStart, validityInfinite);
253
254 }
255
256 //______________________________________________________________________________________________
257 UInt_t AliShuttle::WriteToCDB(const char* mainUri, const char* localUri,
258                         const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
259                         Int_t validityStart, Bool_t validityInfinite)
260 {
261   // write object into the CDB. Parameters are passed by Store and StoreReferenceData functions.
262   // The parameters are:
263   //   1) Uri of the main storage (Grid)
264   //   2) Uri of the backup storage (Local)
265   //   3) the object's path.
266   //   4) the object to be stored
267   //   5) the metaData to be associated with the object
268   //   6) the validity start run number w.r.t. the current run,
269   //      if the data is valid only for this run leave the default 0
270   //   7) specifies if the calibration data is valid for infinity (this means until updated),
271   //      typical for calibration runs, the default is kFALSE
272   //
273   // returns 0 if fail
274   //         1 if stored in main (Grid) storage
275   //         2 if stored in backup (Local) storage
276
277         const char* cdbType = (mainUri == fgkMainCDB) ? "CDB" : "Reference";
278         
279         Int_t firstRun = GetCurrentRun() - validityStart;
280         if(firstRun < 0) {
281                 AliError("First valid run happens to be less than 0! Setting it to 0...");
282                 firstRun=0;
283         }
284
285         Int_t lastRun = -1;
286         if(validityInfinite) {
287                 lastRun = AliCDBRunRange::Infinity();
288         } else {
289                 lastRun = GetCurrentRun();
290         }
291
292         AliCDBId id(path, firstRun, lastRun);
293
294         UInt_t result = 0;
295
296         if (!(AliCDBManager::Instance()->GetStorage(mainUri))) {
297                 Log(fCurrentDetector, Form("Cannot activate main %s storage!", cdbType));
298         } else {
299                 result = (UInt_t) AliCDBManager::Instance()->GetStorage(mainUri)
300                                         ->Put(object, id, metaData);
301         }
302
303         if(!result) {
304
305                 Log(fCurrentDetector,
306                         Form("Problem with main %s storage. Object will go to local storage!", cdbType));
307
308                 result = AliCDBManager::Instance()->GetStorage(localUri)
309                                         ->Put(object, id, metaData);
310
311                 if(result) {
312                         result = 2;
313                         fGridError = kTRUE;
314                 }else{
315                         Log(fCurrentDetector, "Can't store data!");
316                 }
317         }
318         return result;
319
320 }
321
322 //______________________________________________________________________________________________
323 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
324 {
325   // Reads the AliShuttleStatus from the CDB
326
327   if (fStatusEntry)
328   {
329     delete fStatusEntry;
330     fStatusEntry = 0;
331   }
332
333   fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalCDB())
334       ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), fCurrentRun);
335
336   if (!fStatusEntry)
337     return 0;
338
339   TObject* anObject = fStatusEntry->GetObject();
340   if (anObject == NULL || anObject->IsA() != AliShuttleStatus::Class())
341   {
342     AliError("Invalid object stored to CDB!");
343     return 0;
344   }
345
346   AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (anObject);
347   return status;
348 }
349
350 //______________________________________________________________________________________________
351 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
352 {
353   // writes the status for one subdetector
354
355   if (fStatusEntry)
356   {
357     delete fStatusEntry;
358     fStatusEntry = 0;
359   }
360
361   AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), fCurrentRun, fCurrentRun);
362
363   fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
364
365   UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
366
367   if (!result)
368   {
369     AliError(Form("WriteShuttleStatus for %s, run %d failed", fCurrentDetector.Data(), fCurrentRun));
370     return kFALSE;
371   }
372
373   return kTRUE;
374 }
375
376 //______________________________________________________________________________________________
377 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
378 {
379   // changes the AliShuttleStatus for the given detector and run to the given status
380
381   if (!fStatusEntry)
382   {
383     AliError("UNEXPECTED: fStatusEntry empty");
384     return;
385   }
386
387   TObject* anObject = fStatusEntry->GetObject();
388   AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (anObject);
389
390   if (!status)
391   {
392     AliError("UNEXPECTED: status could not be read from current CDB entry");
393     return;
394   }
395
396   Log("SHUTTLE", Form("%s: Changing state from %s to %s", fCurrentDetector.Data(),
397                                 status->GetStatusName(), status->GetStatusName(newStatus)));
398
399   status->SetStatus(newStatus);
400   if (increaseCount)
401     status->IncreaseCount();
402
403   AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
404 }
405
406 //______________________________________________________________________________________________
407 Bool_t AliShuttle::ContinueProcessing()
408 {
409   // this function reads the AliShuttleStatus information from CDB and
410   // checks if the processing should be continued
411   // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
412
413   AliShuttleStatus* status = ReadShuttleStatus();
414   if (!status)
415   {
416     // first time
417
418     Log("SHUTTLE", Form("%s: Processing first time.", fCurrentDetector.Data()));
419     status = new AliShuttleStatus(AliShuttleStatus::kStarted);
420     return WriteShuttleStatus(status);
421   }
422
423   if (status->GetStatus() == AliShuttleStatus::kDone)
424   {
425     Log("SHUTTLE", Form("%s already done for run %d", fCurrentDetector.Data(), fCurrentRun));
426     return kFALSE;
427   }
428
429   if (status->GetStatus() == AliShuttleStatus::kFailed)
430   {
431     Log("SHUTTLE", Form("%s already in failed state for run %d", fCurrentDetector.Data(), fCurrentRun));
432     return kFALSE;
433   }
434
435   // TODO what to do in case of storage error? currently it does not do anything
436   if (status->GetStatus() == AliShuttleStatus::kStoreFailed)
437   {
438     Log("SHUTTLE", Form("%s: Grid storage failed at least once for run %d", fCurrentDetector.Data(), fCurrentRun));
439     return kFALSE;
440   }
441
442   // if we get here, there is a restart
443
444   // abort conditions
445   if (status->GetStatus() == AliShuttleStatus::kPPStarted && status->GetCount() >= fConfig->GetMaxPPRetries() ||
446       status->GetCount() >= fConfig->GetMaxRetries())
447   {
448     Log("SHUTTLE", Form("%s, run %d failed too often, %d times, status %s. Skipping processing.",
449                 fCurrentDetector.Data(), fCurrentRun, status->GetCount(), status->GetStatusName()));
450
451     return kFALSE;
452   }
453
454   Log("SHUTTLE", Form("Restart of %s, run %d. Got stuck before in %s, count %d",
455                 fCurrentDetector.Data(), fCurrentRun, status->GetStatusName(), status->GetCount()));
456
457   UpdateShuttleStatus(AliShuttleStatus::kStarted, kTRUE);
458
459   return kTRUE;
460 }
461
462 //______________________________________________________________________________________________
463 Bool_t AliShuttle::Process(Int_t run, UInt_t startTime, UInt_t endTime)
464 {
465         //
466         // Makes data retrieval for all detectors in the configuration.
467         // run: is the run number used
468         // startTime: is the run start time
469         // endTime: is the run end time
470         // Returns kFALSE in case of error occured and kTRUE otherwise
471         //
472
473         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^* \n", run));
474
475         // Initialization
476         Bool_t hasError = kFALSE;
477         for(Int_t iSys=0;iSys<3;iSys++) fFESCalled[iSys]=kFALSE;
478
479         fCurrentRun = run;
480         fCurrentStartTime = startTime;
481         fCurrentEndTime = endTime;
482
483         // Loop on detectors in the configuration
484         TIter iter(fConfig->GetDetectors());
485         TObjString* aDetector;
486
487         while ((aDetector = (TObjString*) iter.Next())) {
488                 fCurrentDetector = aDetector->String();
489
490                 Bool_t detectorError=kFALSE;
491                 if (!fConfig->HostProcessDetector(fCurrentDetector)) continue;
492
493                 if (ContinueProcessing() == kFALSE) continue;
494
495                 AliInfo(Form("\n\n \t\t\t****** %s: START  ******", aDetector->GetName()));
496
497                 if(!Process()) {
498                         hasError = kTRUE;
499                         detectorError=kTRUE;
500                         continue;
501                 }
502                 AliInfo(Form("\n \t\t\t****** %s: FINISH ****** \n\n", aDetector->GetName()));
503
504                 // Process successful: Update time_processed field in FES logbooks!
505                 if(fFESCalled[kDAQ]) {
506                         hasError = (UpdateDAQTable() == kFALSE);
507                         fFESlist[kDAQ].Clear();
508                 }
509                 //if(fFESCalled[kDCS]) {
510                 //      hasError = UpdateDCSTable(aDetector->GetName());
511                 //      fFESlist[kDCS].Clear();
512                 //}
513                 //if(fFESCalled[kHLT]) {
514                 //      hasError = UpdateHLTTable(aDetector->GetName());
515                 //      fFESlist[kHLT].Clear();
516                 //}
517
518                 // UpdateShuttleStatus(AliShuttleStatus::kDone);
519         }
520
521         fCurrentRun = -1;
522         fCurrentStartTime = 0;
523         fCurrentEndTime = 0;
524
525         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^* \n", run));
526
527         return hasError == kFALSE;
528 }
529
530 //______________________________________________________________________________________________
531 Bool_t AliShuttle::Process()
532 {
533         //
534         // Makes data retrieval just for one specific detector.
535         // Threre should be a configuration for this detector.
536         // run: is the run number used
537         // startTime: is the run start time
538         // endTime: is the run end time
539         // detector: detector for which the retrieval will be made
540         // Returns kFALSE in case of error occured and kTRUE otherwise
541         //
542
543         AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), fCurrentRun));
544
545         if (!fConfig->HasDetector(fCurrentDetector)) {
546                 Log(fCurrentDetector, "There isn't any configuration for %s !");
547                 UpdateShuttleStatus(AliShuttleStatus::kFailed);
548                 return kFALSE;
549         }
550
551         UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
552
553         TString host(fConfig->GetDCSHost(fCurrentDetector));
554         Int_t port = fConfig->GetDCSPort(fCurrentDetector);
555
556         TIter iter(fConfig->GetDCSAliases(fCurrentDetector));
557         TObjString* anAlias;
558         TMap aliasMap;
559
560         Bool_t aDCSError = kFALSE;
561         fGridError = kFALSE;
562         UInt_t aPPResult;
563
564         while ((anAlias = (TObjString*) iter.Next())) {
565                 TObjArray valueSet;
566                 // TODO Test only... I've added a flag that allows to
567                 // exclude DCS archive DB query
568                 if(fgkProcessDCS){
569                         AliInfo("Querying DCS archive DB data...");
570                         aDCSError = (GetValueSet(host, port, anAlias->String(), valueSet) == 0);
571                 } else {
572                         AliInfo(Form("Skipping DCS processing. Port = %d",port));
573                         aDCSError = kFALSE;
574                 }
575                 if(!aDCSError) {
576                         aliasMap.Add(anAlias->Clone(), valueSet.Clone());
577                 }else{
578                         TString message = Form("Error while retrieving alias %s !",
579                                         anAlias->GetName());
580                         Log(fCurrentDetector, message.Data());
581                         aDCSError = kTRUE;
582                         break;
583                 }
584         }
585
586         if (aDCSError)
587         {
588                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
589                 return kFALSE;
590         }
591
592         UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
593
594         AliPreprocessor* aPreprocessor =
595                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
596         if(aPreprocessor)
597         {
598                 aPreprocessor->Initialize(fCurrentRun, fCurrentStartTime, fCurrentEndTime);
599                 aPPResult = aPreprocessor->Process(&aliasMap);
600         }else{
601     // TODO default behaviour?
602                 AliInfo(Form("No Preprocessor for %s: storing TMap of DP arrays into CDB!", fCurrentDetector.Data()));
603                 AliCDBMetaData metaData;
604                 AliDCSValue dcsValue(fCurrentStartTime, fCurrentEndTime);
605                 metaData.SetResponsible(Form("Duck, Donald"));
606                 metaData.SetProperty("StartEndTime", &dcsValue);
607                 metaData.SetComment("Automatically stored by Shuttle!");
608                 AliCDBPath path(fCurrentDetector,"DCS","Data");
609                 aPPResult = Store(path, &aliasMap, &metaData);
610         }
611
612         if (aPPResult == 0) { // Preprocessor error
613                 UpdateShuttleStatus(AliShuttleStatus::kPPError);
614         } else if (fGridError == kFALSE) { // process and Grid storage ok!
615                 UpdateShuttleStatus(AliShuttleStatus::kDone);
616         } else { // Grid storage error (process ok, but object put in local storage)
617                 UpdateShuttleStatus(AliShuttleStatus::kStoreFailed);
618         }
619
620         aliasMap.Delete();
621
622         return (aPPResult > 0);
623 }
624
625 //______________________________________________________________________________________________
626 Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* alias,
627                                 TObjArray& valueSet)
628 {
629 // Retrieve all "alias" data points from the DCS server
630 // host, port: TSocket connection parameters
631 // alias: name of the alias
632 // valueSet: array of retrieved AliDCSValue's 
633
634         AliDCSClient client(host, port, fTimeout, fRetries);
635         if (!client.IsConnected()) {
636                 return kFALSE;
637         }
638
639         Int_t result = client.GetAliasValues(alias,
640                 GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
641
642         if (result < 0) {
643                 AliError(Form("Can't get '%s'! Reason: %s",
644                         alias, AliDCSClient::GetErrorString(result)));
645
646                 if (result == AliDCSClient::fgkServerError) {
647                         AliError(Form("Server error: %s",
648                                 client.GetServerError().Data()));
649                 }
650
651                 return kFALSE;
652         }
653
654         return kTRUE;
655 }
656
657 //______________________________________________________________________________________________
658 const char* AliShuttle::GetFile(Int_t system, const char* detector,
659                 const char* id, const char* source)
660 {
661 // Get calibration file from file exchange servers
662 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
663
664         switch(system){
665                 case kDAQ:
666                         return GetDAQFileName(detector, id, source);
667                         break;
668                 case kDCS:
669                         return GetDCSFileName(detector, id, source);
670                         break;
671                 case kHLT:
672                         return GetHLTFileName(detector, id, source);
673                         break;
674                 default:
675                         AliError(Form("No valid system index: %d",system));
676         }
677
678         return 0;
679 }
680
681 //______________________________________________________________________________________________
682 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
683 {
684 // Get sources producing the condition file Id from file exchange servers
685 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
686
687         switch(system){
688                 case kDAQ:
689                         return GetDAQFileSources(detector, id);
690                         break;
691                 case kDCS:
692                         return GetDCSFileSources(detector, id);
693                         break;
694                 case kHLT:
695                         return GetHLTFileSources(detector, id);
696                         break;
697                 default:
698                         AliError(Form("No valid system index: %d",system));
699         }
700
701         return NULL;
702 }
703
704 //______________________________________________________________________________________________
705 Bool_t AliShuttle::Connect(Int_t system){
706 // Connect to MySQL Server of the system's FES logbook
707
708         // check connection: if already connected return
709         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
710
711         TString aFESlbHost= Form("mysql://%s", fConfig->GetFESlbHost(system));
712
713         fServer[system] = TSQLServer::Connect(aFESlbHost,
714                         fConfig->GetFESlbUser(system),
715                         fConfig->GetFESlbPass(system));
716         if (!fServer[system] || !fServer[system]->IsConnected()) {
717                 AliError(Form("Can't establish connection to FES logbook for %s !",fkSystemNames[system]));
718                 return kFALSE;
719         }
720
721         // Get tables
722         // TODO in the configuration should the table name be there too?
723         switch(system){
724                 case kDAQ:
725                         fServer[kDAQ]->GetTables("REFSYSLOG");
726                         break;
727                 case kDCS:
728                         //fServer[kDCS]->GetTables("REFSYSLOG");
729                         break;
730                 case kHLT:
731                         //fServer[kHLT]->GetTables("REFSYSLOG");
732                         break;
733                 default:
734                         break;
735         }
736
737         return kTRUE;
738 }
739
740 //______________________________________________________________________________________________
741 const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source){
742 // Retrieves a file from the DAQ FES.
743 // First queris the DAQ logbook_fs for the DAQ file name, using the run, detector, id and source info
744 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
745 // run: current run being processed (fCurrentRun)
746 // detector: comes from the Preprocessor name (must be converted into detector code with GetDetCode)
747 // id: provided as a parameter by the Preprocessor
748 // source: provided by the Preprocessor through GetFileSources function
749
750         // check connection, in case connect
751         if(!Connect(kDAQ)){
752                 Log(detector, "GetDAQFileName: Couldn't connect to DAQ Logbook !");
753                 return 0;
754         }
755
756         // Query preparation
757         TString sqlQueryStart = "select filePath from logbook_fs where";
758         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
759                                 fCurrentRun, GetDetCode(detector), id, source);
760         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
761
762         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
763
764         // Query execution
765         TSQLResult* aResult;
766         aResult = fServer[kDAQ]->Query(sqlQuery);
767         if (!aResult) {
768                 Log(detector, Form("Can't execute query <%s>!", sqlQuery.Data()));
769                 return 0;
770         }
771
772         if (aResult->GetRowCount() == 0) {
773                 Log(detector,
774                         Form("GetDAQFileName: No result from SQL query <%s>!", sqlQuery.Data()));
775                 delete aResult;
776                 return 0;
777         }
778
779         if (aResult->GetRowCount() >1) {
780                 Log(detector,
781                         Form("GetDAQFileName: More than one row resulting from SQL query <%s>!", sqlQuery.Data()));
782                 delete aResult;
783                 return 0;
784         }
785
786         TSQLRow* aRow = aResult->Next();
787
788         if(!aRow){
789                 Log(detector, Form("GetDAQFileName: Empty set result from query <%s>!", sqlQuery.Data()));
790                 delete aResult;
791                 return 0;
792         }
793
794         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
795
796         delete aResult;
797
798         AliDebug(2, Form("filePath = %s",filePath.Data()));
799
800         // retrieved file is renamed to make it unique
801         TString localFileName = Form("%s_%d_%s_%s.shuttle",
802                                         detector, fCurrentRun, id, source);
803
804         // file retrieval from DAQ FES
805         Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
806         if(!result) {
807                 Log(detector, Form("copying file %s from DAQ FES failed!", filePath.Data()));
808                 return 0;
809         } else {
810                 AliInfo(Form("File %s copied from DAQ FES into %s/%s !",
811                         filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
812         }
813
814
815         fFESCalled[kDAQ]=kTRUE;
816         TObjString *fileParams = new TObjString(Form("%s_!?!_%s", id, source));
817         fFESlist[kDAQ].Add(fileParams);
818
819         return localFileName.Data();
820
821 }
822
823 //______________________________________________________________________________________________
824 Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName){
825
826         // check temp directory: trying to cd to temp; if it does not exist, create it
827         AliDebug(2, Form("Copy file %s from DAQ FES into folder %s and rename it as %s",
828                         daqFileName,fgkShuttleTempDir, localFileName));
829
830         void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
831         if (dir == NULL) {
832                 if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
833                         AliError(Form("Can't open directory <%s>!", fgkShuttleTempDir));
834                         return kFALSE;
835                 }
836
837         } else {
838                 gSystem->FreeDirectory(dir);
839         }
840
841         TString baseDAQFESFolder = "DAQ";
842         TString command = Form("scp %s@%s:%s/%s %s/%s",
843                 fConfig->GetFESUser(kDAQ),
844                 fConfig->GetFESHost(kDAQ),
845                 baseDAQFESFolder.Data(),
846                 daqFileName,
847                 fgkShuttleTempDir,
848                 localFileName);
849
850         AliDebug(2, Form("%s",command.Data()));
851
852         UInt_t nRetries = 0;
853         UInt_t maxRetries = 3;
854
855         // copy!! if successful TSystem::Exec returns 0
856         while(nRetries++ < maxRetries) {
857                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
858                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
859         }
860
861         return kFALSE;
862
863 }
864
865 //______________________________________________________________________________________________
866 TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id){
867 // Retrieves a file from the DCS FES.
868
869         // check connection, in case connect
870         if(!Connect(kDAQ)){
871                 Log(detector, "GetDAQFileName: Couldn't connect to DAQ Logbook !");
872                 return 0;
873         }
874
875         // Query preparation
876         TString sqlQueryStart = "select DAQsource from logbook_fs where";
877         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
878                                 fCurrentRun, GetDetCode(detector), id);
879         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
880
881         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
882
883         // Query execution
884         TSQLResult* aResult;
885         aResult = fServer[kDAQ]->Query(sqlQuery);
886         if (!aResult) {
887                 Log(detector, Form("GetDAQFileSources: Can't execute query <%s>!", sqlQuery.Data()));
888                 return 0;
889         }
890
891         if (aResult->GetRowCount() == 0) {
892                 Log(detector,
893                         Form("GetDAQFileSources: No result from SQL query <%s>!", sqlQuery.Data()));
894                 delete aResult;
895                 return 0;
896         }
897
898         TSQLRow* aRow;
899         TList *list = new TList();
900         list->SetOwner(1);
901
902         while((aRow = aResult->Next())){
903
904                 TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
905                 AliDebug(2, Form("daqSource = %s", daqSource.Data()));
906                 list->Add(new TObjString(daqSource));
907         }
908         delete aResult;
909
910         return list;
911
912 }
913
914 //______________________________________________________________________________________________
915 Bool_t AliShuttle::UpdateDAQTable(){
916 // Update DAQ table filling time_processed field in all rows corresponding to current run and detector
917
918         // check connection, in case connect
919         if(!Connect(kDAQ)){
920                 Log(fCurrentDetector, "UpdateDAQTable: Couldn't connect to DAQ Logbook !");
921                 return kFALSE;
922         }
923
924         TTimeStamp now; // now
925
926         // Loop on FES list entries
927         TIter iter(&fFESlist[kDAQ]);
928         TObjString *aFESentry=0;
929         while((aFESentry = dynamic_cast<TObjString*> (iter.Next()))){
930                 TString aFESentrystr = aFESentry->String();
931                 TObjArray *aFESarray = aFESentrystr.Tokenize("_!?!_");
932                 if(!aFESarray || aFESarray->GetEntries() != 2 ) {
933                         Log(fCurrentDetector,Form("UpdateDAQTable: error updating FES entry! string = %s",
934                                 aFESentrystr.Data()));
935                         if(aFESarray) delete aFESarray;
936                         return kFALSE;
937                 }
938                 const char* fileId = ((TObjString*) aFESarray->At(0))->GetName();
939                 const char* daqSource = ((TObjString*) aFESarray->At(1))->GetName();
940                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
941                         fCurrentRun,GetDetCode(fCurrentDetector), fileId, daqSource);
942
943                 delete aFESarray;
944
945                 TString sqlQuery = Form("update logbook_fs set time_processed=%d %s", now.GetSec(), whereClause.Data());
946
947                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
948
949                 // Query execution
950                 TSQLResult* aResult;
951                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
952                 if (!aResult) {
953                         Log(fCurrentDetector, Form("UpdateDAQTable: Can't execute query <%s>!", sqlQuery.Data()));
954                         return kFALSE;
955                 }
956                 delete aResult;
957
958                 // check result - TODO Is it necessary?
959                 sqlQuery = Form("select time_processed from logbook_fs %s", whereClause.Data());
960                 AliDebug(2, Form(" CHECK - SQL query: \n%s",sqlQuery.Data()));
961
962                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
963                 if (!aResult) {
964                         AliWarning("Can't check result!");
965                         continue;
966                 }
967
968         if (aResult->GetRowCount() == 0) {
969                 Log(fCurrentDetector,
970                         Form("GetDAQFileName: No result from SQL query <%s>!", sqlQuery.Data()));
971                 delete aResult;
972                 //return 0;
973         }
974
975         if (aResult->GetRowCount() >1) {
976                 Log(fCurrentDetector,
977                         Form("GetDAQFileName: More than one row resulting from SQL query <%s>!", sqlQuery.Data()));
978                 delete aResult;
979                 //return 0;
980         }
981
982                 TSQLRow *row = dynamic_cast<TSQLRow*> (aResult->Next());
983                 TString processedTimeString(row->GetField(0), row->GetFieldLength(0));
984                 Int_t processedTime = processedTimeString.Atoi();
985                 if(processedTime != now.GetSec()){
986                         Log(fCurrentDetector, Form("UpdateDAQTable: Update table error: processed_time=%d, now=%d !",
987                                 processedTime, now.GetSec()));
988                         delete aResult;
989                         return kFALSE;
990                 }
991
992                 delete aResult;
993
994         }
995
996         return kTRUE;
997 }
998
999 //______________________________________________________________________________________________
1000 const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1001 // Retrieves a file from the DCS FES.
1002
1003 return "You're in DCS";
1004
1005 }
1006
1007 //______________________________________________________________________________________________
1008 TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
1009 // Retrieves a file from the DCS FES.
1010
1011 return NULL;
1012
1013 }
1014
1015 //______________________________________________________________________________________________
1016 const char* AliShuttle::GetHLTFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1017 // Retrieves a file from the HLT FES.
1018
1019 return "You're in HLT";
1020
1021 }
1022
1023 //______________________________________________________________________________________________
1024 TList* AliShuttle::GetHLTFileSources(const char* /*detector*/, const char* /*id*/){
1025 // Retrieves a file from the HLT FES.
1026
1027 return NULL;
1028
1029 }
1030
1031 //______________________________________________________________________________________________
1032 const char* AliShuttle::GetDetCode(const char* detector){
1033 // Return detector code
1034
1035         for(int iDet=0; iDet < fgkNDetectors; iDet++){
1036                 if(!strcmp(fgkDetectorName[iDet], detector)) return fgkDetectorCode[iDet];
1037         }
1038
1039         return 0;
1040 }
1041
1042 //______________________________________________________________________________________________
1043 void AliShuttle::Log(const char* detector, const char* message)
1044 {
1045 // Fill log string with a message
1046
1047         void* dir = gSystem->OpenDirectory(fgkShuttleLogDir);
1048         if (dir == NULL) {
1049                 if (gSystem->mkdir(fgkShuttleLogDir, kTRUE)) {
1050                         AliError(Form("Can't open directory <%s>!", fgkShuttleTempDir));
1051                         return;
1052                 }
1053
1054         } else {
1055                 gSystem->FreeDirectory(dir);
1056         }
1057
1058         TString toLog = Form("%s: %s, run %d - %s", TTimeStamp(time(0)).AsString("s"),
1059         detector, GetCurrentRun(), message);
1060         AliInfo(toLog.Data());
1061
1062         TString fileName;
1063         fileName.Form("%s/%s.log", fgkShuttleLogDir, detector);
1064         gSystem->ExpandPathName(fileName);
1065
1066         ofstream logFile;
1067         logFile.open(fileName, ofstream::out | ofstream::app);
1068
1069         if (!logFile.is_open()) {
1070                 AliError(Form("Could not open file %s", fileName.Data()));
1071                 return;
1072         }
1073
1074         logFile << toLog.Data() << "\n";
1075
1076         logFile.close();
1077 }