last run is stored after each run
[u/mrichter/AliRoot.git] / SHUTTLE / AliShuttle.cxx
1 /**************************************************************************
2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
3  *                                                                        *
4  * Author: The ALICE Off-line Project.                                    *
5  * Contributors are mentioned in the code where appropriate.              *
6  *                                                                        *
7  * Permission to use, copy, modify and distribute this software and its   *
8  * documentation strictly for non-commercial purposes is hereby granted   *
9  * without fee, provided that the above copyright notice appears in all   *
10  * copies and that both the copyright notice and this permission notice   *
11  * appear in the supporting documentation. The authors make no claims     *
12  * about the suitability of this software for any purpose. It is          *
13  * provided "as is" without express or implied warranty.                  *
14  **************************************************************************/
15
16 /*
17 $Log$
18 Revision 1.10  2006/07/20 09:54:40  jgrosseo
19 introducing status management: The processing per subdetector is divided into several steps,
20 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
21 can keep track of the number of failures and skips further processing after a certain threshold is
22 exceeded. These thresholds can be configured in LDAP.
23
24 Revision 1.9  2006/07/19 10:09:55  jgrosseo
25 new configuration, accesst to DAQ FES (Alberto)
26
27 Revision 1.8  2006/07/11 12:44:36  jgrosseo
28 adding parameters for extended validity range of data produced by preprocessor
29
30 Revision 1.7  2006/07/10 14:37:09  jgrosseo
31 small fix + todo comment
32
33 Revision 1.6  2006/07/10 13:01:41  jgrosseo
34 enhanced storing of last sucessfully processed run (alberto)
35
36 Revision 1.5  2006/07/04 14:59:57  jgrosseo
37 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
38
39 Revision 1.4  2006/06/12 09:11:16  jgrosseo
40 coding conventions (Alberto)
41
42 Revision 1.3  2006/06/06 14:26:40  jgrosseo
43 o) removed files that were moved to STEER
44 o) shuttle updated to follow the new interface (Alberto)
45
46 Revision 1.2  2006/03/07 07:52:34  hristov
47 New version (B.Yordanov)
48
49 Revision 1.6  2005/11/19 17:19:14  byordano
50 RetrieveDATEEntries and RetrieveConditionsData added
51
52 Revision 1.5  2005/11/19 11:09:27  byordano
53 AliShuttle declaration added
54
55 Revision 1.4  2005/11/17 17:47:34  byordano
56 TList changed to TObjArray
57
58 Revision 1.3  2005/11/17 14:43:23  byordano
59 import to local CVS
60
61 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
62 Initial import as subdirectory in AliRoot
63
64 Revision 1.2  2005/09/13 08:41:15  byordano
65 default startTime endTime added
66
67 Revision 1.4  2005/08/30 09:13:02  byordano
68 some docs added
69
70 Revision 1.3  2005/08/29 21:15:47  byordano
71 some docs added
72
73 */
74
75 //
76 // This class is the main manager for AliShuttle. 
77 // It organizes the data retrieval from DCS and call the 
78 // interface methods of AliPreprocessor.
79 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
80 // data for its set of aliases is retrieved. If there is registered
81 // AliPreprocessor for this detector then it will be used
82 // accroding to the schema (see AliPreprocessor).
83 // If there isn't registered AliPreprocessor than the retrieved
84 // data is stored automatically to the undelying AliCDBStorage.
85 // For detSpec is used the alias name.
86 //
87
88 #include "AliShuttle.h"
89
90 #include "AliCDBManager.h"
91 #include "AliCDBStorage.h"
92 #include "AliCDBId.h"
93 #include "AliCDBEntry.h"
94 #include "AliShuttleConfig.h"
95 #include "AliDCSClient.h"
96 #include "AliLog.h"
97 #include "AliPreprocessor.h"
98 #include "AliDefaultPreprocessor.h"
99 #include "AliShuttleStatus.h"
100
101 #include <TSystem.h>
102 #include <TObject.h>
103 #include <TString.h>
104 #include <TTimeStamp.h>
105 #include <TObjString.h>
106 #include <TSQLServer.h>
107 #include <TSQLResult.h>
108 #include <TSQLRow.h>
109
110 #include <fstream>
111
112 ClassImp(AliShuttle)
113
114 TString AliShuttle::fgkLocalUri("local://$ALICE_ROOT/SHUTTLE/ShuttleCDB");
115 const char* AliShuttle::fgkShuttleTempDir = "$ALICE_ROOT/SHUTTLE/temp";
116 const char* AliShuttle::fgkShuttleLogDir = "$ALICE_ROOT/SHUTTLE/log";
117
118 const char* AliShuttle::fgkDetectorName[AliShuttle::fgkNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
119         "PHOS", "CPV", "RICH", "EMCAL", "MUON_TRK", "MUON_TRG", "FMD", "ZDC", "PMD", "START", "VZERO"};
120
121 const char* AliShuttle::fgkDetectorCode[AliShuttle::fgkNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
122         "PHS", "CPV", "HMP", "EMC", "MCH", "MTR", "FMD", "ZDC", "PMD", "T00", "V00"};
123
124 //______________________________________________________________________________________________
125 AliShuttle::AliShuttle(const AliShuttleConfig* config,
126                 UInt_t timeout, Int_t retries):
127         fConfig(config),
128         fTimeout(timeout),
129         fRetries(retries), fCurrentRun(-1), fCurrentStartTime(0),
130         fCurrentEndTime(0),
131   fStatusEntry(0)
132 {
133         //
134         // config: AliShuttleConfig used
135         // timeout: timeout used for AliDCSClient connection
136         // retries: the number of retries in case of connection error.
137         //
138
139         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
140         for(int iSys=0;iSys<3;iSys++) {
141                 fServer[iSys]=0;
142                 fFESlist[iSys].SetOwner(kTRUE);
143         }
144 }
145
146 //______________________________________________________________________
147 AliShuttle::AliShuttle(const AliShuttle& /*other*/):
148 AliShuttleInterface()
149 {
150 // copy constructor (not implemented)
151
152 }
153
154 //______________________________________________________________________
155 AliShuttle &AliShuttle::operator=(const AliShuttle& /*other*/)
156 {
157 // assignment operator (not implemented)
158
159 return *this;
160 }
161
162 //______________________________________________________________________________________________
163 AliShuttle::~AliShuttle()
164 {
165 // destructor
166
167         fPreprocessorMap.DeleteAll();
168         for(int iSys=0;iSys<3;iSys++)
169                 if(fServer[iSys]) {
170                         fServer[iSys]->Close();
171                         delete fServer[iSys];
172                 }
173 }
174
175 //______________________________________________________________________________________________
176 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
177 {
178         //
179         // Registers new AliPreprocessor.
180         // It uses GetName() for indentificator of the pre processor.
181         // The pre processor is registered it there isn't any other
182         // with the same identificator (GetName()).
183         //
184
185         if (fPreprocessorMap.GetValue(preprocessor->GetName())) {
186                 AliWarning(Form("AliPreprocessor %s is already registered!",
187                         preprocessor->GetName()));
188                 return;
189         }
190
191         fPreprocessorMap.Add(new TObjString(preprocessor->GetName()), preprocessor);
192 }
193
194 //______________________________________________________________________________________________
195 UInt_t AliShuttle::Store(const char* detector,
196                 TObject* object, AliCDBMetaData* metaData, Int_t /*validityStart*/, Bool_t /*validityInfinite*/)
197 {
198         // store data into CDB
199   //
200   // validityStart is the start validity of the data, if not 0 GetCurrentRun() - validityStart is taken
201   // validityInfinite defines if the data is valid until new data arrives (e.g. for calibration runs)
202   //
203         // returns 0 if fail
204         //         1 if stored in main (Grid) storage
205         //         2 if stored in backup (Local) storage
206
207   // TODO implement use of two parameters
208
209   // TODO shouldn't the path be given by the preprocessor???
210         AliCDBId id(AliCDBPath(detector, "DCS", "Data"),
211                 GetCurrentRun(), GetCurrentRun());
212
213         UInt_t result = 0;
214         if (!(AliCDBManager::Instance()->IsDefaultStorageSet())) {
215                 Log(detector, "No CDB storage set!");
216         } else {
217                 result = (UInt_t) AliCDBManager::Instance()->Put(object, id, metaData);
218         }
219         if(!result) {
220
221                 Log(detector, "Error while storing object in main storage!");
222                 AliError("local storage will be used!");
223
224                 result = AliCDBManager::Instance()->GetStorage(fgkLocalUri)
225                                         ->Put(object, id, metaData);
226
227                 if(result) {
228                         result = 2;
229                 }else{
230                         Log(detector, "Can't store data!");
231                 }
232         }
233         return result;
234
235 }
236
237 //______________________________________________________________________________________________
238 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
239 {
240   // Reads the AliShuttleStatus from the CDB
241
242   if (fStatusEntry)
243   {
244     delete fStatusEntry;
245     fStatusEntry = 0;
246   }
247
248   fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalURI())
249       ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), fCurrentRun);
250
251   if (!fStatusEntry)
252     return 0;
253
254   TObject* anObject = fStatusEntry->GetObject();
255   if (anObject == NULL || anObject->IsA() != AliShuttleStatus::Class())
256   {
257     AliError("Invalid object stored to CDB!");
258     return 0;
259   }
260
261   AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (anObject);
262   return status;
263 }
264
265 //______________________________________________________________________________________________
266 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
267 {
268   // writes the status for one subdetector
269
270   if (fStatusEntry)
271   {
272     delete fStatusEntry;
273     fStatusEntry = 0;
274   }
275
276   AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), fCurrentRun, fCurrentRun);
277
278   fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
279
280   UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalUri)->Put(fStatusEntry);
281
282   if (!result)
283   {
284     AliError(Form("WriteShuttleStatus for %s, run %d failed", fCurrentDetector.Data(), fCurrentRun));
285     return kFALSE;
286   }
287
288   return kTRUE;
289 }
290
291 //______________________________________________________________________________________________
292 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
293 {
294   // changes the AliShuttleStatus for the given detector and run to the given status
295
296   if (!fStatusEntry)
297   {
298     AliError("UNEXPECTED: fStatusEntry empty");
299     return;
300   }
301
302   TObject* anObject = fStatusEntry->GetObject();
303   AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (anObject);
304
305   if (!status)
306   {
307     AliError("UNEXPECTED: status could not be read from current CDB entry");
308     return;
309   }
310
311   Log("SHUTTLE", Form("%s: Changing state from %s to %s", fCurrentDetector.Data(),
312                                 status->GetStatusName(), status->GetStatusName(newStatus)));
313
314   status->SetStatus(newStatus);
315   if (increaseCount)
316     status->IncreaseCount();
317
318   AliCDBManager::Instance()->GetStorage(fgkLocalUri)->Put(fStatusEntry);
319 }
320
321 //______________________________________________________________________________________________
322 Bool_t AliShuttle::ContinueProcessing()
323 {
324   // this function reads the AliShuttleStatus information from CDB and
325   // checks if the processing should be continued
326   // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
327
328   AliShuttleStatus* status = ReadShuttleStatus();
329   if (!status)
330   {
331     // first time
332
333     Log("SHUTTLE", Form("%s: Processing first time.", fCurrentDetector.Data()));
334     status = new AliShuttleStatus(AliShuttleStatus::kStarted);
335     return WriteShuttleStatus(status);
336   }
337
338   if (status->GetStatus() == AliShuttleStatus::kDone)
339   {
340     Log("SHUTTLE", Form("%s already done for run %d", fCurrentDetector.Data(), fCurrentRun));
341     return kFALSE;
342   }
343
344   if (status->GetStatus() == AliShuttleStatus::kFailed)
345   {
346     Log("SHUTTLE", Form("%s already in failed state for run %d", fCurrentDetector.Data(), fCurrentRun));
347     return kFALSE;
348   }
349
350   // if we get here, there is a restart
351
352   // abort conditions
353   if (status->GetStatus() == AliShuttleStatus::kPPStarted && status->GetCount() >= fConfig->GetMaxPPRetries() ||
354       status->GetCount() >= fConfig->GetMaxRetries())
355   {
356     Log("SHUTTLE", Form("%s, run %d failed too often, %d times, status %s. Skipping processing.",
357                 fCurrentDetector.Data(), fCurrentRun, status->GetCount(), status->GetStatusName()));
358
359     return kFALSE;
360   }
361
362   Log("SHUTTLE", Form("Restart of %s, run %d. Got stuck before in %s, count %d",
363                 fCurrentDetector.Data(), fCurrentRun, status->GetStatusName(), status->GetCount()));
364
365   UpdateShuttleStatus(AliShuttleStatus::kStarted, kTRUE);
366
367   return kTRUE;
368 }
369
370 //______________________________________________________________________________________________
371 Bool_t AliShuttle::Process(Int_t run, UInt_t startTime, UInt_t endTime)
372 {
373         //
374         // Makes data retrieval for all detectors in the configuration.
375         // run: is the run number used
376         // startTime: is the run start time
377         // endTime: is the run end time
378         // Returns kFALSE in case of error occured and kTRUE otherwise
379         //
380
381         AliInfo(Form("\n\n ^*^*^*^*^*^* Processing run %d ^*^*^*^*^*^*", run));
382
383         // Initialization
384         Bool_t hasError = kFALSE;
385         for(Int_t iSys=0;iSys<3;iSys++) fFESCalled[iSys]=kFALSE;
386
387         fCurrentRun = run;
388         fCurrentStartTime = startTime;
389         fCurrentEndTime = endTime;
390
391         // Loop on detectors in the configuration
392         TIter iter(fConfig->GetDetectors());
393         TObjString* aDetector;
394
395         while ((aDetector = (TObjString*) iter.Next())) {
396                 fCurrentDetector = aDetector->String();
397
398                 Bool_t detectorError=kFALSE;
399                 if (!fConfig->HostProcessDetector(fCurrentDetector)) continue;
400
401                 if (ContinueProcessing() == kFALSE) continue;
402
403                 if(!Process()) {
404                         hasError = kTRUE;
405                         detectorError=kTRUE;
406                         continue;
407                 }
408                 AliInfo(Form("Process ended successfully for detector %s!",aDetector->GetName()));
409
410                 // Process successful: Update time_processed field in FES logbooks!
411                 if(fFESCalled[kDAQ]) {
412                         hasError = (UpdateDAQTable() == kFALSE);
413                         fFESlist[kDAQ].Clear();
414                 }
415                 //if(fFESCalled[kDCS]) {
416                 //      hasError = UpdateDCSTable(aDetector->GetName());
417                 //      fFESlist[kDCS].Clear();
418                 //}
419                 //if(fFESCalled[kHLT]) {
420                 //      hasError = UpdateHLTTable(aDetector->GetName());
421                 //      fFESlist[kHLT].Clear();
422                 //}
423
424                 UpdateShuttleStatus(AliShuttleStatus::kDone);
425         }
426
427         fCurrentRun = -1;
428         fCurrentStartTime = 0;
429         fCurrentEndTime = 0;
430
431         return hasError == kFALSE;
432 }
433
434 //______________________________________________________________________________________________
435 Bool_t AliShuttle::Process()
436 {
437         //
438         // Makes data retrieval just for one specific detector.
439         // Threre should be a configuration for this detector.
440         // run: is the run number used
441         // startTime: is the run start time
442         // endTime: is the run end time
443         // detector: detector for which the retrieval will be made
444         // Returns kFALSE in case of error occured and kTRUE otherwise
445         //
446
447         AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), fCurrentRun));
448
449         if (!fConfig->HasDetector(fCurrentDetector)) {
450                 Log(fCurrentDetector, "There isn't any configuration for %s !");
451                 UpdateShuttleStatus(AliShuttleStatus::kFailed);
452                 return kFALSE;
453         }
454
455         UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
456
457         TString host(fConfig->GetDCSHost(fCurrentDetector));
458         Int_t port = fConfig->GetDCSPort(fCurrentDetector);
459
460         TIter iter(fConfig->GetDCSAliases(fCurrentDetector));
461         TObjString* anAlias;
462         TMap aliasMap;
463
464         Bool_t hasError = kFALSE;
465         Bool_t result=kFALSE;
466
467         while ((anAlias = (TObjString*) iter.Next())) {
468                 TObjArray valueSet;
469                 result = GetValueSet(host, port, anAlias->String(), valueSet);
470                 //AliInfo(Form("Port = %d",port));
471                 //result = kTRUE;
472                 if(result) {
473                         aliasMap.Add(anAlias->Clone(), valueSet.Clone());
474                 }else{
475                         TString message = Form("Error while retrieving alias %s !",
476                                         anAlias->GetName());
477                         Log(fCurrentDetector, message.Data());
478                         hasError = kTRUE;
479                 }
480         }
481
482         if (hasError)
483         {
484                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
485                 return kFALSE;
486         }
487
488   UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
489
490   AliPreprocessor* aPreprocessor =
491                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
492         if(aPreprocessor)
493         {
494                 aPreprocessor->Initialize(fCurrentRun, fCurrentStartTime, fCurrentEndTime);
495                 hasError = (aPreprocessor->Process(&aliasMap) == 0);
496         }else{
497     // TODO default behaviour?
498                 AliInfo(Form("No Preprocessor for %s: storing TMap of DP arrays into CDB!", fCurrentDetector.Data()));
499                 AliCDBMetaData metaData;
500                 AliDCSValue dcsValue(fCurrentStartTime, fCurrentEndTime);
501                 metaData.SetResponsible(Form("Duck, Donald"));
502                 metaData.SetProperty("StartEndTime", &dcsValue);
503                 metaData.SetComment("Automatically stored by Shuttle!");
504                 hasError = (Store(fCurrentDetector, &aliasMap, &metaData) == 0);
505         }
506
507   if (hasError)
508     UpdateShuttleStatus(AliShuttleStatus::kPPError);
509   else
510     UpdateShuttleStatus(AliShuttleStatus::kPPDone);
511
512         aliasMap.Delete();
513
514         return hasError == kFALSE;
515 }
516
517 //______________________________________________________________________________________________
518 Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* alias,
519                                 TObjArray& valueSet)
520 {
521 // Retrieve all "alias" data points from the DCS server
522 // host, port: TSocket connection parameters
523 // alias: name of the alias
524 // valueSet: array of retrieved AliDCSValue's 
525
526         AliDCSClient client(host, port, fTimeout, fRetries);
527         if (!client.IsConnected()) {
528                 return kFALSE;
529         }
530
531         Int_t result = client.GetAliasValues(alias,
532                 GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
533
534         if (result < 0) {
535                 AliError(Form("Can't get '%s'! Reason: %s",
536                         alias, AliDCSClient::GetErrorString(result)));
537
538                 if (result == AliDCSClient::fgkServerError) {
539                         AliError(Form("Server error: %s",
540                                 client.GetServerError().Data()));
541                 }
542
543                 return kFALSE;
544         }
545
546         return kTRUE;
547 }
548
549 //______________________________________________________________________________________________
550 const char* AliShuttle::GetFile(Int_t system, const char* detector,
551                 const char* id, const char* source)
552 {
553 // Get calibration file from file exchange servers
554 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
555
556         switch(system){
557                 case kDAQ:
558                         return GetDAQFileName(detector, id, source);
559                         break;
560                 case kDCS:
561                         return GetDCSFileName(detector, id, source);
562                         break;
563                 case kHLT:
564                         return GetHLTFileName(detector, id, source);
565                         break;
566                 default:
567                         AliError(Form("No valid system index: %d",system));
568         }
569
570         return 0;
571 }
572
573 //______________________________________________________________________________________________
574 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
575 {
576 // Get sources producing the condition file Id from file exchange servers
577 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
578
579         switch(system){
580                 case kDAQ:
581                         return GetDAQFileSources(detector, id);
582                         break;
583                 case kDCS:
584                         return GetDCSFileSources(detector, id);
585                         break;
586                 case kHLT:
587                         return GetHLTFileSources(detector, id);
588                         break;
589                 default:
590                         AliError(Form("No valid system index: %d",system));
591         }
592
593         return NULL;
594 }
595
596 //______________________________________________________________________________________________
597 Bool_t AliShuttle::Connect(Int_t system){
598 // Connect to MySQL Server of the system's FES logbook
599
600         // check connection: if already connected return
601         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
602
603         TString aFESlbHost= Form("mysql://%s", fConfig->GetFESlbHost(system));
604
605         fServer[system] = TSQLServer::Connect(aFESlbHost,
606                         fConfig->GetFESlbUser(system),
607                         fConfig->GetFESlbPass(system));
608         if (!fServer[system] || !fServer[system]->IsConnected()) {
609                 AliError(Form("Can't establish connection to FES logbook for %s !",fkSystemNames[system]));
610                 return kFALSE;
611         }
612
613         // Get tables
614         // TODO in the configuration should the table name be there too?
615         switch(system){
616                 case kDAQ:
617                         fServer[kDAQ]->GetTables("REFSYSLOG");
618                         break;
619                 case kDCS:
620                         //fServer[kDCS]->GetTables("REFSYSLOG");
621                         break;
622                 case kHLT:
623                         //fServer[kHLT]->GetTables("REFSYSLOG");
624                         break;
625                 default:
626                         break;
627         }
628
629         return kTRUE;
630 }
631
632 //______________________________________________________________________________________________
633 const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source){
634 // Retrieves a file from the DAQ FES.
635 // First queris the DAQ logbook_fs for the DAQ file name, using the run, detector, id and source info
636 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
637 // run: current run being processed (fCurrentRun)
638 // detector: comes from the Preprocessor name (must be converted into detector code with GetDetCode)
639 // id: provided as a parameter by the Preprocessor
640 // source: provided by the Preprocessor through GetFileSources function
641
642         // check connection, in case connect
643         if(!Connect(kDAQ)){
644                 Log(detector, "GetDAQFileName: Couldn't connect to DAQ Logbook !");
645                 return 0;
646         }
647
648         // Query preparation
649         TString sqlQueryStart = "select filePath from logbook_fs where";
650         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
651                                 fCurrentRun, GetDetCode(detector), id, source);
652         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
653
654         AliInfo(Form("SQL query: \n%s",sqlQuery.Data()));
655
656         // Query execution
657         TSQLResult* aResult;
658         aResult = fServer[kDAQ]->Query(sqlQuery);
659         if (!aResult) {
660                 Log(detector, Form("Can't execute query <%s>!", sqlQuery.Data()));
661                 return 0;
662         }
663
664         if (aResult->GetRowCount() == 0) {
665                 Log(detector,
666                         Form("GetDAQFileName: No result from SQL query <%s>!", sqlQuery.Data()));
667                 delete aResult;
668                 return 0;
669         }
670
671         if (aResult->GetRowCount() >1) {
672                 Log(detector,
673                         Form("GetDAQFileName: More than one row resulting from SQL query <%s>!", sqlQuery.Data()));
674                 delete aResult;
675                 return 0;
676         }
677
678         TSQLRow* aRow = aResult->Next();
679
680         if(!aRow){
681                 Log(detector, Form("GetDAQFileName: Empty set result from query <%s>!", sqlQuery.Data()));
682                 delete aResult;
683                 return 0;
684         }
685
686         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
687
688         delete aResult;
689
690         AliInfo(Form("filePath = %s",filePath.Data()));
691
692         // retrieved file is renamed to make it unique
693         TString localFileName = Form("%s_%d_%s_%s.shuttle",
694                                         detector, fCurrentRun, id, source);
695
696         // file retrieval from DAQ FES
697         Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
698         if(!result) {
699                 Log(detector, Form("copying file %s from DAQ FES failed!", filePath.Data()));
700                 return 0;
701         } else {
702                 AliInfo(Form("File %s copied from DAQ FES into %s/%s !",
703                         filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
704         }
705
706
707         fFESCalled[kDAQ]=kTRUE;
708         TObjString *fileParams = new TObjString(Form("%s_!?!_%s", id, source));
709         fFESlist[kDAQ].Add(fileParams);
710
711         return localFileName.Data();
712
713 }
714
715 //______________________________________________________________________________________________
716 Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName){
717
718         // check temp directory: trying to cd to temp; if it does not exist, create it
719         AliInfo(Form("Copy file %s from DAQ FES into folder %s and rename it as %s",
720                         daqFileName,fgkShuttleTempDir, localFileName));
721
722         void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
723         if (dir == NULL) {
724                 if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
725                         AliError(Form("Can't open directory <%s>!", fgkShuttleTempDir));
726                         return kFALSE;
727                 }
728
729         } else {
730                 gSystem->FreeDirectory(dir);
731         }
732
733         TString baseDAQFESFolder = "DAQ";
734         TString command = Form("scp %s@%s:%s/%s %s/%s",
735                 fConfig->GetFESUser(kDAQ),
736                 fConfig->GetFESHost(kDAQ),
737                 baseDAQFESFolder.Data(),
738                 daqFileName,
739                 fgkShuttleTempDir,
740                 localFileName);
741
742         AliInfo(Form("%s",command.Data()));
743
744         UInt_t nRetries = 0;
745         UInt_t maxRetries = 3;
746
747         // copy!! if successful TSystem::Exec returns 0
748         while(nRetries++ < maxRetries) {
749                 AliInfo(Form("Trying to copy file. Retry # %d", nRetries));
750                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
751         }
752
753         return kFALSE;
754
755 }
756
757 //______________________________________________________________________________________________
758 TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id){
759 // Retrieves a file from the DCS FES.
760
761         // check connection, in case connect
762         if(!Connect(kDAQ)){
763                 Log(detector, "GetDAQFileName: Couldn't connect to DAQ Logbook !");
764                 return 0;
765         }
766
767         // Query preparation
768         TString sqlQueryStart = "select DAQsource from logbook_fs where";
769         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
770                                 fCurrentRun, GetDetCode(detector), id);
771         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
772
773         AliInfo(Form("SQL query: \n%s",sqlQuery.Data()));
774
775         // Query execution
776         TSQLResult* aResult;
777         aResult = fServer[kDAQ]->Query(sqlQuery);
778         if (!aResult) {
779                 Log(detector, Form("GetDAQFileSources: Can't execute query <%s>!", sqlQuery.Data()));
780                 return 0;
781         }
782
783         if (aResult->GetRowCount() == 0) {
784                 Log(detector,
785                         Form("GetDAQFileSources: No result from SQL query <%s>!", sqlQuery.Data()));
786                 delete aResult;
787                 return 0;
788         }
789
790         TSQLRow* aRow;
791         TList *list = new TList();
792         list->SetOwner(1);
793
794         while((aRow = aResult->Next())){
795
796                 TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
797                 AliInfo(Form("daqSource = %s", daqSource.Data()));
798                 list->Add(new TObjString(daqSource));
799         }
800         delete aResult;
801
802         return list;
803
804 }
805
806 //______________________________________________________________________________________________
807 Bool_t AliShuttle::UpdateDAQTable(){
808 // Update DAQ table filling time_processed field in all rows corresponding to current run and detector
809
810         // check connection, in case connect
811         if(!Connect(kDAQ)){
812                 Log(fCurrentDetector, "UpdateDAQTable: Couldn't connect to DAQ Logbook !");
813                 return kFALSE;
814         }
815
816         TTimeStamp now; // now
817
818         // Loop on FES list entries
819         TIter iter(&fFESlist[kDAQ]);
820         TObjString *aFESentry=0;
821         while((aFESentry = dynamic_cast<TObjString*> (iter.Next()))){
822                 TString aFESentrystr = aFESentry->String();
823                 TObjArray *aFESarray = aFESentrystr.Tokenize("_!?!_");
824                 if(!aFESarray || aFESarray->GetEntries() != 2 ) {
825                         Log(fCurrentDetector,Form("UpdateDAQTable: error updating FES entry! string = %s",
826                                 aFESentrystr.Data()));
827                         if(aFESarray) delete aFESarray;
828                         return kFALSE;
829                 }
830                 const char* fileId = ((TObjString*) aFESarray->At(0))->GetName();
831                 const char* daqSource = ((TObjString*) aFESarray->At(1))->GetName();
832                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
833                         fCurrentRun,GetDetCode(fCurrentDetector), fileId, daqSource);
834
835                 delete aFESarray;
836
837                 TString sqlQuery = Form("update logbook_fs set time_processed=%d %s", now.GetSec(), whereClause.Data());
838
839                 AliInfo(Form("SQL query: \n%s",sqlQuery.Data()));
840
841                 // Query execution
842                 TSQLResult* aResult;
843                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
844                 if (!aResult) {
845                         Log(fCurrentDetector, Form("UpdateDAQTable: Can't execute query <%s>!", sqlQuery.Data()));
846                         return kFALSE;
847                 }
848                 delete aResult;
849
850                 // check result - TODO Is it necessary?
851                 sqlQuery = Form("select time_processed from logbook_fs %s", whereClause.Data());
852                 AliInfo(Form(" CHECK - SQL query: \n%s",sqlQuery.Data()));
853
854                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
855                 if (!aResult) {
856                         AliWarning("Can't check result!");
857                         continue;
858                 }
859
860         if (aResult->GetRowCount() == 0) {
861                 Log(fCurrentDetector,
862                         Form("GetDAQFileName: No result from SQL query <%s>!", sqlQuery.Data()));
863                 delete aResult;
864                 //return 0;
865         }
866
867         if (aResult->GetRowCount() >1) {
868                 Log(fCurrentDetector,
869                         Form("GetDAQFileName: More than one row resulting from SQL query <%s>!", sqlQuery.Data()));
870                 delete aResult;
871                 //return 0;
872         }
873
874                 TSQLRow *row = dynamic_cast<TSQLRow*> (aResult->Next());
875                 TString processedTimeString(row->GetField(0), row->GetFieldLength(0));
876                 Int_t processedTime = processedTimeString.Atoi();
877                 if(processedTime != now.GetSec()){
878                         Log(fCurrentDetector, Form("UpdateDAQTable: Update table error: processed_time=%d, now=%d !",
879                                 processedTime, now.GetSec()));
880                         delete aResult;
881                         return kFALSE;
882                 }
883
884                 delete aResult;
885
886         }
887
888         return kTRUE;
889 }
890
891 //______________________________________________________________________________________________
892 const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
893 // Retrieves a file from the DCS FES.
894
895 return "You're in DCS";
896
897 }
898
899 //______________________________________________________________________________________________
900 TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
901 // Retrieves a file from the DCS FES.
902
903 return NULL;
904
905 }
906
907 //______________________________________________________________________________________________
908 const char* AliShuttle::GetHLTFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
909 // Retrieves a file from the HLT FES.
910
911 return "You're in HLT";
912
913 }
914
915 //______________________________________________________________________________________________
916 TList* AliShuttle::GetHLTFileSources(const char* /*detector*/, const char* /*id*/){
917 // Retrieves a file from the HLT FES.
918
919 return NULL;
920
921 }
922
923 //______________________________________________________________________________________________
924 const char* AliShuttle::GetDetCode(const char* detector){
925 // Return detector code
926
927         for(int iDet=0; iDet < fgkNDetectors; iDet++){
928                 if(!strcmp(fgkDetectorName[iDet], detector)) return fgkDetectorCode[iDet];
929         }
930
931         return 0;
932 }
933
934 //______________________________________________________________________________________________
935 void AliShuttle::Log(const char* detector, const char* message)
936 {
937 // Fill log string with a message
938
939   TString toLog = Form("%s: %s, run %d - %s", TTimeStamp(time(0)).AsString(),
940     detector, GetCurrentRun(), message);
941   AliInfo(toLog.Data());
942
943   TString fileName;
944   fileName.Form("%s/%s.log", fgkShuttleLogDir, detector);
945   gSystem->ExpandPathName(fileName);
946
947   ofstream logFile;
948   logFile.open(fileName, ofstream::out | ofstream::app);
949
950   if (!logFile.is_open())
951   {
952     AliError(Form("Could not open file %s", fileName.Data()));
953     return;
954   }
955
956   logFile << toLog.Data() << "\n";
957
958   logFile.close();
959 }