introducing status management: The processing per subdetector is divided into several...
[u/mrichter/AliRoot.git] / SHUTTLE / AliShuttle.cxx
1 /**************************************************************************
2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
3  *                                                                        *
4  * Author: The ALICE Off-line Project.                                    *
5  * Contributors are mentioned in the code where appropriate.              *
6  *                                                                        *
7  * Permission to use, copy, modify and distribute this software and its   *
8  * documentation strictly for non-commercial purposes is hereby granted   *
9  * without fee, provided that the above copyright notice appears in all   *
10  * copies and that both the copyright notice and this permission notice   *
11  * appear in the supporting documentation. The authors make no claims     *
12  * about the suitability of this software for any purpose. It is          *
13  * provided "as is" without express or implied warranty.                  *
14  **************************************************************************/
15
16 /*
17 $Log$
18 Revision 1.9  2006/07/19 10:09:55  jgrosseo
19 new configuration, accesst to DAQ FES (Alberto)
20
21 Revision 1.8  2006/07/11 12:44:36  jgrosseo
22 adding parameters for extended validity range of data produced by preprocessor
23
24 Revision 1.7  2006/07/10 14:37:09  jgrosseo
25 small fix + todo comment
26
27 Revision 1.6  2006/07/10 13:01:41  jgrosseo
28 enhanced storing of last sucessfully processed run (alberto)
29
30 Revision 1.5  2006/07/04 14:59:57  jgrosseo
31 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
32
33 Revision 1.4  2006/06/12 09:11:16  jgrosseo
34 coding conventions (Alberto)
35
36 Revision 1.3  2006/06/06 14:26:40  jgrosseo
37 o) removed files that were moved to STEER
38 o) shuttle updated to follow the new interface (Alberto)
39
40 Revision 1.2  2006/03/07 07:52:34  hristov
41 New version (B.Yordanov)
42
43 Revision 1.6  2005/11/19 17:19:14  byordano
44 RetrieveDATEEntries and RetrieveConditionsData added
45
46 Revision 1.5  2005/11/19 11:09:27  byordano
47 AliShuttle declaration added
48
49 Revision 1.4  2005/11/17 17:47:34  byordano
50 TList changed to TObjArray
51
52 Revision 1.3  2005/11/17 14:43:23  byordano
53 import to local CVS
54
55 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
56 Initial import as subdirectory in AliRoot
57
58 Revision 1.2  2005/09/13 08:41:15  byordano
59 default startTime endTime added
60
61 Revision 1.4  2005/08/30 09:13:02  byordano
62 some docs added
63
64 Revision 1.3  2005/08/29 21:15:47  byordano
65 some docs added
66
67 */
68
69 //
70 // This class is the main manager for AliShuttle. 
71 // It organizes the data retrieval from DCS and call the 
72 // interface methods of AliPreprocessor.
73 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
74 // data for its set of aliases is retrieved. If there is registered
75 // AliPreprocessor for this detector then it will be used
76 // accroding to the schema (see AliPreprocessor).
77 // If there isn't registered AliPreprocessor than the retrieved
78 // data is stored automatically to the undelying AliCDBStorage.
79 // For detSpec is used the alias name.
80 //
81
82 #include "AliShuttle.h"
83
84 #include "AliCDBManager.h"
85 #include "AliCDBStorage.h"
86 #include "AliCDBId.h"
87 #include "AliCDBEntry.h"
88 #include "AliShuttleConfig.h"
89 #include "AliDCSClient.h"
90 #include "AliLog.h"
91 #include "AliPreprocessor.h"
92 #include "AliDefaultPreprocessor.h"
93 #include "AliShuttleStatus.h"
94
95 #include <TSystem.h>
96 #include <TObject.h>
97 #include <TString.h>
98 #include <TTimeStamp.h>
99 #include <TObjString.h>
100 #include <TSQLServer.h>
101 #include <TSQLResult.h>
102 #include <TSQLRow.h>
103
104 #include <fstream>
105
106 ClassImp(AliShuttle)
107
108 TString AliShuttle::fgkLocalUri("local://$ALICE_ROOT/SHUTTLE/ShuttleCDB");
109 const char* AliShuttle::fgkShuttleTempDir = "$ALICE_ROOT/SHUTTLE/temp";
110 const char* AliShuttle::fgkShuttleLogDir = "$ALICE_ROOT/SHUTTLE/log";
111
112 const char* AliShuttle::fgkDetectorName[AliShuttle::fgkNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
113         "PHOS", "CPV", "RICH", "EMCAL", "MUON_TRK", "MUON_TRG", "FMD", "ZDC", "PMD", "START", "VZERO"};
114
115 const char* AliShuttle::fgkDetectorCode[AliShuttle::fgkNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
116         "PHS", "CPV", "HMP", "EMC", "MCH", "MTR", "FMD", "ZDC", "PMD", "T00", "V00"};
117
118 //______________________________________________________________________________________________
119 AliShuttle::AliShuttle(const AliShuttleConfig* config,
120                 UInt_t timeout, Int_t retries):
121         fConfig(config),
122         fTimeout(timeout),
123         fRetries(retries), fCurrentRun(-1), fCurrentStartTime(0),
124         fCurrentEndTime(0),
125   fStatusEntry(0)
126 {
127         //
128         // config: AliShuttleConfig used
129         // timeout: timeout used for AliDCSClient connection
130         // retries: the number of retries in case of connection error.
131         //
132
133         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
134         for(int iSys=0;iSys<3;iSys++) {
135                 fServer[iSys]=0;
136                 fFESlist[iSys].SetOwner(kTRUE);
137         }
138 }
139
140 //______________________________________________________________________
141 AliShuttle::AliShuttle(const AliShuttle& /*other*/):
142 AliShuttleInterface()
143 {
144 // copy constructor (not implemented)
145
146 }
147
148 //______________________________________________________________________
149 AliShuttle &AliShuttle::operator=(const AliShuttle& /*other*/)
150 {
151 // assignment operator (not implemented)
152
153 return *this;
154 }
155
156 //______________________________________________________________________________________________
157 AliShuttle::~AliShuttle()
158 {
159 // destructor
160
161         fPreprocessorMap.DeleteAll();
162         for(int iSys=0;iSys<3;iSys++)
163                 if(fServer[iSys]) {
164                         fServer[iSys]->Close();
165                         delete fServer[iSys];
166                 }
167 }
168
169 //______________________________________________________________________________________________
170 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
171 {
172         //
173         // Registers new AliPreprocessor.
174         // It uses GetName() for indentificator of the pre processor.
175         // The pre processor is registered it there isn't any other
176         // with the same identificator (GetName()).
177         //
178
179         if (fPreprocessorMap.GetValue(preprocessor->GetName())) {
180                 AliWarning(Form("AliPreprocessor %s is already registered!",
181                         preprocessor->GetName()));
182                 return;
183         }
184
185         fPreprocessorMap.Add(new TObjString(preprocessor->GetName()), preprocessor);
186 }
187
188 //______________________________________________________________________________________________
189 UInt_t AliShuttle::Store(const char* detector,
190                 TObject* object, AliCDBMetaData* metaData, Int_t /*validityStart*/, Bool_t /*validityInfinite*/)
191 {
192         // store data into CDB
193   //
194   // validityStart is the start validity of the data, if not 0 GetCurrentRun() - validityStart is taken
195   // validityInfinite defines if the data is valid until new data arrives (e.g. for calibration runs)
196   //
197         // returns 0 if fail
198         //         1 if stored in main (Grid) storage
199         //         2 if stored in backup (Local) storage
200
201   // TODO implement use of two parameters
202
203   // TODO shouldn't the path be given by the preprocessor???
204         AliCDBId id(AliCDBPath(detector, "DCS", "Data"),
205                 GetCurrentRun(), GetCurrentRun());
206
207         UInt_t result = 0;
208         if (!(AliCDBManager::Instance()->IsDefaultStorageSet())) {
209                 Log(detector, "No CDB storage set!");
210         } else {
211                 result = (UInt_t) AliCDBManager::Instance()->Put(object, id, metaData);
212         }
213         if(!result) {
214
215                 Log(detector, "Error while storing object in main storage!");
216                 AliError("local storage will be used!");
217
218                 AliCDBStorage *origStorage = AliCDBManager::Instance()->GetDefaultStorage();
219
220                 result = AliCDBManager::Instance()->GetStorage(fgkLocalUri)
221                                         ->Put(object, id, metaData);
222
223                 AliCDBManager::Instance()->SetDefaultStorage(origStorage);
224
225                 if(result) {
226                         result = 2;
227                 }else{
228                         Log(detector, "Can't store data!");
229                 }
230         }
231         return result;
232
233 }
234
235 //______________________________________________________________________________________________
236 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
237 {
238   // Reads the AliShuttleStatus from the CDB
239
240   if (fStatusEntry)
241   {
242     delete fStatusEntry;
243     fStatusEntry = 0;
244   }
245
246   AliCDBStorage *origStorage = AliCDBManager::Instance()->GetDefaultStorage();
247
248   fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalURI())
249       ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), fCurrentRun);
250
251   AliCDBManager::Instance()->SetDefaultStorage(origStorage);
252
253   if (!fStatusEntry)
254     return 0;
255
256   TObject* anObject = fStatusEntry->GetObject();
257   if (anObject == NULL || anObject->IsA() != AliShuttleStatus::Class())
258   {
259     AliError("Invalid object stored to CDB!");
260     return 0;
261   }
262
263   AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (anObject);
264   return status;
265 }
266
267 //______________________________________________________________________________________________
268 void AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
269 {
270   // writes the status for one subdetector
271
272   if (fStatusEntry)
273   {
274     delete fStatusEntry;
275     fStatusEntry = 0;
276   }
277
278   AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), fCurrentRun, fCurrentRun);
279
280   fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
281
282   AliCDBStorage *origStorage = AliCDBManager::Instance()->GetDefaultStorage();
283   AliCDBManager::Instance()->GetStorage(fgkLocalUri)->Put(fStatusEntry);
284   AliCDBManager::Instance()->SetDefaultStorage(origStorage);
285 }
286
287 //______________________________________________________________________________________________
288 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
289 {
290   // changes the AliShuttleStatus for the given detector and run to the given status
291
292   if (!fStatusEntry)
293   {
294     AliError("UNEXPECTED: fStatusEntry empty");
295     return;
296   }
297
298   TObject* anObject = fStatusEntry->GetObject();
299   AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (anObject);
300
301   if (!status)
302   {
303     AliError("UNEXPECTED: status could not be read from current CDB entry");
304     return;
305   }
306
307   Log("SHUTTLE", Form("%s: Changing state from %s to %s", fCurrentDetector.Data(), status->GetStatusName(), status->GetStatusName(newStatus)));
308
309   status->SetStatus(newStatus);
310   if (increaseCount)
311     status->IncreaseCount();
312
313   AliCDBStorage *origStorage = AliCDBManager::Instance()->GetDefaultStorage();
314   AliCDBManager::Instance()->GetStorage(fgkLocalUri)->Put(fStatusEntry);
315   AliCDBManager::Instance()->SetDefaultStorage(origStorage);
316 }
317
318 //______________________________________________________________________________________________
319 Bool_t AliShuttle::ContinueProcessing()
320 {
321   // this function reads the AliShuttleStatus information from CDB and
322   // checks if the processing should be continued
323   // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
324
325   AliShuttleStatus* status = ReadShuttleStatus();
326   if (!status)
327   {
328     // first time
329
330     Log("SHUTTLE", Form("%s: Processing first time.", fCurrentDetector.Data()));
331     status = new AliShuttleStatus(AliShuttleStatus::kStarted);
332     WriteShuttleStatus(status);
333
334     return kTRUE;
335   }
336
337   if (status->GetStatus() == AliShuttleStatus::kDone)
338   {
339     Log("SHUTTLE", Form("%s already done for run %d", fCurrentDetector.Data(), fCurrentRun));
340     return kFALSE;
341   }
342
343   if (status->GetStatus() == AliShuttleStatus::kFailed)
344   {
345     Log("SHUTTLE", Form("%s already in failed state for run %d", fCurrentDetector.Data(), fCurrentRun));
346     return kFALSE;
347   }
348
349   // if we get here, there is a restart
350
351   // abort conditions
352   if (status->GetStatus() == AliShuttleStatus::kPPStarted && status->GetCount() > fConfig->GetMaxPPRetries() ||
353       status->GetCount() > fConfig->GetMaxRetries())
354   {
355     Log("SHUTTLE", Form("%s, run %d failed to often, %d times, status %s. Skipping processing.", fCurrentDetector.Data(), fCurrentRun, status->GetCount(), status->GetStatusName()));
356
357     return kFALSE;
358   }
359
360   Log("SHUTTLE", Form("Restart of %s, run %d. Got stuck before in %s, count %d", fCurrentDetector.Data(), fCurrentRun, status->GetStatusName(), status->GetCount()));
361
362   UpdateShuttleStatus(AliShuttleStatus::kStarted, kTRUE);
363
364   return kTRUE;
365 }
366
367 //______________________________________________________________________________________________
368 Bool_t AliShuttle::Process(Int_t run, UInt_t startTime, UInt_t endTime)
369 {
370         //
371         // Makes data retrieval for all detectors in the configuration.
372         // run: is the run number used
373         // startTime: is the run start time
374         // endTime: is the run end time
375         // Returns kFALSE in case of error occured and kTRUE otherwise
376         //
377
378         AliInfo(Form("\n\n ^*^*^*^*^*^* Processing run %d ^*^*^*^*^*^*", run));
379
380         // Initialization
381         Bool_t hasError = kFALSE;
382         for(Int_t iSys=0;iSys<3;iSys++) fFESCalled[iSys]=kFALSE;
383
384   fCurrentRun = run;
385         fCurrentStartTime = startTime;
386         fCurrentEndTime = endTime;
387
388         // Loop on detectors in the configuration
389         TIter iter(fConfig->GetDetectors());
390         TObjString* aDetector;
391
392         while ((aDetector = (TObjString*) iter.Next())) {
393     fCurrentDetector = aDetector->String();
394
395                 Bool_t detectorError=kFALSE;
396                 if (!fConfig->HostProcessDetector(fCurrentDetector)) continue;
397
398     if (ContinueProcessing() == kFALSE)
399       continue;
400
401                 if(!Process()) {
402                         hasError = kTRUE;
403                         detectorError=kTRUE;
404                         continue;
405                 }
406                 AliInfo(Form("Process ended successfully for detector %s!",aDetector->GetName()));
407
408                 // Process successful: Update time_processed field in FES logbooks!
409                 if(fFESCalled[kDAQ]) {
410                         hasError = (UpdateDAQTable() == kFALSE);
411                         fFESlist[kDAQ].Clear();
412                 }
413                 //if(fFESCalled[kDCS]) {
414                 //      hasError = UpdateDCSTable(aDetector->GetName());
415                 //      fFESlist[kDCS].Clear();
416                 //}
417                 //if(fFESCalled[kHLT]) {
418                 //      hasError = UpdateHLTTable(aDetector->GetName());
419                 //      fFESlist[kHLT].Clear();
420                 //}
421
422     UpdateShuttleStatus(AliShuttleStatus::kDone);
423   }
424
425         fCurrentRun = -1;
426         fCurrentStartTime = 0;
427         fCurrentEndTime = 0;
428
429         return hasError == kFALSE;
430 }
431
432 //______________________________________________________________________________________________
433 Bool_t AliShuttle::Process()
434 {
435         //
436         // Makes data retrieval just for one specific detector.
437         // Threre should be a configuration for this detector.
438         // run: is the run number used
439         // startTime: is the run start time
440         // endTime: is the run end time
441         // detector: detector for which the retrieval will be made
442         // Returns kFALSE in case of error occured and kTRUE otherwise
443         //
444
445         AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), fCurrentRun));
446
447         if (!fConfig->HasDetector(fCurrentDetector)) {
448                 Log(fCurrentDetector, "There isn't any configuration for %s !");
449     UpdateShuttleStatus(AliShuttleStatus::kFailed);
450                 return kFALSE;
451         }
452
453   UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
454
455   TString host(fConfig->GetDCSHost(fCurrentDetector));
456         Int_t port = fConfig->GetDCSPort(fCurrentDetector);
457
458         TIter iter(fConfig->GetDCSAliases(fCurrentDetector));
459         TObjString* anAlias;
460         TMap aliasMap;
461
462         Bool_t hasError = kFALSE;
463         Bool_t result=kFALSE;
464
465         while ((anAlias = (TObjString*) iter.Next())) {
466                 TObjArray valueSet;
467                 result = GetValueSet(host, port, anAlias->String(), valueSet);
468                 //AliInfo(Form("Port = %d",port));
469                 //result = kTRUE;
470                 if(result) {
471                         aliasMap.Add(anAlias->Clone(), valueSet.Clone());
472                 }else{
473                         TString message = Form("Error while retrieving alias %s !",
474                                         anAlias->GetName());
475                         Log(fCurrentDetector, message.Data());
476                         hasError = kTRUE;
477                 }
478         }
479
480         // even if hasError is TRUE the Shuttle should keep on processing the detector (calib files!)
481
482         if (hasError)
483   {
484     UpdateShuttleStatus(AliShuttleStatus::kDCSError);
485     return kFALSE;
486   }
487
488   UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
489
490   AliPreprocessor* aPreprocessor =
491                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
492         if(aPreprocessor)
493         {
494                 aPreprocessor->Initialize(fCurrentRun, fCurrentStartTime, fCurrentEndTime);
495                 hasError = (aPreprocessor->Process(&aliasMap) == 0);
496         }else{
497     // TODO default behaviour?
498                 AliInfo(Form("No Preprocessor for %s: storing TMap of DP arrays into CDB!", fCurrentDetector.Data()));
499                 AliCDBMetaData metaData;
500                 AliDCSValue dcsValue(fCurrentStartTime, fCurrentEndTime);
501                 metaData.SetResponsible(Form("Duck, Donald"));
502                 metaData.SetProperty("StartEndTime", &dcsValue);
503                 metaData.SetComment("Automatically stored by Shuttle!");
504                 hasError = (Store(fCurrentDetector, &aliasMap, &metaData) == 0);
505         }
506
507   if (hasError)
508     UpdateShuttleStatus(AliShuttleStatus::kPPError);
509   else
510     UpdateShuttleStatus(AliShuttleStatus::kPPDone);
511
512         aliasMap.Delete();
513
514         return hasError == kFALSE;
515 }
516
517 //______________________________________________________________________________________________
518 Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* alias,
519                                 TObjArray& valueSet)
520 {
521 // Retrieve all "alias" data points from the DCS server
522 // host, port: TSocket connection parameters
523 // alias: name of the alias
524 // valueSet: array of retrieved AliDCSValue's 
525
526         AliDCSClient client(host, port, fTimeout, fRetries);
527         if (!client.IsConnected()) {
528                 return kFALSE;
529         }
530
531         Int_t result = client.GetAliasValues(alias,
532                 GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
533
534         if (result < 0) {
535                 AliError(Form("Can't get '%s'! Reason: %s",
536                         alias, AliDCSClient::GetErrorString(result)));
537
538                 if (result == AliDCSClient::fgkServerError) {
539                         AliError(Form("Server error: %s",
540                                 client.GetServerError().Data()));
541                 }
542
543                 return kFALSE;
544         }
545
546         return kTRUE;
547 }
548
549 //______________________________________________________________________________________________
550 const char* AliShuttle::GetFile(Int_t system, const char* detector,
551                 const char* id, const char* source)
552 {
553 // Get calibration file from file exchange servers
554 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
555
556         switch(system){
557                 case kDAQ:
558                         return GetDAQFileName(detector, id, source);
559                         break;
560                 case kDCS:
561                         return GetDCSFileName(detector, id, source);
562                         break;
563                 case kHLT:
564                         return GetHLTFileName(detector, id, source);
565                         break;
566                 default:
567                         AliError(Form("No valid system index: %d",system));
568         }
569
570         return 0;
571 }
572
573 //______________________________________________________________________________________________
574 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
575 {
576 // Get sources producing the condition file Id from file exchange servers
577 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
578
579         switch(system){
580                 case kDAQ:
581                         return GetDAQFileSources(detector, id);
582                         break;
583                 case kDCS:
584                         return GetDCSFileSources(detector, id);
585                         break;
586                 case kHLT:
587                         return GetHLTFileSources(detector, id);
588                         break;
589                 default:
590                         AliError(Form("No valid system index: %d",system));
591         }
592
593         return NULL;
594 }
595
596 //______________________________________________________________________________________________
597 Bool_t AliShuttle::Connect(Int_t system){
598 // Connect to MySQL Server of the system's FES logbook
599
600         // check connection: if already connected return
601         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
602
603         TString aFESlbHost= Form("mysql://%s", fConfig->GetFESlbHost(system));
604
605         fServer[system] = TSQLServer::Connect(aFESlbHost,
606                         fConfig->GetFESlbUser(system),
607                         fConfig->GetFESlbPass(system));
608         if (!fServer[system] || !fServer[system]->IsConnected()) {
609                 AliError(Form("Can't establish connection to FES logbook for %s !",fkSystemNames[system]));
610                 return kFALSE;
611         }
612
613         // Get tables
614         // TODO in the configuration should the table name be there too?
615         switch(system){
616                 case kDAQ:
617                         fServer[kDAQ]->GetTables("REFSYSLOG");
618                         break;
619                 case kDCS:
620                         //fServer[kDCS]->GetTables("REFSYSLOG");
621                         break;
622                 case kHLT:
623                         //fServer[kHLT]->GetTables("REFSYSLOG");
624                         break;
625                 default:
626                         break;
627         }
628
629         return kTRUE;
630 }
631
632 //______________________________________________________________________________________________
633 const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source){
634 // Retrieves a file from the DAQ FES.
635 // First queris the DAQ logbook_fs for the DAQ file name, using the run, detector, id and source info
636 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
637 // run: current run being processed (fCurrentRun)
638 // detector: comes from the Preprocessor name (must be converted into detector code with GetDetCode)
639 // id: provided as a parameter by the Preprocessor
640 // source: provided by the Preprocessor through GetFileSources function
641
642         // check connection, in case connect
643         if(!Connect(kDAQ)){
644                 Log(detector, "GetDAQFileName: Couldn't connect to DAQ Logbook !");
645                 return 0;
646         }
647
648         // Query preparation
649         TString sqlQueryStart = "select filePath from logbook_fs where";
650         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
651                                 fCurrentRun, GetDetCode(detector), id, source);
652         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
653
654         AliInfo(Form("SQL query: \n%s",sqlQuery.Data()));
655
656         // Query execution
657         TSQLResult* aResult;
658         aResult = fServer[kDAQ]->Query(sqlQuery);
659         if (!aResult) {
660                 Log(detector, Form("Can't execute query <%s>!", sqlQuery.Data()));
661                 return 0;
662         }
663
664         if (aResult->GetRowCount() == 0) {
665                 Log(detector,
666                         Form("GetDAQFileName: No result from SQL query <%s>!", sqlQuery.Data()));
667                 delete aResult;
668                 return 0;
669         }
670
671         if (aResult->GetRowCount() >1) {
672                 Log(detector,
673                         Form("GetDAQFileName: More than one row resulting from SQL query <%s>!", sqlQuery.Data()));
674                 delete aResult;
675                 return 0;
676         }
677
678         TSQLRow* aRow = aResult->Next();
679
680         if(!aRow){
681                 Log(detector, Form("GetDAQFileName: Empty set result from query <%s>!", sqlQuery.Data()));
682                 delete aResult;
683                 return 0;
684         }
685
686         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
687
688         delete aResult;
689
690         AliInfo(Form("filePath = %s",filePath.Data()));
691
692         // retrieved file is renamed to make it unique
693         TString localFileName = Form("%s_%d_%s_%s.shuttle",
694                                         detector, fCurrentRun, id, source);
695
696         // file retrieval from DAQ FES
697         Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
698         if(!result) {
699                 Log(detector, Form("copying file %s from DAQ FES failed!", filePath.Data()));
700                 return 0;
701         } else {
702                 AliInfo(Form("File %s copied from DAQ FES into %s/%s !",
703                         filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
704         }
705
706
707         fFESCalled[kDAQ]=kTRUE;
708         TObjString *fileParams = new TObjString(Form("%s_!?!_%s", id, source));
709         fFESlist[kDAQ].Add(fileParams);
710
711         return localFileName.Data();
712
713 }
714
715 //______________________________________________________________________________________________
716 Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName){
717
718         // check temp directory: trying to cd to temp; if it does not exist, create it
719         AliInfo(Form("Copy file %s from DAQ FES into folder %s and rename it as %s",
720                         daqFileName,fgkShuttleTempDir, localFileName));
721
722         void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
723         if (dir == NULL) {
724                 if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
725                         AliError(Form("Can't open directory <%s>!", fgkShuttleTempDir));
726                         return kFALSE;
727                 }
728
729         } else {
730                 gSystem->FreeDirectory(dir);
731         }
732
733         TString baseDAQFESFolder = "DAQ";
734         TString command = Form("scp %s@%s:%s/%s %s/%s",
735                 fConfig->GetFESUser(kDAQ),
736                 fConfig->GetFESHost(kDAQ),
737                 baseDAQFESFolder.Data(),
738                 daqFileName,
739                 fgkShuttleTempDir,
740                 localFileName);
741
742         AliInfo(Form("%s",command.Data()));
743
744         UInt_t nRetries = 0;
745         UInt_t maxRetries = 3;
746
747         // copy!! if successful TSystem::Exec returns 0
748         while(nRetries++ < maxRetries) {
749                 AliInfo(Form("Trying to copy file. Retry # %d", nRetries));
750                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
751         }
752
753         return kFALSE;
754
755 }
756
757 //______________________________________________________________________________________________
758 TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id){
759 // Retrieves a file from the DCS FES.
760
761         // check connection, in case connect
762         if(!Connect(kDAQ)){
763                 Log(detector, "GetDAQFileName: Couldn't connect to DAQ Logbook !");
764                 return 0;
765         }
766
767         // Query preparation
768         TString sqlQueryStart = "select DAQsource from logbook_fs where";
769         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
770                                 fCurrentRun, GetDetCode(detector), id);
771         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
772
773         AliInfo(Form("SQL query: \n%s",sqlQuery.Data()));
774
775         // Query execution
776         TSQLResult* aResult;
777         aResult = fServer[kDAQ]->Query(sqlQuery);
778         if (!aResult) {
779                 Log(detector, Form("GetDAQFileSources: Can't execute query <%s>!", sqlQuery.Data()));
780                 return 0;
781         }
782
783         if (aResult->GetRowCount() == 0) {
784                 Log(detector,
785                         Form("GetDAQFileSources: No result from SQL query <%s>!", sqlQuery.Data()));
786                 delete aResult;
787                 return 0;
788         }
789
790         TSQLRow* aRow;
791         TList *list = new TList();
792         list->SetOwner(1);
793
794         while((aRow = aResult->Next())){
795
796                 TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
797                 AliInfo(Form("daqSource = %s", daqSource.Data()));
798                 list->Add(new TObjString(daqSource));
799         }
800         delete aResult;
801
802         return list;
803
804 }
805
806 //______________________________________________________________________________________________
807 Bool_t AliShuttle::UpdateDAQTable(){
808 // Update DAQ table filling time_processed field in all rows corresponding to current run and detector
809
810         // check connection, in case connect
811         if(!Connect(kDAQ)){
812                 Log(fCurrentDetector, "UpdateDAQTable: Couldn't connect to DAQ Logbook !");
813                 return kFALSE;
814         }
815
816         TTimeStamp now; // now
817
818         // Loop on FES list entries
819         TIter iter(&fFESlist[kDAQ]);
820         TObjString *aFESentry=0;
821         while((aFESentry = dynamic_cast<TObjString*> (iter.Next()))){
822                 TString aFESentrystr = aFESentry->String();
823                 TObjArray *aFESarray = aFESentrystr.Tokenize("_!?!_");
824                 if(!aFESarray || aFESarray->GetEntries() != 2 ) {
825                         Log(fCurrentDetector,Form("UpdateDAQTable: error updating FES entry! string = %s",
826                                 aFESentrystr.Data()));
827                         if(aFESarray) delete aFESarray;
828                         return kFALSE;
829                 }
830                 const char* fileId = ((TObjString*) aFESarray->At(0))->GetName();
831                 const char* daqSource = ((TObjString*) aFESarray->At(1))->GetName();
832                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
833                         fCurrentRun,GetDetCode(fCurrentDetector), fileId, daqSource);
834
835                 delete aFESarray;
836
837                 TString sqlQuery = Form("update logbook_fs set time_processed=%d %s", now.GetSec(), whereClause.Data());
838
839                 AliInfo(Form("SQL query: \n%s",sqlQuery.Data()));
840
841                 // Query execution
842                 TSQLResult* aResult;
843                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
844                 if (!aResult) {
845                         Log(fCurrentDetector, Form("UpdateDAQTable: Can't execute query <%s>!", sqlQuery.Data()));
846                         return kFALSE;
847                 }
848                 delete aResult;
849
850                 // check result - TODO Is it necessary?
851                 sqlQuery = Form("select time_processed from logbook_fs %s", whereClause.Data());
852                 AliInfo(Form(" CHECK - SQL query: \n%s",sqlQuery.Data()));
853
854                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
855                 if (!aResult) {
856                         AliWarning("Can't check result!");
857                         continue;
858                 }
859
860         if (aResult->GetRowCount() == 0) {
861                 Log(fCurrentDetector,
862                         Form("GetDAQFileName: No result from SQL query <%s>!", sqlQuery.Data()));
863                 delete aResult;
864                 //return 0;
865         }
866
867         if (aResult->GetRowCount() >1) {
868                 Log(fCurrentDetector,
869                         Form("GetDAQFileName: More than one row resulting from SQL query <%s>!", sqlQuery.Data()));
870                 delete aResult;
871                 //return 0;
872         }
873
874                 TSQLRow *row = dynamic_cast<TSQLRow*> (aResult->Next());
875                 TString processedTimeString(row->GetField(0), row->GetFieldLength(0));
876                 Int_t processedTime = processedTimeString.Atoi();
877                 if(processedTime != now.GetSec()){
878                         Log(fCurrentDetector, Form("UpdateDAQTable: Update table error: processed_time=%d, now=%d !",
879                                 processedTime, now.GetSec()));
880                         delete aResult;
881                         return kFALSE;
882                 }
883
884                 delete aResult;
885
886         }
887
888         return kTRUE;
889 }
890
891 //______________________________________________________________________________________________
892 const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
893 // Retrieves a file from the DCS FES.
894
895 return "You're in DCS";
896
897 }
898
899 //______________________________________________________________________________________________
900 TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
901 // Retrieves a file from the DCS FES.
902
903 return NULL;
904
905 }
906
907 //______________________________________________________________________________________________
908 const char* AliShuttle::GetHLTFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
909 // Retrieves a file from the HLT FES.
910
911 return "You're in HLT";
912
913 }
914
915 //______________________________________________________________________________________________
916 TList* AliShuttle::GetHLTFileSources(const char* /*detector*/, const char* /*id*/){
917 // Retrieves a file from the HLT FES.
918
919 return NULL;
920
921 }
922
923 //______________________________________________________________________________________________
924 const char* AliShuttle::GetDetCode(const char* detector){
925 // Return detector code
926
927         for(int iDet=0; iDet < fgkNDetectors; iDet++){
928                 if(!strcmp(fgkDetectorName[iDet], detector)) return fgkDetectorCode[iDet];
929         }
930
931         return 0;
932 }
933
934 //______________________________________________________________________________________________
935 void AliShuttle::Log(const char* detector, const char* message)
936 {
937 // Fill log string with a message
938
939         TString toLog = Form("%s, run %d - %s", detector, GetCurrentRun(), message);
940         AliInfo(toLog.Data());
941
942   TString fileName;
943   fileName.Form("%s/%s.log", fgkShuttleLogDir, detector);
944   gSystem->ExpandPathName(fileName);
945
946   AliInfo(fileName.Data());
947
948   ofstream logFile;
949   logFile.open(fileName, ofstream::out | ofstream::app);
950
951   logFile << toLog.Data() << "\n";
952
953   logFile.close();
954 }