adapting to new CDB classes
[u/mrichter/AliRoot.git] / SHUTTLE / AliShuttle.cxx
1 /**************************************************************************
2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
3  *                                                                        *
4  * Author: The ALICE Off-line Project.                                    *
5  * Contributors are mentioned in the code where appropriate.              *
6  *                                                                        *
7  * Permission to use, copy, modify and distribute this software and its   *
8  * documentation strictly for non-commercial purposes is hereby granted   *
9  * without fee, provided that the above copyright notice appears in all   *
10  * copies and that both the copyright notice and this permission notice   *
11  * appear in the supporting documentation. The authors make no claims     *
12  * about the suitability of this software for any purpose. It is          *
13  * provided "as is" without express or implied warranty.                  *
14  **************************************************************************/
15
16 /*
17 $Log$
18 Revision 1.16  2006/10/05 15:46:26  jgrosseo
19 applying to the new interface
20
21 Revision 1.15  2006/10/02 16:38:39  jgrosseo
22 update (alberto):
23 fixed memory leaks
24 storing of objects that failed to be stored to the grid before
25 interfacing of shuttle status table in daq system
26
27 Revision 1.14  2006/08/29 09:16:05  jgrosseo
28 small update
29
30 Revision 1.13  2006/08/15 10:50:00  jgrosseo
31 effc++ corrections (alberto)
32
33 Revision 1.12  2006/08/08 14:19:29  jgrosseo
34 Update to shuttle classes (Alberto)
35
36 - Possibility to set the full object's path in the Preprocessor's and
37 Shuttle's  Store functions
38 - Possibility to extend the object's run validity in the same classes
39 ("startValidity" and "validityInfinite" parameters)
40 - Implementation of the StoreReferenceData function to store reference
41 data in a dedicated CDB storage.
42
43 Revision 1.11  2006/07/21 07:37:20  jgrosseo
44 last run is stored after each run
45
46 Revision 1.10  2006/07/20 09:54:40  jgrosseo
47 introducing status management: The processing per subdetector is divided into several steps,
48 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
49 can keep track of the number of failures and skips further processing after a certain threshold is
50 exceeded. These thresholds can be configured in LDAP.
51
52 Revision 1.9  2006/07/19 10:09:55  jgrosseo
53 new configuration, accesst to DAQ FES (Alberto)
54
55 Revision 1.8  2006/07/11 12:44:36  jgrosseo
56 adding parameters for extended validity range of data produced by preprocessor
57
58 Revision 1.7  2006/07/10 14:37:09  jgrosseo
59 small fix + todo comment
60
61 Revision 1.6  2006/07/10 13:01:41  jgrosseo
62 enhanced storing of last sucessfully processed run (alberto)
63
64 Revision 1.5  2006/07/04 14:59:57  jgrosseo
65 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
66
67 Revision 1.4  2006/06/12 09:11:16  jgrosseo
68 coding conventions (Alberto)
69
70 Revision 1.3  2006/06/06 14:26:40  jgrosseo
71 o) removed files that were moved to STEER
72 o) shuttle updated to follow the new interface (Alberto)
73
74 Revision 1.2  2006/03/07 07:52:34  hristov
75 New version (B.Yordanov)
76
77 Revision 1.6  2005/11/19 17:19:14  byordano
78 RetrieveDATEEntries and RetrieveConditionsData added
79
80 Revision 1.5  2005/11/19 11:09:27  byordano
81 AliShuttle declaration added
82
83 Revision 1.4  2005/11/17 17:47:34  byordano
84 TList changed to TObjArray
85
86 Revision 1.3  2005/11/17 14:43:23  byordano
87 import to local CVS
88
89 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
90 Initial import as subdirectory in AliRoot
91
92 Revision 1.2  2005/09/13 08:41:15  byordano
93 default startTime endTime added
94
95 Revision 1.4  2005/08/30 09:13:02  byordano
96 some docs added
97
98 Revision 1.3  2005/08/29 21:15:47  byordano
99 some docs added
100
101 */
102
103 //
104 // This class is the main manager for AliShuttle. 
105 // It organizes the data retrieval from DCS and call the 
106 // interface methods of AliPreprocessor.
107 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
108 // data for its set of aliases is retrieved. If there is registered
109 // AliPreprocessor for this detector then it will be used
110 // accroding to the schema (see AliPreprocessor).
111 // If there isn't registered AliPreprocessor than the retrieved
112 // data is stored automatically to the undelying AliCDBStorage.
113 // For detSpec is used the alias name.
114 //
115
116 #include "AliShuttle.h"
117
118 #include "AliCDBManager.h"
119 #include "AliCDBStorage.h"
120 #include "AliCDBId.h"
121 #include "AliCDBRunRange.h"
122 #include "AliCDBPath.h"
123 #include "AliCDBEntry.h"
124 #include "AliShuttleConfig.h"
125 #include "AliDCSClient.h"
126 #include "AliLog.h"
127 #include "AliPreprocessor.h"
128 #include "AliShuttleStatus.h"
129 #include "AliShuttleLogbookEntry.h"
130
131 #include <TSystem.h>
132 #include <TObject.h>
133 #include <TString.h>
134 #include <TTimeStamp.h>
135 #include <TObjString.h>
136 #include <TSQLServer.h>
137 #include <TSQLResult.h>
138 #include <TSQLRow.h>
139
140 #include <fstream>
141
142 ClassImp(AliShuttle)
143
144 TString AliShuttle::fgkMainCDB("alien://folder=ShuttleCDB");
145 TString AliShuttle::fgkLocalCDB("local://LocalShuttleCDB");
146 TString AliShuttle::fgkMainRefStorage("alien://folder=ShuttleReference");
147 TString AliShuttle::fgkLocalRefStorage("local://LocalReferenceStorage");
148
149 Bool_t AliShuttle::fgkProcessDCS(kTRUE); 
150
151
152 const char* AliShuttle::fgkShuttleTempDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/temp");
153 const char* AliShuttle::fgkShuttleLogDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/log");
154
155 const char* AliShuttle::fgkDetectorName[AliShuttle::kNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
156         "PHOS", "CPV", "RICH", "EMCAL", "MUON_TRK", "MUON_TRG", "FMD", "ZDC", "PMD", "START", "VZERO"};
157
158 const char* AliShuttle::fgkDetectorCode[AliShuttle::kNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
159         "PHS", "CPV", "HMP", "EMC", "MCH", "MTR", "FMD", "ZDC", "PMD", "T00", "V00"};
160
161 //______________________________________________________________________________________________
162 AliShuttle::AliShuttle(const AliShuttleConfig* config,
163                 UInt_t timeout, Int_t retries):
164 fConfig(config),
165 fTimeout(timeout), fRetries(retries),
166 fPreprocessorMap(),
167 fLogbookEntry(0),
168 fCurrentDetector(""),
169 fStatusEntry(0),
170 fGridError(kFALSE)
171 {
172         //
173         // config: AliShuttleConfig used
174         // timeout: timeout used for AliDCSClient connection
175         // retries: the number of retries in case of connection error.
176         //
177
178         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
179         for(int iSys=0;iSys<3;iSys++) {
180                 fServer[iSys]=0;
181                 fFESlist[iSys].SetOwner(kTRUE);
182         }
183         fPreprocessorMap.SetOwner(kTRUE);
184 }
185
186 //______________________________________________________________________
187 AliShuttle::AliShuttle(const AliShuttle& /*other*/):
188 AliShuttleInterface(),
189 fConfig(0),
190 fTimeout(0), fRetries(0),
191 fPreprocessorMap(),
192 fLogbookEntry(0),
193 fCurrentDetector(""),
194 fStatusEntry(0),
195 fGridError(kFALSE)
196 {
197 // copy constructor (not implemented)
198
199 }
200
201 //______________________________________________________________________
202 AliShuttle &AliShuttle::operator=(const AliShuttle& /*other*/)
203 {
204 // assignment operator (not implemented)
205
206 return *this;
207 }
208
209 //______________________________________________________________________________________________
210 AliShuttle::~AliShuttle()
211 {
212 // destructor
213
214         fPreprocessorMap.DeleteAll();
215         for(int iSys=0;iSys<3;iSys++)
216                 if(fServer[iSys]) {
217                         fServer[iSys]->Close();
218                         delete fServer[iSys];
219                 }
220
221         if (fStatusEntry){
222                 delete fStatusEntry;
223                 fStatusEntry = 0;
224         }
225 }
226
227 //______________________________________________________________________________________________
228 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
229 {
230         //
231         // Registers new AliPreprocessor.
232         // It uses GetName() for indentificator of the pre processor.
233         // The pre processor is registered it there isn't any other
234         // with the same identificator (GetName()).
235         //
236
237         if (fPreprocessorMap.GetValue(preprocessor->GetName())) {
238                 AliWarning(Form("AliPreprocessor %s is already registered!",
239                         preprocessor->GetName()));
240                 return;
241         }
242
243         fPreprocessorMap.Add(new TObjString(preprocessor->GetName()), preprocessor);
244 }
245 //______________________________________________________________________________________________
246 UInt_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
247                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
248 {
249   // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
250   // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
251   // using this function. Use StoreReferenceData instead!
252   // It calls WriteToCDB function which perform actual storage
253
254         return WriteToCDB(fgkMainCDB, fgkLocalCDB, path, object,
255                                 metaData, validityStart, validityInfinite);
256
257 }
258
259 //______________________________________________________________________________________________
260 UInt_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData)
261 {
262   // Stores a CDB object in the storage for reference data. This objects will not be available during
263   // offline reconstrunction. Use this function for reference data only!
264   // It calls WriteToCDB function which perform actual storage
265
266         return WriteToCDB(fgkMainRefStorage, fgkLocalRefStorage, path, object, metaData);
267
268 }
269
270 //______________________________________________________________________________________________
271 UInt_t AliShuttle::WriteToCDB(const char* mainUri, const char* localUri,
272                         const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
273                         Int_t validityStart, Bool_t validityInfinite)
274 {
275   // write object into the CDB. Parameters are passed by Store and StoreReferenceData functions.
276   // The parameters are:
277   //   1) Uri of the main storage (Grid)
278   //   2) Uri of the backup storage (Local)
279   //   3) the object's path.
280   //   4) the object to be stored
281   //   5) the metaData to be associated with the object
282   //   6) the validity start run number w.r.t. the current run,
283   //      if the data is valid only for this run leave the default 0
284   //   7) specifies if the calibration data is valid for infinity (this means until updated),
285   //      typical for calibration runs, the default is kFALSE
286   //
287   // returns 0 if fail
288   //         1 if stored in main (Grid) storage
289   //         2 if stored in backup (Local) storage
290
291         const char* cdbType = (mainUri == fgkMainCDB) ? "CDB" : "Reference";
292
293         Int_t firstRun = GetCurrentRun() - validityStart;
294         if(firstRun < 0) {
295                 AliError("First valid run happens to be less than 0! Setting it to 0.");
296                 firstRun=0;
297         }
298
299         Int_t lastRun = -1;
300         if(validityInfinite) {
301                 lastRun = AliCDBRunRange::Infinity();
302         } else {
303                 lastRun = GetCurrentRun();
304         }
305
306         AliCDBId id(path, firstRun, lastRun, -1, -1);
307
308         if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
309                 TObjString runUsed = Form("%d", GetCurrentRun());
310                 metaData->SetProperty("RunUsed(TObjString)",&runUsed);
311         }
312
313         UInt_t result = 0;
314
315         if (!(AliCDBManager::Instance()->GetStorage(mainUri))) {
316                 AliError(Form("WriteToCDB - Cannot activate main %s storage", cdbType));
317         } else {
318                 result = (UInt_t) AliCDBManager::Instance()->GetStorage(mainUri)
319                                         ->Put(object, id, metaData);
320         }
321
322         if(!result) {
323
324                 Log(fCurrentDetector,
325                         Form("WriteToCDB - Problem with main %s storage. Putting <%s> into backup storage",
326                                 cdbType, path.GetPath().Data()));
327
328                 // Set Grid version to current run number, to ease retrieval later
329                 id.SetVersion(GetCurrentRun());
330
331                 result = AliCDBManager::Instance()->GetStorage(localUri)
332                                         ->Put(object, id, metaData);
333
334                 if(result) {
335                         result = 2;
336                         fGridError = kTRUE;
337                 }else{
338                         Log(fCurrentDetector, "WriteToCDB - Can't store data!");
339                 }
340         }
341
342         return result;
343
344 }
345
346 //______________________________________________________________________________________________
347 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
348 {
349 // Reads the AliShuttleStatus from the CDB
350
351         if (fStatusEntry){
352                 delete fStatusEntry;
353                 fStatusEntry = 0;
354         }
355
356         fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalCDB())
357                 ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), GetCurrentRun());
358
359         if (!fStatusEntry) return 0;
360         fStatusEntry->SetOwner(1);
361
362         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
363         if (!status) {
364                 AliError("Invalid object stored to CDB!");
365                 return 0;
366         }
367
368         return status;
369 }
370
371 //______________________________________________________________________________________________
372 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
373 {
374 // writes the status for one subdetector
375
376         if (fStatusEntry){
377                 delete fStatusEntry;
378                 fStatusEntry = 0;
379         }
380
381         Int_t run = GetCurrentRun();
382
383         AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), run, run);
384
385         fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
386         fStatusEntry->SetOwner(1);
387
388         UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
389
390         if (!result) {
391                 AliError(Form("WriteShuttleStatus for %s, run %d failed", fCurrentDetector.Data(), run));
392                 return kFALSE;
393         }
394
395         return kTRUE;
396 }
397
398 //______________________________________________________________________________________________
399 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
400 {
401   // changes the AliShuttleStatus for the given detector and run to the given status
402
403         if (!fStatusEntry){
404                 AliError("UNEXPECTED: fStatusEntry empty");
405                 return;
406         }
407
408         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
409
410         if (!status){
411                 AliError("UNEXPECTED: status could not be read from current CDB entry");
412                 return;
413         }
414
415         Log("SHUTTLE", Form("UpdateShuttleStatus - %s: Changing state from %s to %s", fCurrentDetector.Data(),
416                                 status->GetStatusName(), status->GetStatusName(newStatus)));
417
418         status->SetStatus(newStatus);
419         if (increaseCount) status->IncreaseCount();
420
421         AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
422 }
423 //______________________________________________________________________________________________
424 Bool_t AliShuttle::ContinueProcessing()
425 {
426 // this function reads the AliShuttleStatus information from CDB and
427 // checks if the processing should be continued
428 // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
429
430         if(!GetDetCode(fCurrentDetector)) {
431                 Log("SHUTTLE", Form("ContinueProcessing - %s: unknown detector",
432                                 fCurrentDetector.Data()));
433                 return kFALSE;
434         }
435
436         AliShuttleLogbookEntry::Status entryStatus =
437                 fLogbookEntry->GetDetectorStatus(GetDetCode(fCurrentDetector));
438
439         if(entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
440                 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s",
441                                 fCurrentDetector.Data(),
442                                 fLogbookEntry->GetDetectorStatusName(entryStatus)));
443                 return kFALSE;
444         }
445
446         // if we get here, according to Shuttle logbook subdetector is in UNPROCESSED state
447         AliShuttleStatus* status = ReadShuttleStatus();
448         if (!status) {
449                 // first time
450                 Log("SHUTTLE", Form("ContinueProcessing - %s: Processing first time",
451                                 fCurrentDetector.Data()));
452                 status = new AliShuttleStatus(AliShuttleStatus::kStarted);
453                 return WriteShuttleStatus(status);
454         }
455
456         // The following two cases shouldn't happen if Shuttle Logbook was correctly updated.
457         // If it happens it may mean Logbook updating failed... let's do it now!
458         if (status->GetStatus() == AliShuttleStatus::kDone ||
459             status->GetStatus() == AliShuttleStatus::kFailed){
460                 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s. Updating Shuttle Logbook",
461                                         fCurrentDetector.Data(),
462                                         status->GetStatusName(status->GetStatus())));
463                 UpdateShuttleLogbook(fCurrentDetector.Data(),
464                                         status->GetStatusName(status->GetStatus()));
465                 return kFALSE;
466         }
467
468         if (status->GetStatus() == AliShuttleStatus::kStoreFailed) {
469                 Log("SHUTTLE",
470                         Form("ContinueProcessing - %s: Grid storage of one or more objects failed. Trying again now",
471                                 fCurrentDetector.Data()));
472                 if(TryToStoreAgain()){
473                         Log(fCurrentDetector.Data(), "ContinueProcessing - All objects successfully stored into OCDB");
474                         UpdateShuttleStatus(AliShuttleStatus::kDone);
475                         UpdateShuttleLogbook(fCurrentDetector.Data(), "DONE");
476                 } else {
477                         Log("SHUTTLE",
478                                 Form("ContinueProcessing - %s: Grid storage failed again",
479                                         fCurrentDetector.Data()));
480                 }
481                 return kFALSE;
482         }
483
484         // if we get here, there is a restart
485
486         // abort conditions
487         // TODO we should add two counters, one for PP and one for DCS!
488         if ((status->GetStatus() == AliShuttleStatus::kPPStarted ||
489              status->GetStatus() == AliShuttleStatus::kPPError) &&
490             (status->GetCount() >= fConfig->GetMaxPPRetries() ||
491              status->GetCount() >= fConfig->GetMaxRetries())) {
492                 Log("SHUTTLE",
493                         Form("ContinueProcessing - %s failed %d times in status %s - Updating Shuttle Logbook",
494                                 fCurrentDetector.Data(),
495                                 status->GetCount(), status->GetStatusName()));
496                 UpdateShuttleLogbook(fCurrentDetector.Data(), "FAILED");
497                 return kFALSE;
498         }
499
500         Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. Got stuck before in %s. Retry number %d.",
501                         fCurrentDetector.Data(),
502                         status->GetStatusName(), status->GetCount()));
503
504         UpdateShuttleStatus(AliShuttleStatus::kStarted);
505
506         return kTRUE;
507 }
508
509 //______________________________________________________________________________________________
510 Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
511 {
512         //
513         // Makes data retrieval for all detectors in the configuration.
514         // entry: Shuttle logbook entry, contains run paramenters and status of detectors
515         // (Unprocessed, Inactive, Failed or Done).
516         // Returns kFALSE in case of error occured and kTRUE otherwise
517         //
518
519         if(!entry) return kFALSE;
520
521         fLogbookEntry = entry;
522
523         if(fLogbookEntry->IsDone()){
524                 Log("SHUTTLE","Process - Shuttle is already DONE. Updating logbook");
525                 UpdateShuttleLogbook("shuttle_done");
526                 fLogbookEntry = 0;
527                 return kTRUE;
528         }
529
530
531         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^* \n",
532                                         GetCurrentRun()));
533
534         fLogbookEntry->Print("");
535
536         // Initialization
537         Bool_t hasError = kFALSE;
538         for(Int_t iSys=0;iSys<3;iSys++) fFESCalled[iSys]=kFALSE;
539
540         AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
541         if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
542         AliCDBStorage *mainRefSto = AliCDBManager::Instance()->GetStorage(fgkMainRefStorage);
543         if(mainRefSto) mainRefSto->QueryCDB(GetCurrentRun());
544
545         // Loop on detectors in the configuration
546         TIter iter(fConfig->GetDetectors());
547         TObjString* aDetector = 0;
548
549         while ((aDetector = (TObjString*) iter.Next())) {
550                 fCurrentDetector = aDetector->String();
551
552                 if (!fConfig->HostProcessDetector(fCurrentDetector)) continue;
553
554                 AliPreprocessor* aPreprocessor =
555                         dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
556                 if(!aPreprocessor){
557                         Log("SHUTTLE",Form("Process - %s: no preprocessor registered. Skipping"));
558                         continue;
559                 }
560
561                 if (ContinueProcessing() == kFALSE) continue;
562
563                 AliInfo(Form("\n\n \t\t\t****** run %d - %s: START  ******",
564                                                 GetCurrentRun(), aDetector->GetName()));
565
566                 UInt_t result = ProcessCurrentDetector();
567
568                 if(!result) {
569                         hasError = kTRUE;
570                         AliInfo(Form("\n \t\t\t****** run %d - %s: PREPROCESSOR ERROR ****** \n\n",
571                                                         GetCurrentRun(), aDetector->GetName()));
572                         continue;
573                 }
574
575                 if(result == 2) {
576                         AliInfo(Form("\n \t\t\t****** run %d - %s: STORAGE ERROR ****** \n\n",
577                                                         GetCurrentRun(), aDetector->GetName()));
578                 } else {
579                         AliInfo(Form("\n \t\t\t****** run %d - %s: DONE ****** \n\n",
580                                                         GetCurrentRun(), aDetector->GetName()));
581                 }
582
583                 // Process successful: Update time_processed field in FES logbooks!
584                 if(fFESCalled[kDAQ]) {
585                         hasError = (UpdateDAQTable() == kFALSE);
586                         fFESlist[kDAQ].Clear();
587                 }
588                 //if(fFESCalled[kDCS]) {
589                 //      hasError = UpdateDCSTable(aDetector->GetName());
590                 //      fFESlist[kDCS].Clear();
591                 //}
592                 //if(fFESCalled[kHLT]) {
593                 //      hasError = UpdateHLTTable(aDetector->GetName());
594                 //      fFESlist[kHLT].Clear();
595                 //}
596
597         }
598
599         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^* \n",
600                                                         GetCurrentRun()));
601
602         //check if shuttle is done for this run, if so update logbook
603         TObjArray checkEntryArray;
604         checkEntryArray.SetOwner(1);
605         TString whereClause = Form("where run=%d",GetCurrentRun());
606         if(QueryShuttleLogbook(whereClause.Data(), checkEntryArray)) {
607
608                 AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
609                                                         (checkEntryArray.At(0));
610
611                 if(checkEntry && checkEntry->IsDone()){
612                         Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
613                         UpdateShuttleLogbook("shuttle_done");
614                 }
615         }
616
617         fLogbookEntry = 0;
618
619         return hasError == kFALSE;
620 }
621
622 //______________________________________________________________________________________________
623 UInt_t AliShuttle::ProcessCurrentDetector()
624 {
625         //
626         // Makes data retrieval just for a specific detector (fCurrentDetector).
627         // Threre should be a configuration for this detector.
628
629         AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), GetCurrentRun()));
630
631         UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
632
633         TString host(fConfig->GetDCSHost(fCurrentDetector));
634         Int_t port = fConfig->GetDCSPort(fCurrentDetector);
635
636         TIter iter(fConfig->GetDCSAliases(fCurrentDetector));
637         TObjString* anAlias;
638         TMap aliasMap;
639         aliasMap.SetOwner(1);
640
641         Bool_t aDCSError = kFALSE;
642         fGridError = kFALSE;
643
644         while ((anAlias = (TObjString*) iter.Next())) {
645                 TObjArray *valueSet = new TObjArray();
646                 valueSet->SetOwner(1);
647                 // TODO Test only... I've added a flag that allows to
648                 // exclude DCS archive DB query
649                 if(fgkProcessDCS){
650                         AliInfo("Querying DCS archive DB data...");
651                         aDCSError = (GetValueSet(host, port, anAlias->String(), valueSet) == 0);
652                 } else {
653                         AliInfo(Form("Skipping DCS processing. Port = %d",port));
654                         aDCSError = kFALSE;
655                 }
656                 if(!aDCSError) {
657                         aliasMap.Add(anAlias->Clone(), valueSet);
658                 }else{
659                         Log(fCurrentDetector, Form("ProcessCurrentDetector - Error while retrieving alias %s",
660                                         anAlias->GetName()));
661                         UpdateShuttleStatus(AliShuttleStatus::kDCSError, kTRUE);
662                         aliasMap.DeleteAll();
663                         return 0;
664                 }
665         }
666
667         // DCS Archive DB processing successful. Call Preprocessor!
668         UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
669
670         AliPreprocessor* aPreprocessor =
671                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
672
673         aPreprocessor->Initialize(GetCurrentRun(), GetCurrentStartTime(), GetCurrentEndTime());
674
675         UInt_t aPPResult = aPreprocessor->Process(&aliasMap);
676
677         UInt_t returnValue = 0;
678         if (aPPResult == 0) { // Preprocessor error
679                 UpdateShuttleStatus(AliShuttleStatus::kPPError, kTRUE);
680                 returnValue = 0;
681         } else if (fGridError == kFALSE) { // process and Grid storage ok!
682                 UpdateShuttleStatus(AliShuttleStatus::kDone);
683                 UpdateShuttleLogbook(fCurrentDetector, "DONE");
684                 Log(fCurrentDetector.Data(),
685                         "ProcessCurrentDetector - Preprocessor and Grid storage ended successfully");
686                 returnValue = 1;
687         } else { // Grid storage error (process ok, but object put in local storage)
688                 UpdateShuttleStatus(AliShuttleStatus::kStoreFailed);
689                 returnValue = 2;
690         }
691
692         aliasMap.DeleteAll();
693
694         return returnValue;
695 }
696
697 //______________________________________________________________________________________________
698 Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
699                 TObjArray& entries)
700 {
701 // Query DAQ's Shuttle logbook and fills detector status object.
702 // Call QueryRunParameters to query DAQ logbook for run parameters.
703
704         // check connection, in case connect
705         if(!Connect(kDAQ)) return kFALSE;
706
707         TString sqlQuery;
708         sqlQuery = Form("select * from logbook_shuttle %s order by run", whereClause);
709
710         TSQLResult* aResult = fServer[kDAQ]->Query(sqlQuery);
711         if (!aResult) {
712                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
713                 return kFALSE;
714         }
715
716         if(aResult->GetRowCount() == 0) {
717                 if(sqlQuery.Contains("where shuttle_done=0")){
718                         Log("SHUTTLE", "QueryShuttleLogbook - All runs in Shuttle Logbook are already DONE");
719                         delete aResult;
720                         return kTRUE;
721                 } else {
722                         AliError("No entries in Shuttle Logbook match request");
723                         delete aResult;
724                         return kFALSE;
725                 }
726         }
727
728         // TODO Check field count!
729         const UInt_t nCols = 24;
730         if (aResult->GetFieldCount() != (Int_t) nCols) {
731                 AliError("Invalid SQL result field number!");
732                 delete aResult;
733                 return kFALSE;
734         }
735
736         entries.SetOwner(1);
737
738         TSQLRow* aRow;
739         while ((aRow = aResult->Next())) {
740                 TString runString(aRow->GetField(0), aRow->GetFieldLength(0));
741                 Int_t run = runString.Atoi();
742
743                 UInt_t startTime, endTime;
744                 if(!QueryRunParameters(run, startTime, endTime)) continue;
745
746                 const UInt_t nDet = AliShuttle::kNDetectors;
747                 AliShuttleLogbookEntry::Status detStatus[nDet];
748
749                 // loop on detectors
750                 for(UInt_t ii = 0; ii < nCols; ii++){
751                         TString detCode(aResult->GetFieldName(ii));
752                         Int_t detPos = AliShuttle::GetDetPos(detCode.Data());
753                         if(detPos < 0) continue;
754                         TString statusString(aRow->GetField(ii), aRow->GetFieldLength(ii));
755                         if(statusString == "UNPROCESSED"){
756                                 detStatus[detPos] = AliShuttleLogbookEntry::kUnprocessed;
757                         } else if (statusString == "INACTIVE") {
758                                 detStatus[detPos] = AliShuttleLogbookEntry::kInactive;
759                         } else if (statusString == "FAILED") {
760                                 detStatus[detPos] = AliShuttleLogbookEntry::kFailed;
761                         } else if (statusString == "DONE") {
762                                 detStatus[detPos] = AliShuttleLogbookEntry::kDone;
763                         }
764                 }
765
766                 entries.AddLast(new AliShuttleLogbookEntry(run, startTime, endTime, detStatus));
767                 delete aRow;
768         }
769
770         if(sqlQuery.Contains("where shuttle_done=0"))
771                 Log("SHUTTLE", Form("QueryShuttleLogbook - Found %d unprocessed runs in Shuttle Logbook",
772                                                         entries.GetEntriesFast()));
773         delete aResult;
774         return kTRUE;
775 }
776
777 //______________________________________________________________________________________________
778 Bool_t AliShuttle::QueryRunParameters(Int_t& run, UInt_t& startTime, UInt_t& endTime)
779 {
780 // Retrieve start time and end time for run in the DAQ logbook
781
782         // check connection, in case connect
783         if(!Connect(kDAQ)) return kFALSE;
784
785         TString sqlQuery;
786         sqlQuery = Form("select time_start, time_end from logbook where run=%d", run);
787
788         TSQLResult* aResult = fServer[kDAQ]->Query(sqlQuery);
789         if (!aResult) {
790                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
791                 return kFALSE;
792         }
793
794         if(aResult->GetRowCount() == 0) {
795                 Log("SHUTTLE", Form("QueryRunParameters - No entry in DAQ Logbook for run %d. Skipping", run));
796                 delete aResult;
797                 return kFALSE;
798         }
799
800         if(aResult->GetRowCount() > 1) {
801                 AliError(Form("More than one entry in DAQ Logbook for run %d. Skipping", run));
802                 delete aResult;
803                 return kFALSE;
804         }
805
806         TSQLRow* aRow;
807         while ((aRow = aResult->Next())) {
808
809                 TString startTimeString(aRow->GetField(0),
810                                 aRow->GetFieldLength(0));
811                 startTime = startTimeString.Atoi();
812                 TString endTimeString(aRow->GetField(1),
813                                 aRow->GetFieldLength(1));
814                 endTime = endTimeString.Atoi();
815
816                 if (!startTime || !endTime || startTime > endTime) {
817                         Log("SHUTTLE",
818                                 Form("QueryRunParameters - Invalid parameters for Run %d: startTime = %d, endTime = %d",
819                                         run, startTime, endTime));
820                         delete aRow;
821                         delete aResult;
822                         return kFALSE;
823                 }
824
825                 delete aRow;
826         }
827
828         delete aResult;
829         return kTRUE;
830 }
831
832 //______________________________________________________________________________________________
833 Bool_t AliShuttle::TryToStoreAgain()
834 {
835   // Called in case the detector failed to store the object in Grid OCDB
836   // It tries to store the object again, if it does not find more recent and overlapping objects
837   // Calls underlying TryToStoreAgain(const char*) function twice, for OCDB and Reference storage.
838
839         AliInfo("Trying to store OCDB data again...");
840         Bool_t resultCDB = TryToStoreAgain(fgkMainCDB);
841
842         AliInfo("Trying to store reference data again...");
843         Bool_t resultRef = TryToStoreAgain(fgkMainRefStorage);
844
845         return resultCDB && resultRef;
846 }
847
848 //______________________________________________________________________________________________
849 Bool_t AliShuttle::TryToStoreAgain(TString& gridURI)
850 {
851   // Called by TryToStoreAgain(), performs actual storage retry
852
853         TObjArray* gridIds=0;
854
855         Bool_t result = kTRUE;
856
857         const char* type = 0;
858         TString backupURI;
859         if(gridURI == fgkMainCDB) {
860                 type = "OCDB";
861                 backupURI = fgkLocalCDB;
862         } else if(gridURI == fgkMainRefStorage) {
863                 type = "reference";
864                 backupURI = fgkLocalRefStorage;
865         } else {
866                 AliError(Form("Invalid storage URI: %s", gridURI.Data()));
867                 return kFALSE;
868         }
869
870         AliCDBManager* man = AliCDBManager::Instance();
871
872         AliCDBStorage *gridSto = man->GetStorage(gridURI);
873         if(!gridSto) {
874                 Log(fCurrentDetector.Data(),
875                         Form("TryToStoreAgain - cannot activate main %s storage", type));
876                 return kFALSE;
877         }
878
879         gridIds = gridSto->GetQueryCDBList();
880
881         // get objects previously stored in local CDB
882         AliCDBStorage *backupSto = man->GetStorage(backupURI);
883         AliCDBPath aPath(fCurrentDetector,"*","*");
884         // Local objects were stored with current run as Grid version!
885         TList* localEntries = backupSto->GetAll(aPath.GetPath(), GetCurrentRun(), GetCurrentRun());
886         localEntries->SetOwner(1);
887
888         // loop on local stored objects
889         TIter localIter(localEntries);
890         AliCDBEntry *aLocEntry = 0;
891         while((aLocEntry = dynamic_cast<AliCDBEntry*> (localIter.Next()))){
892                 aLocEntry->SetOwner(1);
893                 AliCDBId aLocId = aLocEntry->GetId();
894                 aLocEntry->SetVersion(-1);
895                 aLocEntry->SetSubVersion(-1);
896
897                 // loop on Grid valid Id's
898                 Bool_t store = kTRUE;
899                 TIter gridIter(gridIds);
900                 AliCDBId* aGridId = 0;
901                 while((aGridId = dynamic_cast<AliCDBId*> (gridIter.Next()))){
902                         // If local object is valid up to infinity we store it anyway
903                         // TODO This does not work! It may hide more recent objects...
904                         if(aLocId.GetLastRun() == AliCDBRunRange::Infinity()) {
905                                 // TODO Check that it won't hide more recent files! how????
906                                 break;
907                         }
908                         if(aGridId->GetPath() != aLocId.GetPath()) continue;
909                         // skip all objects valid up to infinity
910                         if(aGridId->GetLastRun() == AliCDBRunRange::Infinity()) continue;
911                         // if we get here, it means there's already some more recent object stored on Grid!
912                         store = kFALSE;
913                         break;
914                 }
915
916                 if(!store){
917                         Log(fCurrentDetector.Data(),
918                                 Form("TryToStoreAgain - A more recent object already exists in %s storage: <%s>",
919                                         type, aGridId->ToString().Data()));
920                         // removing local filename...
921                         // TODO maybe it's better not to remove it, it was not copied to the Grid!
922                         TString filename;
923                         backupSto->IdToFilename(aLocId, filename);
924                         AliInfo(Form("Removing local file %s", filename.Data()));
925                         gSystem->Exec(Form("rm %s",filename.Data()));
926                         continue;
927                 }
928
929                 // If we get here, the file can be stored!
930                 Bool_t storeOk = gridSto->Put(aLocEntry);
931                 if(storeOk){
932                         Log(fCurrentDetector.Data(),
933                                 Form("TryToStoreAgain - Object <%s> successfully put into %s storage",
934                                         aLocId.ToString().Data(), type));
935
936                         // removing local filename...
937                         TString filename;
938                         backupSto->IdToFilename(aLocId, filename);
939                         AliInfo(Form("Removing local file %s", filename.Data()));
940                         gSystem->Exec(Form("rm %s", filename.Data()));
941                         continue;
942                 } else  {
943                         Log(fCurrentDetector.Data(),
944                                 Form("TryToStoreAgain - Grid %s storage of object <%s> failed again",
945                                         type, aLocId.ToString().Data()));
946                         result = kFALSE;
947                 }
948         }
949         localEntries->Clear();
950
951         return result;
952 }
953
954 //______________________________________________________________________________________________
955 Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* alias,
956                                 TObjArray* valueSet)
957 {
958 // Retrieve all "alias" data points from the DCS server
959 // host, port: TSocket connection parameters
960 // alias: name of the alias
961 // valueSet: array of retrieved AliDCSValue's
962
963         AliDCSClient client(host, port, fTimeout, fRetries);
964         if (!client.IsConnected()) {
965                 return kFALSE;
966         }
967
968         Int_t result = client.GetAliasValues(alias,
969                 GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
970
971         if (result < 0) {
972                 Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get '%s'! Reason: %s",
973                         alias, AliDCSClient::GetErrorString(result)));
974
975                 if (result == AliDCSClient::fgkServerError) {
976                         Log(fCurrentDetector.Data(), Form("GetValueSet - Server error: %s",
977                                 client.GetServerError().Data()));
978                 }
979
980                 return kFALSE;
981         }
982
983         return kTRUE;
984 }
985
986 //______________________________________________________________________________________________
987 const char* AliShuttle::GetFile(Int_t system, const char* detector,
988                 const char* id, const char* source)
989 {
990 // Get calibration file from file exchange servers
991 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
992
993         switch(system){
994                 case kDAQ:
995                         return GetDAQFileName(detector, id, source);
996                         break;
997                 case kDCS:
998                         return GetDCSFileName(detector, id, source);
999                         break;
1000                 case kHLT:
1001                         return GetHLTFileName(detector, id, source);
1002                         break;
1003                 default:
1004                         AliError(Form("No valid system index: %d",system));
1005         }
1006
1007         return 0;
1008 }
1009
1010 //______________________________________________________________________________________________
1011 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
1012 {
1013 // Get sources producing the condition file Id from file exchange servers
1014 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
1015
1016         switch(system){
1017                 case kDAQ:
1018                         return GetDAQFileSources(detector, id);
1019                         break;
1020                 case kDCS:
1021                         return GetDCSFileSources(detector, id);
1022                         break;
1023                 case kHLT:
1024                         return GetHLTFileSources(detector, id);
1025                         break;
1026                 default:
1027                         AliError(Form("No valid system index: %d",system));
1028         }
1029
1030         return NULL;
1031 }
1032
1033 //______________________________________________________________________________________________
1034 Bool_t AliShuttle::Connect(Int_t system)
1035 {
1036 // Connect to MySQL Server of the system's FES logbook
1037 // DAQ Logbook, Shuttle Logbook and DAQ FES Logbook are on the same host
1038
1039         // check connection: if already connected return
1040         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
1041
1042         TString aFESlbHost= Form("mysql://%s", fConfig->GetFESlbHost(system));
1043
1044         fServer[system] = TSQLServer::Connect(aFESlbHost,
1045                         fConfig->GetFESlbUser(system),
1046                         fConfig->GetFESlbPass(system));
1047         if (!fServer[system] || !fServer[system]->IsConnected()) {
1048                 AliError(Form("Can't establish connection to FES logbook for %s",fkSystemNames[system]));
1049                 if(fServer[system]) delete fServer[system];
1050                 return kFALSE;
1051         }
1052
1053         // Get tables
1054         // TODO in the configuration should the table name be there too?
1055         TSQLResult* aResult=0;
1056         switch(system){
1057                 case kDAQ:
1058                         aResult = fServer[kDAQ]->GetTables("REFSYSLOG");
1059                         break;
1060                 case kDCS:
1061                         //aResult = fServer[kDCS]->GetTables("REFSYSLOG");
1062                         break;
1063                 case kHLT:
1064                         //aResult = fServer[kHLT]->GetTables("REFSYSLOG");
1065                         break;
1066                 default:
1067                         break;
1068         }
1069
1070         delete aResult;
1071         return kTRUE;
1072 }
1073
1074 //______________________________________________________________________________________________
1075 const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source)
1076 {
1077 // Retrieves a file from the DAQ FES.
1078 // First queris the DAQ logbook_fs for the DAQ file name, using the run, detector, id and source info
1079 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
1080 // run: current run being processed (given by Logbook entry fLogbookEntry)
1081 // detector: comes from the Preprocessor name (must be converted into detector code with GetDetCode)
1082 // id: provided as a parameter by the Preprocessor
1083 // source: provided by the Preprocessor through GetFileSources function
1084
1085         // check connection, in case connect
1086         if(!Connect(kDAQ)){
1087                 Log(detector, "GetDAQFileName - Couldn't connect to DAQ Logbook");
1088                 return 0;
1089         }
1090
1091         // Query preparation
1092         TString sqlQueryStart = "select filePath from logbook_fs where";
1093         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
1094                                 GetCurrentRun(), GetDetCode(detector), id, source);
1095         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1096
1097         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1098
1099         // Query execution
1100         TSQLResult* aResult = 0;
1101         aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1102         if (!aResult) {
1103                 Log(detector, Form("GetDAQFileName - Can't execute SQL query for: id = %s, source = %s",
1104                                 id, source));
1105                 return 0;
1106         }
1107
1108         if (aResult->GetRowCount() == 0) {
1109                 Log(detector,
1110                         Form("GetDAQFileName - No entry in FES table for: id = %s, source = %s",
1111                                 id, source));
1112                 delete aResult;
1113                 return 0;
1114         }
1115
1116         if (aResult->GetRowCount() >1) {
1117                 Log(detector,
1118                         Form("GetDAQFileName - More than one entry in FES table for: id = %s, source = %s",
1119                                 id, source));
1120                 delete aResult;
1121                 return 0;
1122         }
1123
1124         TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
1125
1126         if(!aRow){
1127                 Log(detector, Form("GetDAQFileName - Empty set result from query: id = %s, source = %s",
1128                                 id, source));
1129                 delete aResult;
1130                 return 0;
1131         }
1132
1133         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
1134
1135         delete aResult;
1136         delete aRow;
1137
1138         AliDebug(2, Form("filePath = %s",filePath.Data()));
1139
1140         // retrieved file is renamed to make it unique
1141         TString localFileName = Form("%s_%d_%s_%s.shuttle",
1142                                         detector, GetCurrentRun(), id, source);
1143
1144         // file retrieval from DAQ FES
1145         Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
1146         if(!result) {
1147                 Log(detector, Form("GetDAQFileName - Copy of file %s from DAQ FES failed", filePath.Data()));
1148                 return 0;
1149         } else {
1150                 AliInfo(Form("File %s copied from DAQ FES into %s/%s",
1151                         filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
1152         }
1153
1154
1155         fFESCalled[kDAQ]=kTRUE;
1156         TObjString *fileParams = new TObjString(Form("%s_!?!_%s", id, source));
1157         fFESlist[kDAQ].Add(fileParams);
1158
1159         return localFileName.Data();
1160
1161 }
1162
1163 //______________________________________________________________________________________________
1164 Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName)
1165 {
1166
1167         // check temp directory: trying to cd to temp; if it does not exist, create it
1168         AliDebug(2, Form("Copy file %s from DAQ FES into folder %s and rename it as %s",
1169                         daqFileName,fgkShuttleTempDir, localFileName));
1170
1171         void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
1172         if (dir == NULL) {
1173                 if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
1174                         AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
1175                         return kFALSE;
1176                 }
1177
1178         } else {
1179                 gSystem->FreeDirectory(dir);
1180         }
1181
1182         TString baseDAQFESFolder = "DAQ";
1183         TString command = Form("scp %s@%s:%s/%s %s/%s",
1184                 fConfig->GetFESUser(kDAQ),
1185                 fConfig->GetFESHost(kDAQ),
1186                 baseDAQFESFolder.Data(),
1187                 daqFileName,
1188                 fgkShuttleTempDir,
1189                 localFileName);
1190
1191         AliDebug(2, Form("%s",command.Data()));
1192
1193         UInt_t nRetries = 0;
1194         UInt_t maxRetries = 3;
1195
1196         // copy!! if successful TSystem::Exec returns 0
1197         while(nRetries++ < maxRetries) {
1198                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
1199                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
1200         }
1201
1202         return kFALSE;
1203
1204 }
1205
1206 //______________________________________________________________________________________________
1207 TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id)
1208 {
1209 // Retrieves a file from the DCS FES.
1210
1211         // check connection, in case connect
1212         if(!Connect(kDAQ)){
1213                 Log(detector, "GetDAQFileSources - Couldn't connect to DAQ Logbook");
1214                 return 0;
1215         }
1216
1217         // Query preparation
1218         TString sqlQueryStart = "select DAQsource from logbook_fs where";
1219         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
1220                                 GetCurrentRun(), GetDetCode(detector), id);
1221         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1222
1223         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1224
1225         // Query execution
1226         TSQLResult* aResult;
1227         aResult = fServer[kDAQ]->Query(sqlQuery);
1228         if (!aResult) {
1229                 Log(detector, Form("GetDAQFileSources - Can't execute SQL query for id: %s", id));
1230                 return 0;
1231         }
1232
1233         if (aResult->GetRowCount() == 0) {
1234                 Log(detector,
1235                         Form("GetDAQFileSources - No entry in FES table for id: %s", id));
1236                 delete aResult;
1237                 return 0;
1238         }
1239
1240         TSQLRow* aRow;
1241         TList *list = new TList();
1242         list->SetOwner(1);
1243
1244         while((aRow = aResult->Next())){
1245
1246                 TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
1247                 AliDebug(2, Form("daqSource = %s", daqSource.Data()));
1248                 list->Add(new TObjString(daqSource));
1249                 delete aRow;
1250         }
1251         delete aResult;
1252
1253         return list;
1254
1255 }
1256
1257 //______________________________________________________________________________________________
1258 const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1259 // Retrieves a file from the DCS FES.
1260
1261 return "You're in DCS";
1262
1263 }
1264
1265 //______________________________________________________________________________________________
1266 TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
1267 // Retrieves a file from the DCS FES.
1268
1269 return NULL;
1270
1271 }
1272
1273 //______________________________________________________________________________________________
1274 const char* AliShuttle::GetHLTFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1275 // Retrieves a file from the HLT FES.
1276
1277 return "You're in HLT";
1278
1279 }
1280
1281 //______________________________________________________________________________________________
1282 TList* AliShuttle::GetHLTFileSources(const char* /*detector*/, const char* /*id*/){
1283 // Retrieves a file from the HLT FES.
1284
1285 return NULL;
1286
1287 }
1288
1289 //______________________________________________________________________________________________
1290 Bool_t AliShuttle::UpdateDAQTable()
1291 {
1292 // Update DAQ table filling time_processed field in all rows corresponding to current run and detector
1293
1294         // check connection, in case connect
1295         if(!Connect(kDAQ)){
1296                 Log(fCurrentDetector, "UpdateDAQTable - Couldn't connect to DAQ Logbook");
1297                 return kFALSE;
1298         }
1299
1300         TTimeStamp now; // now
1301
1302         // Loop on FES list entries
1303         TIter iter(&fFESlist[kDAQ]);
1304         TObjString *aFESentry=0;
1305         while((aFESentry = dynamic_cast<TObjString*> (iter.Next()))){
1306                 TString aFESentrystr = aFESentry->String();
1307                 TObjArray *aFESarray = aFESentrystr.Tokenize("_!?!_");
1308                 if(!aFESarray || aFESarray->GetEntries() != 2 ) {
1309                         Log(fCurrentDetector, Form("UpdateDAQTable - error updating FES entry. Check string: <%s>",
1310                                 aFESentrystr.Data()));
1311                         if(aFESarray) delete aFESarray;
1312                         return kFALSE;
1313                 }
1314                 const char* fileId = ((TObjString*) aFESarray->At(0))->GetName();
1315                 const char* daqSource = ((TObjString*) aFESarray->At(1))->GetName();
1316                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
1317                         GetCurrentRun(), GetDetCode(fCurrentDetector), fileId, daqSource);
1318
1319                 delete aFESarray;
1320
1321                 TString sqlQuery = Form("update logbook_fs set time_processed=%d %s", now.GetSec(), whereClause.Data());
1322
1323                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1324
1325                 // Query execution
1326                 TSQLResult* aResult;
1327                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1328                 if (!aResult) {
1329                         Log(fCurrentDetector, Form("UpdateDAQTable - Can't execute SQL query <%s>", sqlQuery.Data()));
1330                         return kFALSE;
1331                 }
1332                 delete aResult;
1333         }
1334
1335         return kTRUE;
1336 }
1337
1338
1339 //______________________________________________________________________________________________
1340 Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status)
1341 {
1342 // Update Shuttle logbook filling detector or shuttle_done column
1343 // ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
1344
1345         // check connection, in case connect
1346         if(!Connect(kDAQ)){
1347                 Log("SHUTTLE", "UpdateShuttleLogbook - Couldn't connect to DAQ Logbook.");
1348                 return kFALSE;
1349         }
1350
1351         TString detName(detector);
1352         TString setClause;
1353         if(detName == "shuttle_done") {
1354                 setClause = "set shuttle_done=1";
1355         } else {
1356                 TString detCode = GetDetCode(detector);
1357                 if(detCode.IsNull()) {
1358                         Log("SHUTTLE", Form("UpdateShuttleLogbook - Unknown detector %s", detector));
1359                         return kFALSE;
1360                 }
1361                 TString statusStr(status);
1362                 if(statusStr.Contains("done", TString::kIgnoreCase) ||
1363                    statusStr.Contains("failed", TString::kIgnoreCase)){
1364                         setClause = Form("set %s=\"%s\"", detCode.Data(), status);
1365                 } else {
1366                         Log("SHUTTLE",
1367                                 Form("UpdateShuttleLogbook - Invalid status <%s> for detector %s",
1368                                         status, detector));
1369                         return kFALSE;
1370                 }
1371         }
1372
1373         TString whereClause = Form("where run=%d", GetCurrentRun());
1374
1375         TString sqlQuery = Form("update logbook_shuttle %s %s",
1376                                         setClause.Data(), whereClause.Data());
1377
1378         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1379
1380         // Query execution
1381         TSQLResult* aResult;
1382         aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1383         if (!aResult) {
1384                 Log("SHUTTLE", Form("UpdateShuttleLogbook - Can't execute query <%s>", sqlQuery.Data()));
1385                 return kFALSE;
1386         }
1387         delete aResult;
1388
1389         return kTRUE;
1390 }
1391
1392 //______________________________________________________________________________________________
1393 Int_t AliShuttle::GetCurrentRun() const
1394 {
1395 // Get current run from logbook entry
1396
1397         return fLogbookEntry ? fLogbookEntry->GetRun() : -1;
1398 }
1399
1400 //______________________________________________________________________________________________
1401 UInt_t AliShuttle::GetCurrentStartTime() const
1402 {
1403 // get current start time
1404
1405         return fLogbookEntry ? fLogbookEntry->GetStartTime() : 0;
1406 }
1407
1408 //______________________________________________________________________________________________
1409 UInt_t AliShuttle::GetCurrentEndTime() const
1410 {
1411 // get current end time from logbook entry
1412
1413         return fLogbookEntry ? fLogbookEntry->GetEndTime() : 0;
1414 }
1415
1416 //______________________________________________________________________________________________
1417 const char* AliShuttle::GetDetCode(const char* detector){
1418 // Return detector code
1419
1420         for(UInt_t iDet=0; iDet < kNDetectors; iDet++){
1421                 if(!strcmp(fgkDetectorName[iDet], detector)) return fgkDetectorCode[iDet];
1422         }
1423
1424         AliErrorClass(Form("Unknown detector: %s",detector));
1425         return 0;
1426 }
1427
1428 //______________________________________________________________________________________________
1429 const char* AliShuttle::GetDetCode(UInt_t detPos){
1430 // Return detector code
1431
1432         if( detPos >= kNDetectors) {
1433                 AliErrorClass(Form("Invalid parameter: %d", detPos));
1434                 return 0;
1435         }
1436         return fgkDetectorCode[detPos];
1437 }
1438
1439 //______________________________________________________________________________________________
1440 const Int_t AliShuttle::GetDetPos(const char* detCode){
1441 // Return detector position in the detector code array
1442
1443         for(UInt_t iDet=0; iDet < kNDetectors; iDet++){
1444                 if(!strcmp(fgkDetectorCode[iDet], detCode)) return iDet;
1445         }
1446         return -1;
1447 }
1448
1449 //______________________________________________________________________________________________
1450 void AliShuttle::Log(const char* detector, const char* message)
1451 {
1452 // Fill log string with a message
1453
1454         void* dir = gSystem->OpenDirectory(fgkShuttleLogDir);
1455         if (dir == NULL) {
1456                 if (gSystem->mkdir(fgkShuttleLogDir, kTRUE)) {
1457                         AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
1458                         return;
1459                 }
1460
1461         } else {
1462                 gSystem->FreeDirectory(dir);
1463         }
1464
1465         TString toLog = Form("%s: %s - ", TTimeStamp(time(0)).AsString("s"), detector);
1466         if(GetCurrentRun()>=0 ) toLog += Form("run %d - ", GetCurrentRun());
1467         toLog += Form("%s", message);
1468
1469         AliInfo(toLog.Data());
1470
1471         TString fileName;
1472         fileName.Form("%s/%s.log", fgkShuttleLogDir, detector);
1473         gSystem->ExpandPathName(fileName);
1474
1475         ofstream logFile;
1476         logFile.open(fileName, ofstream::out | ofstream::app);
1477
1478         if (!logFile.is_open()) {
1479                 AliError(Form("Could not open file %s", fileName.Data()));
1480                 return;
1481         }
1482
1483         logFile << toLog.Data() << "\n";
1484
1485         logFile.close();
1486 }
1487
1488
1489 //______________________________________________________________________________________________
1490 Bool_t AliShuttle::Collect(Int_t run)
1491 {
1492         //
1493         // Collects conditions data for the given run.
1494         //
1495
1496         AliInfo(Form("Collecting conditions data for run %d", run));
1497
1498         TString whereClause("where run=");
1499         whereClause += run;
1500
1501         TObjArray dateEntries;
1502         if (!QueryShuttleLogbook(whereClause, dateEntries)) {
1503                 AliError("Can't retrieve entries from Shuttle logbook!");
1504                 return kFALSE;
1505         }
1506
1507         if (!dateEntries.GetEntriesFast()) {
1508                 AliError(Form("Retrieval of parameters for run %d failed!", run));
1509                 return kFALSE;
1510         }
1511
1512         if (dateEntries.GetEntriesFast() > 1) {
1513                 AliError(Form("There is more than one entry for run <%d> in Shuttle logbook!", run));
1514                 return kFALSE;
1515         }
1516
1517         if (!RetrieveConditionsData(dateEntries)) {
1518                 AliError("An error occured during conditions data retrieval!");
1519                 return kFALSE;
1520         }
1521
1522         return kTRUE;
1523 }
1524
1525 //______________________________________________________________________________________________
1526 Bool_t AliShuttle::CollectNew()
1527 {
1528         //
1529         // Collects conditions data for all UNPROCESSED run written to DAQ LogBook.
1530         // In operational mode, this is the Shuttle function triggered by the EOR signal.
1531         //
1532
1533         Log("SHUTTLE","CollectNew - Shuttle called. Collecting conditions data for unprocessed runs");
1534
1535         TString whereClause("where shuttle_done=0");
1536
1537         TObjArray shuttleLogbookEntries;
1538         if (!QueryShuttleLogbook(whereClause, shuttleLogbookEntries)) {
1539                 Log("SHUTTLE", "CollectNew - Can't retrieve entries from Shuttle logbook");
1540                 return kFALSE;
1541         }
1542
1543         if (!RetrieveConditionsData(shuttleLogbookEntries)) {
1544                 Log("SHUTTLE", "CollectNew - Process of at least one run failed");
1545                 return kFALSE;
1546         }
1547
1548         return kTRUE;
1549 }
1550
1551 //______________________________________________________________________________________________
1552 Bool_t AliShuttle::CollectAll()
1553 {
1554         //
1555         // Collects conditions data for all runs (even if they're already done!) written in Shuttle LogBook.
1556         //
1557
1558         AliInfo("Collecting conditions data for all runs ...");
1559
1560         TObjArray dateEntries;
1561         if (!QueryShuttleLogbook("", dateEntries)) {
1562                 AliError("Can't retrieve entries from Shuttle logbook");
1563                 return kFALSE;
1564         }
1565
1566         if (!RetrieveConditionsData(dateEntries)) {
1567                 AliError("An error occured during conditions data retrieval!");
1568                 return kFALSE;
1569         }
1570
1571         return kTRUE;
1572 }
1573
1574
1575 //______________________________________________________________________________________________
1576 Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
1577 {
1578 // Retrieve conditions data for all runs that aren't processed yet
1579
1580         Bool_t hasError = kFALSE;
1581
1582         TIter iter(&dateEntries);
1583         AliShuttleLogbookEntry* anEntry;
1584
1585         while ((anEntry = (AliShuttleLogbookEntry*) iter.Next())){
1586                 if (!Process(anEntry)){
1587                         hasError = kTRUE;
1588                 }
1589         }
1590
1591         return hasError == kFALSE;
1592 }