2fc32d1453b32e58bcc809699da9648faa6cae86
[u/mrichter/AliRoot.git] / SHUTTLE / AliShuttle.cxx
1 /**************************************************************************
2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
3  *                                                                        *
4  * Author: The ALICE Off-line Project.                                    *
5  * Contributors are mentioned in the code where appropriate.              *
6  *                                                                        *
7  * Permission to use, copy, modify and distribute this software and its   *
8  * documentation strictly for non-commercial purposes is hereby granted   *
9  * without fee, provided that the above copyright notice appears in all   *
10  * copies and that both the copyright notice and this permission notice   *
11  * appear in the supporting documentation. The authors make no claims     *
12  * about the suitability of this software for any purpose. It is          *
13  * provided "as is" without express or implied warranty.                  *
14  **************************************************************************/
15
16 /*
17 $Log$
18 Revision 1.20  2006/11/16 16:16:48  jgrosseo
19 introducing strict run ordering flag
20 removed giving preprocessor name to preprocessor, they have to know their name themselves ;-)
21
22 Revision 1.19  2006/11/06 14:23:04  jgrosseo
23 major update (Alberto)
24 o) reading of run parameters from the logbook
25 o) online offline naming conversion
26 o) standalone DCSclient package
27
28 Revision 1.18  2006/10/20 15:22:59  jgrosseo
29 o) Adding time out to the execution of the preprocessors: The Shuttle forks and the parent process monitors the child
30 o) Merging Collect, CollectAll, CollectNew function
31 o) Removing implementation of empty copy constructors (declaration still there!)
32
33 Revision 1.17  2006/10/05 16:20:55  jgrosseo
34 adapting to new CDB classes
35
36 Revision 1.16  2006/10/05 15:46:26  jgrosseo
37 applying to the new interface
38
39 Revision 1.15  2006/10/02 16:38:39  jgrosseo
40 update (alberto):
41 fixed memory leaks
42 storing of objects that failed to be stored to the grid before
43 interfacing of shuttle status table in daq system
44
45 Revision 1.14  2006/08/29 09:16:05  jgrosseo
46 small update
47
48 Revision 1.13  2006/08/15 10:50:00  jgrosseo
49 effc++ corrections (alberto)
50
51 Revision 1.12  2006/08/08 14:19:29  jgrosseo
52 Update to shuttle classes (Alberto)
53
54 - Possibility to set the full object's path in the Preprocessor's and
55 Shuttle's  Store functions
56 - Possibility to extend the object's run validity in the same classes
57 ("startValidity" and "validityInfinite" parameters)
58 - Implementation of the StoreReferenceData function to store reference
59 data in a dedicated CDB storage.
60
61 Revision 1.11  2006/07/21 07:37:20  jgrosseo
62 last run is stored after each run
63
64 Revision 1.10  2006/07/20 09:54:40  jgrosseo
65 introducing status management: The processing per subdetector is divided into several steps,
66 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
67 can keep track of the number of failures and skips further processing after a certain threshold is
68 exceeded. These thresholds can be configured in LDAP.
69
70 Revision 1.9  2006/07/19 10:09:55  jgrosseo
71 new configuration, accesst to DAQ FES (Alberto)
72
73 Revision 1.8  2006/07/11 12:44:36  jgrosseo
74 adding parameters for extended validity range of data produced by preprocessor
75
76 Revision 1.7  2006/07/10 14:37:09  jgrosseo
77 small fix + todo comment
78
79 Revision 1.6  2006/07/10 13:01:41  jgrosseo
80 enhanced storing of last sucessfully processed run (alberto)
81
82 Revision 1.5  2006/07/04 14:59:57  jgrosseo
83 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
84
85 Revision 1.4  2006/06/12 09:11:16  jgrosseo
86 coding conventions (Alberto)
87
88 Revision 1.3  2006/06/06 14:26:40  jgrosseo
89 o) removed files that were moved to STEER
90 o) shuttle updated to follow the new interface (Alberto)
91
92 Revision 1.2  2006/03/07 07:52:34  hristov
93 New version (B.Yordanov)
94
95 Revision 1.6  2005/11/19 17:19:14  byordano
96 RetrieveDATEEntries and RetrieveConditionsData added
97
98 Revision 1.5  2005/11/19 11:09:27  byordano
99 AliShuttle declaration added
100
101 Revision 1.4  2005/11/17 17:47:34  byordano
102 TList changed to TObjArray
103
104 Revision 1.3  2005/11/17 14:43:23  byordano
105 import to local CVS
106
107 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
108 Initial import as subdirectory in AliRoot
109
110 Revision 1.2  2005/09/13 08:41:15  byordano
111 default startTime endTime added
112
113 Revision 1.4  2005/08/30 09:13:02  byordano
114 some docs added
115
116 Revision 1.3  2005/08/29 21:15:47  byordano
117 some docs added
118
119 */
120
121 //
122 // This class is the main manager for AliShuttle. 
123 // It organizes the data retrieval from DCS and call the 
124 // interface methods of AliPreprocessor.
125 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
126 // data for its set of aliases is retrieved. If there is registered
127 // AliPreprocessor for this detector then it will be used
128 // accroding to the schema (see AliPreprocessor).
129 // If there isn't registered AliPreprocessor than the retrieved
130 // data is stored automatically to the undelying AliCDBStorage.
131 // For detSpec is used the alias name.
132 //
133
134 #include "AliShuttle.h"
135
136 #include "AliCDBManager.h"
137 #include "AliCDBStorage.h"
138 #include "AliCDBId.h"
139 #include "AliCDBRunRange.h"
140 #include "AliCDBPath.h"
141 #include "AliCDBEntry.h"
142 #include "AliShuttleConfig.h"
143 #include "DCSClient/AliDCSClient.h"
144 #include "AliLog.h"
145 #include "AliPreprocessor.h"
146 #include "AliShuttleStatus.h"
147 #include "AliShuttleLogbookEntry.h"
148
149 #include <TSystem.h>
150 #include <TObject.h>
151 #include <TString.h>
152 #include <TTimeStamp.h>
153 #include <TObjString.h>
154 #include <TSQLServer.h>
155 #include <TSQLResult.h>
156 #include <TSQLRow.h>
157 #include <TMutex.h>
158
159 #include <fstream>
160
161 #include <sys/types.h>
162 #include <sys/wait.h>
163
164 ClassImp(AliShuttle)
165
166 TString AliShuttle::fgkMainCDB("alien://folder=ShuttleCDB");
167 TString AliShuttle::fgkLocalCDB("local://LocalShuttleCDB");
168 TString AliShuttle::fgkMainRefStorage("alien://folder=ShuttleReference");
169 TString AliShuttle::fgkLocalRefStorage("local://LocalReferenceStorage");
170
171 Bool_t AliShuttle::fgkProcessDCS(kTRUE); 
172
173 const char* AliShuttle::fgkShuttleTempDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/temp");
174 const char* AliShuttle::fgkShuttleLogDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/log");
175
176 //______________________________________________________________________________________________
177 AliShuttle::AliShuttle(const AliShuttleConfig* config,
178                 UInt_t timeout, Int_t retries):
179 fConfig(config),
180 fTimeout(timeout), fRetries(retries),
181 fPreprocessorMap(),
182 fLogbookEntry(0),
183 fCurrentDetector(),
184 fStatusEntry(0),
185 fGridError(kFALSE),
186 fMonitoringMutex(0),
187 fLastActionTime(0),
188 fLastAction()
189 {
190         //
191         // config: AliShuttleConfig used
192         // timeout: timeout used for AliDCSClient connection
193         // retries: the number of retries in case of connection error.
194         //
195
196         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
197         for(int iSys=0;iSys<4;iSys++) {
198                 fServer[iSys]=0;
199                 if (iSys < 3)
200                         fFXSlist[iSys].SetOwner(kTRUE);
201         }
202         fPreprocessorMap.SetOwner(kTRUE);
203
204         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
205                 fFirstUnprocessed[iDet] = kFALSE;
206
207         fMonitoringMutex = new TMutex();
208 }
209
210 //______________________________________________________________________________________________
211 AliShuttle::~AliShuttle()
212 {
213 // destructor
214
215         fPreprocessorMap.DeleteAll();
216         for(int iSys=0;iSys<4;iSys++)
217                 if(fServer[iSys]) {
218                         fServer[iSys]->Close();
219                         delete fServer[iSys];
220                         fServer[iSys] = 0;
221                 }
222
223         if (fStatusEntry){
224                 delete fStatusEntry;
225                 fStatusEntry = 0;
226         }
227         
228         if (fMonitoringMutex) 
229         {
230                 delete fMonitoringMutex;
231                 fMonitoringMutex = 0;
232         }
233 }
234
235 //______________________________________________________________________________________________
236 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
237 {
238         //
239         // Registers new AliPreprocessor.
240         // It uses GetName() for indentificator of the pre processor.
241         // The pre processor is registered it there isn't any other
242         // with the same identificator (GetName()).
243         //
244
245         const char* detName = preprocessor->GetName();
246         if(GetDetPos(detName) < 0)
247                 AliFatal(Form("********** !!!!! Invalid detector name: %s !!!!! **********", detName));
248
249         if (fPreprocessorMap.GetValue(detName)) {
250                 AliWarning(Form("AliPreprocessor %s is already registered!", detName));
251                 return;
252         }
253
254         fPreprocessorMap.Add(new TObjString(detName), preprocessor);
255 }
256 //______________________________________________________________________________________________
257 UInt_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
258                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
259 {
260   // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
261   // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
262   // using this function. Use StoreReferenceData instead!
263   // It calls WriteToCDB function which perform actual storage
264
265         return WriteToCDB(fgkMainCDB, fgkLocalCDB, path, object,
266                                 metaData, validityStart, validityInfinite);
267
268 }
269
270 //______________________________________________________________________________________________
271 UInt_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData)
272 {
273   // Stores a CDB object in the storage for reference data. This objects will not be available during
274   // offline reconstrunction. Use this function for reference data only!
275   // It calls WriteToCDB function which perform actual storage
276
277         return WriteToCDB(fgkMainRefStorage, fgkLocalRefStorage, path, object, metaData);
278
279 }
280
281 //______________________________________________________________________________________________
282 UInt_t AliShuttle::WriteToCDB(const char* mainUri, const char* localUri,
283                         const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
284                         Int_t validityStart, Bool_t validityInfinite)
285 {
286   // write object into the CDB. Parameters are passed by Store and StoreReferenceData functions.
287   // The parameters are:
288   //   1) Uri of the main storage (Grid)
289   //   2) Uri of the backup storage (Local)
290   //   3) the object's path.
291   //   4) the object to be stored
292   //   5) the metaData to be associated with the object
293   //   6) the validity start run number w.r.t. the current run,
294   //      if the data is valid only for this run leave the default 0
295   //   7) specifies if the calibration data is valid for infinity (this means until updated),
296   //      typical for calibration runs, the default is kFALSE
297   //
298   // returns 0 if fail
299   //         1 if stored in main (Grid) storage
300   //         2 if stored in backup (Local) storage
301
302         const char* cdbType = (mainUri == fgkMainCDB) ? "CDB" : "Reference";
303
304         Int_t firstRun = GetCurrentRun() - validityStart;
305         if(firstRun < 0) {
306                 AliError("First valid run happens to be less than 0! Setting it to 0.");
307                 firstRun=0;
308         }
309
310         Int_t lastRun = -1;
311         if(validityInfinite) {
312                 lastRun = AliCDBRunRange::Infinity();
313         } else {
314                 lastRun = GetCurrentRun();
315         }
316
317         AliCDBId id(path, firstRun, lastRun, -1, -1);
318
319         if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
320                 TObjString runUsed = Form("%d", GetCurrentRun());
321                 metaData->SetProperty("RunUsed(TObjString)",&runUsed);
322         }
323
324         UInt_t result = 0;
325
326         if (!(AliCDBManager::Instance()->GetStorage(mainUri))) {
327                 AliError(Form("WriteToCDB - Cannot activate main %s storage", cdbType));
328         } else {
329                 result = (UInt_t) AliCDBManager::Instance()->GetStorage(mainUri)
330                                         ->Put(object, id, metaData);
331         }
332
333         if(!result) {
334
335                 Log(fCurrentDetector,
336                         Form("WriteToCDB - Problem with main %s storage. Putting <%s> into backup storage",
337                                 cdbType, path.GetPath().Data()));
338
339                 // Set Grid version to current run number, to ease retrieval later
340                 id.SetVersion(GetCurrentRun());
341
342                 result = AliCDBManager::Instance()->GetStorage(localUri)
343                                         ->Put(object, id, metaData);
344
345                 if(result) {
346                         result = 2;
347                         fGridError = kTRUE;
348                 }else{
349                         Log(fCurrentDetector, "WriteToCDB - Can't store data!");
350                 }
351         }
352
353         return result;
354
355 }
356
357 //______________________________________________________________________________________________
358 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
359 {
360 // Reads the AliShuttleStatus from the CDB
361
362         if (fStatusEntry){
363                 delete fStatusEntry;
364                 fStatusEntry = 0;
365         }
366
367         fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalCDB())
368                 ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), GetCurrentRun());
369
370         if (!fStatusEntry) return 0;
371         fStatusEntry->SetOwner(1);
372
373         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
374         if (!status) {
375                 AliError("Invalid object stored to CDB!");
376                 return 0;
377         }
378
379         return status;
380 }
381
382 //______________________________________________________________________________________________
383 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
384 {
385 // writes the status for one subdetector
386
387         if (fStatusEntry){
388                 delete fStatusEntry;
389                 fStatusEntry = 0;
390         }
391
392         Int_t run = GetCurrentRun();
393
394         AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), run, run);
395
396         fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
397         fStatusEntry->SetOwner(1);
398
399         UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
400
401         if (!result) {
402                 AliError(Form("WriteShuttleStatus for %s, run %d failed", fCurrentDetector.Data(), run));
403                 return kFALSE;
404         }
405
406         return kTRUE;
407 }
408
409 //______________________________________________________________________________________________
410 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
411 {
412   // changes the AliShuttleStatus for the given detector and run to the given status
413
414         if (!fStatusEntry){
415                 AliError("UNEXPECTED: fStatusEntry empty");
416                 return;
417         }
418
419         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
420
421         if (!status){
422                 AliError("UNEXPECTED: status could not be read from current CDB entry");
423                 return;
424         }
425
426         TString actionStr = Form("UpdateShuttleStatus - %s: Changing state from %s to %s",
427                                 fCurrentDetector.Data(),
428                                 status->GetStatusName(), 
429                                 status->GetStatusName(newStatus));
430         Log("SHUTTLE", actionStr);
431         SetLastAction(actionStr);
432
433         status->SetStatus(newStatus);
434         if (increaseCount) status->IncreaseCount();
435
436         AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
437 }
438 //______________________________________________________________________________________________
439 Bool_t AliShuttle::ContinueProcessing()
440 {
441 // this function reads the AliShuttleStatus information from CDB and
442 // checks if the processing should be continued
443 // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
444
445         AliShuttleLogbookEntry::Status entryStatus =
446                 fLogbookEntry->GetDetectorStatus(fCurrentDetector);
447
448         if(entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
449                 Log("SHUTTLE", Form("ContinueProcessing - %s is %s",
450                                 fCurrentDetector.Data(),
451                                 fLogbookEntry->GetDetectorStatusName(entryStatus)));
452                 return kFALSE;
453         }
454
455         // if we get here, according to Shuttle logbook subdetector is in UNPROCESSED state
456
457         // check if current run is first unprocessed run for current detector
458         if (fConfig->StrictRunOrder(fCurrentDetector) &&
459                 !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
460         {
461                 Log("SHUTTLE", Form("ContinueProcessing - %s requires strict run ordering but this is not the first unprocessed run!"));
462                 return kFALSE;
463         }
464
465         AliShuttleStatus* status = ReadShuttleStatus();
466         if (!status) {
467                 // first time
468                 Log("SHUTTLE", Form("ContinueProcessing - %s: Processing first time",
469                                 fCurrentDetector.Data()));
470                 status = new AliShuttleStatus(AliShuttleStatus::kStarted);
471                 return WriteShuttleStatus(status);
472         }
473
474         // The following two cases shouldn't happen if Shuttle Logbook was correctly updated.
475         // If it happens it may mean Logbook updating failed... let's do it now!
476         if (status->GetStatus() == AliShuttleStatus::kDone ||
477             status->GetStatus() == AliShuttleStatus::kFailed){
478                 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s. Updating Shuttle Logbook",
479                                         fCurrentDetector.Data(),
480                                         status->GetStatusName(status->GetStatus())));
481                 UpdateShuttleLogbook(fCurrentDetector.Data(),
482                                         status->GetStatusName(status->GetStatus()));
483                 return kFALSE;
484         }
485
486         if (status->GetStatus() == AliShuttleStatus::kStoreFailed) {
487                 Log("SHUTTLE",
488                         Form("ContinueProcessing - %s: Grid storage of one or more objects failed. Trying again now",
489                                 fCurrentDetector.Data()));
490                 if(TryToStoreAgain()){
491                         Log(fCurrentDetector.Data(), "ContinueProcessing - All objects successfully stored into OCDB");
492                         UpdateShuttleStatus(AliShuttleStatus::kDone);
493                         UpdateShuttleLogbook(fCurrentDetector.Data(), "DONE");
494                 } else {
495                         Log("SHUTTLE",
496                                 Form("ContinueProcessing - %s: Grid storage failed again",
497                                         fCurrentDetector.Data()));
498                 }
499                 return kFALSE;
500         }
501
502         // if we get here, there is a restart
503
504         // abort conditions
505         if (status->GetCount() >= fConfig->GetMaxRetries()) {
506                 Log("SHUTTLE",
507                         Form("ContinueProcessing - %s failed %d times in status %s - Updating Shuttle Logbook",
508                                 fCurrentDetector.Data(),
509                                 status->GetCount(), status->GetStatusName()));
510                 UpdateShuttleLogbook(fCurrentDetector.Data(), "FAILED");
511                 return kFALSE;
512         }
513
514         Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. Aborted before with %s. Retry number %d.",
515                         fCurrentDetector.Data(),
516                         status->GetStatusName(), status->GetCount()));
517
518         UpdateShuttleStatus(AliShuttleStatus::kStarted, kTRUE);
519
520         return kTRUE;
521 }
522
523 //______________________________________________________________________________________________
524 Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
525 {
526         //
527         // Makes data retrieval for all detectors in the configuration.
528         // entry: Shuttle logbook entry, contains run paramenters and status of detectors
529         // (Unprocessed, Inactive, Failed or Done).
530         // Returns kFALSE in case of error occured and kTRUE otherwise
531         //
532
533         if(!entry) return kFALSE;
534
535         fLogbookEntry = entry;
536
537         if(fLogbookEntry->IsDone()){
538                 Log("SHUTTLE","Process - Shuttle is already DONE. Updating logbook");
539                 UpdateShuttleLogbook("shuttle_done");
540                 fLogbookEntry = 0;
541                 return kTRUE;
542         }
543
544
545         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^* \n",
546                                         GetCurrentRun()));
547
548         fLogbookEntry->Print("all");
549
550         // Initialization
551         Bool_t hasError = kFALSE;
552         for(Int_t iSys=0;iSys<3;iSys++) fFXSCalled[iSys]=kFALSE;
553
554         AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
555         if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
556         AliCDBStorage *mainRefSto = AliCDBManager::Instance()->GetStorage(fgkMainRefStorage);
557         if(mainRefSto) mainRefSto->QueryCDB(GetCurrentRun());
558
559         // Loop on detectors in the configuration
560         TIter iter(fConfig->GetDetectors());
561         TObjString* aDetector = 0;
562
563         while ((aDetector = (TObjString*) iter.Next()))
564         {
565                 fCurrentDetector = aDetector->String();
566
567                 if (!fConfig->HostProcessDetector(fCurrentDetector)) continue;
568
569                 AliPreprocessor* aPreprocessor =
570                         dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
571                 if (!aPreprocessor)
572                 {
573                         Log("SHUTTLE",Form("Process - %s: no preprocessor registered. Skipping",
574                                                         fCurrentDetector.Data()));
575                         continue;
576                 }
577
578                 if (ContinueProcessing() == kFALSE) continue;
579
580                 AliInfo(Form("\n\n \t\t\t****** run %d - %s: START  ******",
581                                                 GetCurrentRun(), aDetector->GetName()));
582
583
584                 Int_t pid = fork();
585
586                 if (pid < 0)
587                 {
588                         Log("SHUTTLE", "ERROR: Forking failed");
589                 }
590                 else if (pid > 0)
591                 {
592                         // parent
593                         AliInfo(Form("In parent process of %d - %s: Starting monitoring",
594                                                         GetCurrentRun(), aDetector->GetName()));
595
596                         Long_t begin = time(0);
597
598                         int status; // to be used with waitpid, on purpose an int (not Int_t)!
599                         while (waitpid(pid, &status, WNOHANG) == 0)
600                         {
601                                 Long_t expiredTime = time(0) - begin;
602
603                                 if (expiredTime > fConfig->GetPPTimeOut())
604                                 {
605                                         Log("SHUTTLE", Form("Process time out. Run time: %d seconds. Killing...",
606                                                                 expiredTime));
607
608                                         kill(pid, 9);
609
610                                         hasError = kTRUE;
611
612                                         gSystem->Sleep(1000);
613                                 }
614                                 else
615                                 {
616                                         if (expiredTime % 60 == 0)
617                                         Log("SHUTTLE", Form("Checked process. Run time: %d seconds.",
618                                                                 expiredTime));
619                                         gSystem->Sleep(1000);
620                                 }
621                         }
622
623                         AliInfo(Form("In parent process of %d - %s: Client has terminated.",
624                                                                 GetCurrentRun(), aDetector->GetName()));
625
626                         if (WIFEXITED(status))
627                         {
628                                 Int_t returnCode = WEXITSTATUS(status);
629
630                                 Log("SHUTTLE", Form("The return code is %d", returnCode));
631
632                                 if (returnCode != 0)
633                                 hasError = kTRUE;
634                         }
635                 }
636                 else if (pid == 0)
637                 {
638                         // client
639                         AliInfo(Form("In client process of %d - %s", GetCurrentRun(), aDetector->GetName()));
640
641                         UInt_t result = ProcessCurrentDetector();
642
643                         Int_t returnCode = 0; // will be set to 1 in case of an error
644
645                         if (!result)
646                         {
647                                 returnCode = 1;
648                                 AliInfo(Form("\n \t\t\t****** run %d - %s: PREPROCESSOR ERROR ****** \n\n",
649                                                         GetCurrentRun(), aDetector->GetName()));
650                         }
651                         else if (result == 2)
652                         {
653                                 AliInfo(Form("\n \t\t\t****** run %d - %s: STORAGE ERROR ****** \n\n",
654                                                         GetCurrentRun(), aDetector->GetName()));
655                         } else
656                         {
657                                 AliInfo(Form("\n \t\t\t****** run %d - %s: DONE ****** \n\n",
658                                                         GetCurrentRun(), aDetector->GetName()));
659                         }
660
661                         if (result > 0)
662                         {
663                                 // Process successful: Update time_processed field in FXS logbooks!
664                                 if (fFXSCalled[kDAQ])
665                                 {
666                                         if (UpdateDAQTable() == kFALSE)
667                                         returnCode = 1;
668                                         fFXSlist[kDAQ].Clear();
669                                 }
670                                 //if(fFXSCalled[kDCS]) {
671                                 //  if (UpdateDCSTable(aDetector->GetName()) == kFALSE)
672                                 //    returnCode = 1;
673                                 //  fFXSlist[kDCS].Clear();
674                                 //}
675                                 //if(fFXSCalled[kHLT]) {
676                                 //  if (UpdateHLTTable(aDetector->GetName()) == kFALSE)
677                                 //    returnCode = 1;
678                                 //      fFXSlist[kHLT].Clear();
679                                 //}
680                         }
681
682                         AliInfo(Form("Client process of %d - %s is exiting now with %d.",
683                                                         GetCurrentRun(), aDetector->GetName(), returnCode));
684
685                         // the client exits here
686                         gSystem->Exit(returnCode);
687
688                         AliError("We should never get here!!!");
689                 }
690         }
691
692         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^* \n",
693                                                         GetCurrentRun()));
694
695         //check if shuttle is done for this run, if so update logbook
696         TObjArray checkEntryArray;
697         checkEntryArray.SetOwner(1);
698         TString whereClause = Form("where run=%d",GetCurrentRun());
699         if (QueryShuttleLogbook(whereClause.Data(), checkEntryArray)) {
700
701                 AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
702                                                         (checkEntryArray.At(0));
703
704                 if (checkEntry)
705                 {
706                         if (checkEntry->IsDone())
707                         {
708                                 Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
709                                 UpdateShuttleLogbook("shuttle_done");
710                         }
711                         else
712                         {
713                                 for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
714                                 {
715                                         if (checkEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
716                                         {
717                                                 AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
718                                                                 checkEntry->GetRun(), GetDetName(iDet)));
719                                                 fFirstUnprocessed[iDet] = kFALSE;
720                                         }
721                                 }
722                         }
723                 }
724         }
725
726         fLogbookEntry = 0;
727
728         return hasError == kFALSE;
729 }
730
731 //______________________________________________________________________________________________
732 UInt_t AliShuttle::ProcessCurrentDetector()
733 {
734         //
735         // Makes data retrieval just for a specific detector (fCurrentDetector).
736         // Threre should be a configuration for this detector.
737
738         AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), GetCurrentRun()));
739
740         UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
741
742         TMap dcsMap;
743         dcsMap.SetOwner(1);
744
745         Bool_t aDCSError = kFALSE;
746         fGridError = kFALSE;
747
748         // TODO Test only... I've added a flag that allows to
749         // exclude DCS archive DB query
750         if (!fgkProcessDCS)
751         {
752                 AliInfo("Skipping DCS processing!");
753                 aDCSError = kFALSE;
754         } else {
755                 TString host(fConfig->GetDCSHost(fCurrentDetector));
756                 Int_t port = fConfig->GetDCSPort(fCurrentDetector);
757
758                 // Retrieval of Aliases
759                 TObjString* anAlias = 0;
760                 TIter iterAliases(fConfig->GetDCSAliases(fCurrentDetector));
761                 while ((anAlias = (TObjString*) iterAliases.Next()))
762                 {
763                         TObjArray *valueSet = new TObjArray();
764                         valueSet->SetOwner(1);
765
766                         AliInfo("Querying DCS archive DB (Aliases)...");
767                         aDCSError = (GetValueSet(host, port, anAlias->String(), valueSet, kAlias) == 0);
768
769                         if(!aDCSError)
770                         {
771                                 dcsMap.Add(anAlias->Clone(), valueSet);
772                         } else {
773                                 Log(fCurrentDetector,
774                                         Form("ProcessCurrentDetector - Error while retrieving alias %s",
775                                                 anAlias->GetName()));
776                                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
777                                 dcsMap.DeleteAll();
778                                 return 0;
779                         }
780                 }
781
782                 // Retrieval of Data Points
783                 TObjString* aDP = 0;
784                 TIter iterDP(fConfig->GetDCSDataPoints(fCurrentDetector));
785                 while ((aDP = (TObjString*) iterDP.Next()))
786                 {
787                         TObjArray *valueSet = new TObjArray();
788                         valueSet->SetOwner(1);
789                         AliInfo("Querying DCS archive DB (Data Points)...");
790                         aDCSError = (GetValueSet(host, port, aDP->String(), valueSet, kDP) == 0);
791
792                         if(!aDCSError)
793                         {
794                                 dcsMap.Add(aDP->Clone(), valueSet);
795                         } else {
796                                 Log(fCurrentDetector,
797                                         Form("ProcessCurrentDetector - Error while retrieving data point %s",
798                                                 aDP->GetName()));
799                                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
800                                 dcsMap.DeleteAll();
801                                 return 0;
802                         }
803                 }
804         }
805
806         // DCS Archive DB processing successful. Call Preprocessor!
807         UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
808
809         AliPreprocessor* aPreprocessor =
810                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
811
812         aPreprocessor->Initialize(GetCurrentRun(), GetCurrentStartTime(), GetCurrentEndTime());
813         UInt_t aPPResult = aPreprocessor->Process(&dcsMap);
814
815         UInt_t returnValue = 0;
816         if (aPPResult == 0) { // Preprocessor error
817                 UpdateShuttleStatus(AliShuttleStatus::kPPError);
818                 returnValue = 0;
819         } else if (fGridError == kFALSE) { // process and Grid storage ok!
820                 UpdateShuttleStatus(AliShuttleStatus::kDone);
821                 UpdateShuttleLogbook(fCurrentDetector, "DONE");
822                 Log(fCurrentDetector.Data(),
823                         "ProcessCurrentDetector - Preprocessor and Grid storage ended successfully");
824                 returnValue = 1;
825         } else { // Grid storage error (process ok, but object put in local storage)
826                 UpdateShuttleStatus(AliShuttleStatus::kStoreFailed);
827                 returnValue = 2;
828         }
829
830         dcsMap.DeleteAll();
831
832         return returnValue;
833 }
834
835 //______________________________________________________________________________________________
836 Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
837                 TObjArray& entries)
838 {
839 // Query DAQ's Shuttle logbook and fills detector status object.
840 // Call QueryRunParameters to query DAQ logbook for run parameters.
841
842         // check connection, in case connect
843         if(!Connect(3)) return kFALSE;
844
845         TString sqlQuery;
846         sqlQuery = Form("select * from logbook_shuttle %s order by run", whereClause);
847
848         TSQLResult* aResult = fServer[3]->Query(sqlQuery);
849         if (!aResult) {
850                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
851                 return kFALSE;
852         }
853
854         if(aResult->GetRowCount() == 0) {
855                 if(sqlQuery.Contains("where shuttle_done=0")){
856                         Log("SHUTTLE", "QueryShuttleLogbook - All runs in Shuttle Logbook are already DONE");
857                         delete aResult;
858                         return kTRUE;
859                 } else {
860                         AliError("No entries in Shuttle Logbook match request");
861                         delete aResult;
862                         return kFALSE;
863                 }
864         }
865
866         // TODO Check field count!
867         const UInt_t nCols = 24;
868         if (aResult->GetFieldCount() != (Int_t) nCols) {
869                 AliError("Invalid SQL result field number!");
870                 delete aResult;
871                 return kFALSE;
872         }
873
874         entries.SetOwner(1);
875
876         TSQLRow* aRow;
877         while ((aRow = aResult->Next())) {
878                 TString runString(aRow->GetField(0), aRow->GetFieldLength(0));
879                 Int_t run = runString.Atoi();
880
881                 AliShuttleLogbookEntry *entry = QueryRunParameters(run);
882                 if (!entry)
883                         continue;
884
885                 // loop on detectors
886                 for(UInt_t ii = 0; ii < nCols; ii++)
887                         entry->SetDetectorStatus(aResult->GetFieldName(ii), aRow->GetField(ii));
888
889                 entries.AddLast(entry);
890                 delete aRow;
891         }
892
893         if(sqlQuery.Contains("where shuttle_done=0"))
894                 Log("SHUTTLE", Form("QueryShuttleLogbook - Found %d unprocessed runs in Shuttle Logbook",
895                                                         entries.GetEntriesFast()));
896         delete aResult;
897         return kTRUE;
898 }
899
900 //______________________________________________________________________________________________
901 AliShuttleLogbookEntry* AliShuttle::QueryRunParameters(Int_t run)
902 {
903         //
904         // Retrieve run parameters written in the DAQ logbook and sets them into AliShuttleLogbookEntry object
905         //
906
907         // check connection, in case connect
908         if (!Connect(3))
909                 return 0;
910
911         TString sqlQuery;
912         sqlQuery.Form("select * from %s where run=%d", fConfig->GetDAQlbTable(), run);
913
914         TSQLResult* aResult = fServer[3]->Query(sqlQuery);
915         if (!aResult) {
916                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
917                 return 0;
918         }
919
920         if (aResult->GetRowCount() == 0) {
921                 Log("SHUTTLE", Form("QueryRunParameters - No entry in DAQ Logbook for run %d. Skipping", run));
922                 delete aResult;
923                 return 0;
924         }
925
926         if (aResult->GetRowCount() > 1) {
927                 AliError(Form("More than one entry in DAQ Logbook for run %d. Skipping", run));
928                 delete aResult;
929                 return 0;
930         }
931
932         TSQLRow* aRow = aResult->Next();
933         if (!aRow)
934         {
935                 AliError(Form("Could not retrieve row for run %d. Skipping", run));
936                 delete aResult;
937                 return 0;
938         }
939
940         AliShuttleLogbookEntry* entry = new AliShuttleLogbookEntry(run);
941
942         for (Int_t ii = 0; ii < aResult->GetFieldCount(); ii++)
943                 entry->SetRunParameter(aResult->GetFieldName(ii), aRow->GetField(ii));
944
945         UInt_t startTime = entry->GetStartTime();
946         UInt_t endTime = entry->GetEndTime();
947
948         if (!startTime || !endTime || startTime > endTime) {
949                 Log("SHUTTLE",
950                         Form("QueryRunParameters - Invalid parameters for Run %d: startTime = %d, endTime = %d",
951                                 run, startTime, endTime));
952                 delete entry;
953                 delete aRow;
954                 delete aResult;
955                 return 0;
956         }
957
958         delete aRow;
959         delete aResult;
960
961         return entry;
962 }
963
964 //______________________________________________________________________________________________
965 Bool_t AliShuttle::TryToStoreAgain()
966 {
967   // Called in case the detector failed to store the object in Grid OCDB
968   // It tries to store the object again, if it does not find more recent and overlapping objects
969   // Calls underlying TryToStoreAgain(const char*) function twice, for OCDB and Reference storage.
970
971         AliInfo("Trying to store OCDB data again...");
972         Bool_t resultCDB = TryToStoreAgain(fgkMainCDB);
973
974         AliInfo("Trying to store reference data again...");
975         Bool_t resultRef = TryToStoreAgain(fgkMainRefStorage);
976
977         return resultCDB && resultRef;
978 }
979
980 //______________________________________________________________________________________________
981 Bool_t AliShuttle::TryToStoreAgain(TString& gridURI)
982 {
983   // Called by TryToStoreAgain(), performs actual storage retry
984
985         TObjArray* gridIds=0;
986
987         Bool_t result = kTRUE;
988
989         const char* type = 0;
990         TString backupURI;
991         if(gridURI == fgkMainCDB) {
992                 type = "OCDB";
993                 backupURI = fgkLocalCDB;
994         } else if(gridURI == fgkMainRefStorage) {
995                 type = "reference";
996                 backupURI = fgkLocalRefStorage;
997         } else {
998                 AliError(Form("Invalid storage URI: %s", gridURI.Data()));
999                 return kFALSE;
1000         }
1001
1002         AliCDBManager* man = AliCDBManager::Instance();
1003
1004         AliCDBStorage *gridSto = man->GetStorage(gridURI);
1005         if(!gridSto) {
1006                 Log(fCurrentDetector.Data(),
1007                         Form("TryToStoreAgain - cannot activate main %s storage", type));
1008                 return kFALSE;
1009         }
1010
1011         gridIds = gridSto->GetQueryCDBList();
1012
1013         // get objects previously stored in local CDB
1014         AliCDBStorage *backupSto = man->GetStorage(backupURI);
1015         AliCDBPath aPath(GetOfflineDetName(fCurrentDetector.Data()),"*","*");
1016         // Local objects were stored with current run as Grid version!
1017         TList* localEntries = backupSto->GetAll(aPath.GetPath(), GetCurrentRun(), GetCurrentRun());
1018         localEntries->SetOwner(1);
1019
1020         // loop on local stored objects
1021         TIter localIter(localEntries);
1022         AliCDBEntry *aLocEntry = 0;
1023         while((aLocEntry = dynamic_cast<AliCDBEntry*> (localIter.Next()))){
1024                 aLocEntry->SetOwner(1);
1025                 AliCDBId aLocId = aLocEntry->GetId();
1026                 aLocEntry->SetVersion(-1);
1027                 aLocEntry->SetSubVersion(-1);
1028
1029                 // loop on Grid valid Id's
1030                 Bool_t store = kTRUE;
1031                 TIter gridIter(gridIds);
1032                 AliCDBId* aGridId = 0;
1033                 while((aGridId = dynamic_cast<AliCDBId*> (gridIter.Next()))){
1034                         // If local object is valid up to infinity we store it only if it is
1035                         // the first unprocessed run!
1036                         if (aLocId.GetLastRun() == AliCDBRunRange::Infinity())
1037                         {
1038                                 if (!fFirstUnprocessed[GetDetPos(fCurrentDetector)])
1039                                 {
1040                                         Log(fCurrentDetector.Data(),
1041                                                 ("TryToStoreAgain - This object has validity infinite but "
1042                                                  "there are previous unprocessed runs!"));
1043                                         continue;
1044                                 } else {
1045                                         break;
1046                                 }
1047                         }
1048                         if(aGridId->GetPath() != aLocId.GetPath()) continue;
1049                         // skip all objects valid up to infinity
1050                         if(aGridId->GetLastRun() == AliCDBRunRange::Infinity()) continue;
1051                         // if we get here, it means there's already some more recent object stored on Grid!
1052                         store = kFALSE;
1053                         break;
1054                 }
1055
1056                 if(!store){
1057                         Log(fCurrentDetector.Data(),
1058                                 Form("TryToStoreAgain - A more recent object already exists in %s storage: <%s>",
1059                                         type, aGridId->ToString().Data()));
1060                         // removing local filename...
1061                         // TODO maybe it's better not to remove it, it was not copied to the Grid!
1062                         TString filename;
1063                         backupSto->IdToFilename(aLocId, filename);
1064                         AliInfo(Form("Removing local file %s", filename.Data()));
1065                         gSystem->Exec(Form("rm %s",filename.Data()));
1066                         continue;
1067                 }
1068
1069                 // If we get here, the file can be stored!
1070                 Bool_t storeOk = gridSto->Put(aLocEntry);
1071                 if(storeOk){
1072                         Log(fCurrentDetector.Data(),
1073                                 Form("TryToStoreAgain - Object <%s> successfully put into %s storage",
1074                                         aLocId.ToString().Data(), type));
1075
1076                         // removing local filename...
1077                         TString filename;
1078                         backupSto->IdToFilename(aLocId, filename);
1079                         AliInfo(Form("Removing local file %s", filename.Data()));
1080                         gSystem->Exec(Form("rm %s", filename.Data()));
1081                         continue;
1082                 } else  {
1083                         Log(fCurrentDetector.Data(),
1084                                 Form("TryToStoreAgain - Grid %s storage of object <%s> failed again",
1085                                         type, aLocId.ToString().Data()));
1086                         result = kFALSE;
1087                 }
1088         }
1089         localEntries->Clear();
1090
1091         return result;
1092 }
1093
1094 //______________________________________________________________________________________________
1095 Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* entry,
1096                                 TObjArray* valueSet, DCSType type)
1097 {
1098 // Retrieve all "entry" data points from the DCS server
1099 // host, port: TSocket connection parameters
1100 // entry: name of the alias or data point
1101 // valueSet: array of retrieved AliDCSValue's
1102 // type: kAlias or kDP
1103
1104         AliDCSClient client(host, port, fTimeout, fRetries);
1105         if (!client.IsConnected())
1106         {
1107                 return kFALSE;
1108         }
1109
1110         Int_t result=0;
1111
1112         if (type == kAlias)
1113         {
1114                 result = client.GetAliasValues(entry,
1115                         GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
1116         } else
1117         if (type == kDP)
1118         {
1119                 result = client.GetDPValues(entry,
1120                         GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
1121         }
1122
1123         if (result < 0)
1124         {
1125                 Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get '%s'! Reason: %s",
1126                         entry, AliDCSClient::GetErrorString(result)));
1127
1128                 if (result == AliDCSClient::fgkServerError)
1129                 {
1130                         Log(fCurrentDetector.Data(), Form("GetValueSet - Server error: %s",
1131                                 client.GetServerError().Data()));
1132                 }
1133
1134                 return kFALSE;
1135         }
1136
1137         return kTRUE;
1138 }
1139
1140 //______________________________________________________________________________________________
1141 const char* AliShuttle::GetFile(Int_t system, const char* detector,
1142                 const char* id, const char* source)
1143 {
1144 // Get calibration file from file exchange servers
1145 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
1146
1147         switch(system){
1148                 case kDAQ:
1149                         return GetDAQFileName(detector, id, source);
1150                         break;
1151                 case kDCS:
1152                         return GetDCSFileName(detector, id, source);
1153                         break;
1154                 case kHLT:
1155                         return GetHLTFileName(detector, id, source);
1156                         break;
1157                 default:
1158                         AliError(Form("No valid system index: %d",system));
1159         }
1160
1161         return 0;
1162 }
1163
1164 //______________________________________________________________________________________________
1165 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
1166 {
1167 // Get sources producing the condition file Id from file exchange servers
1168 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
1169
1170         switch(system){
1171                 case kDAQ:
1172                         return GetDAQFileSources(detector, id);
1173                         break;
1174                 case kDCS:
1175                         return GetDCSFileSources(detector, id);
1176                         break;
1177                 case kHLT:
1178                         return GetHLTFileSources(detector, id);
1179                         break;
1180                 default:
1181                         AliError(Form("No valid system index: %d",system));
1182         }
1183
1184         return NULL;
1185 }
1186
1187 //______________________________________________________________________________________________
1188 Bool_t AliShuttle::Connect(Int_t system)
1189 {
1190 // Connect to MySQL Server of the system's FXS MySQL databases
1191 // DAQ Logbook, Shuttle Logbook and DAQ FXS db are on the same host
1192
1193         // check connection: if already connected return
1194         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
1195
1196         TString dbHost, dbUser, dbPass, dbName;
1197
1198         if (system < 3) // FXS db servers
1199         {
1200                 dbHost = Form("mysql://%s", fConfig->GetFXSdbHost(system));
1201                 dbUser = fConfig->GetFXSdbUser(system);
1202                 dbPass = fConfig->GetFXSdbPass(system);
1203                 dbName =   fConfig->GetFXSdbName(system);
1204         } else { // Run & Shuttle logbook servers
1205         // TODO Will the Shuttle logbook server be the same as the Run logbook server ???
1206                 dbHost = Form("mysql://%s", fConfig->GetDAQlbHost());
1207                 dbUser = fConfig->GetDAQlbUser();
1208                 dbPass = fConfig->GetDAQlbPass();
1209                 dbName =   fConfig->GetDAQlbDB();
1210         }
1211
1212         fServer[system] = TSQLServer::Connect(dbHost.Data(), dbUser.Data(), dbPass.Data());
1213         if (!fServer[system] || !fServer[system]->IsConnected()) {
1214                 if(system < 3)
1215                 {
1216                 AliError(Form("Can't establish connection to FXS database for %s",
1217                                         AliShuttleInterface::GetSystemName(system)));
1218                 } else {
1219                 AliError("Can't establish connection to Run logbook.");
1220                 }
1221                 if(fServer[system]) delete fServer[system];
1222                 return kFALSE;
1223         }
1224
1225         // Get tables
1226         // TODO in the configuration should the table name be there too?
1227         TSQLResult* aResult=0;
1228         switch(system){
1229                 case kDAQ:
1230                         aResult = fServer[kDAQ]->GetTables(dbName.Data());
1231                         break;
1232                 case kDCS:
1233                         //aResult = fServer[kDCS]->GetTables(dbName.Data());
1234                         break;
1235                 case kHLT:
1236                         //aResult = fServer[kHLT]->GetTables(dbName.Data());
1237                         break;
1238                 default:
1239                         aResult = fServer[3]->GetTables(dbName.Data());
1240                         break;
1241         }
1242
1243         delete aResult;
1244         return kTRUE;
1245 }
1246
1247 //______________________________________________________________________________________________
1248 const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source)
1249 {
1250 // Retrieves a file from the DAQ FXS.
1251 // First queris the DAQ FXS database for the DAQ file name, using the run, detector, id and source info
1252 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
1253 // run: current run being processed (given by Logbook entry fLogbookEntry)
1254 // detector: the Preprocessor name
1255 // id: provided as a parameter by the Preprocessor
1256 // source: provided by the Preprocessor through GetFileSources function
1257
1258         // check connection, in case connect
1259         if (!Connect(kDAQ))
1260         {
1261                 Log(detector, "GetDAQFileName - Couldn't connect to DAQ FXS database");
1262                 return 0;
1263         }
1264
1265         // Query preparation
1266         TString sqlQueryStart = Form("select filePath from %s where", fConfig->GetFXSdbTable(kDAQ));
1267         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
1268                                 GetCurrentRun(), detector, id, source);
1269         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1270
1271         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1272
1273         // Query execution
1274         TSQLResult* aResult = 0;
1275         aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1276         if (!aResult) {
1277                 Log(detector, Form("GetDAQFileName - Can't execute SQL query for: id = %s, source = %s",
1278                                 id, source));
1279                 return 0;
1280         }
1281
1282         if(aResult->GetRowCount() == 0)
1283         {
1284                 Log(detector,
1285                         Form("GetDAQFileName - No entry in FXS table for: id = %s, source = %s",
1286                                 id, source));
1287                 delete aResult;
1288                 return 0;
1289         }
1290
1291         if (aResult->GetRowCount() > 1) {
1292                 Log(detector,
1293                         Form("GetDAQFileName - More than one entry in FXS table for: id = %s, source = %s",
1294                                 id, source));
1295                 delete aResult;
1296                 return 0;
1297         }
1298
1299         TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
1300
1301         if (!aRow){
1302                 Log(detector, Form("GetDAQFileName - Empty set result from query: id = %s, source = %s",
1303                                 id, source));
1304                 delete aResult;
1305                 return 0;
1306         }
1307
1308         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
1309
1310         delete aResult;
1311         delete aRow;
1312
1313         AliDebug(2, Form("filePath = %s",filePath.Data()));
1314
1315         // retrieved file is renamed to make it unique
1316         TString localFileName = Form("%s_%d_%s_%s.shuttle",
1317                                         detector, GetCurrentRun(), id, source);
1318
1319         // file retrieval from DAQ FXS
1320         Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
1321         if(!result) {
1322                 Log(detector, Form("GetDAQFileName - Copy of file %s from DAQ FXS failed", filePath.Data()));
1323                 return 0;
1324         } else {
1325                 AliInfo(Form("File %s copied from DAQ FXS into %s/%s",
1326                         filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
1327         }
1328
1329
1330         fFXSCalled[kDAQ]=kTRUE;
1331         TObjString *fileParams = new TObjString(Form("%s_!?!_%s", id, source));
1332         fFXSlist[kDAQ].Add(fileParams);
1333
1334         return localFileName.Data();
1335
1336 }
1337
1338 //______________________________________________________________________________________________
1339 Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName)
1340 {
1341
1342         // check temp directory: trying to cd to temp; if it does not exist, create it
1343         AliDebug(2, Form("Copy file %s from DAQ FXS into folder %s and rename it as %s",
1344                         daqFileName,fgkShuttleTempDir, localFileName));
1345
1346         void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
1347         if (dir == NULL) {
1348                 if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
1349                         AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
1350                         return kFALSE;
1351                 }
1352
1353         } else {
1354                 gSystem->FreeDirectory(dir);
1355         }
1356
1357         TString baseDAQFXSFolder = "DAQ";
1358         TString command = Form("scp %s@%s:%s/%s %s/%s",
1359                 fConfig->GetFXSUser(kDAQ),
1360                 fConfig->GetFXSHost(kDAQ),
1361                 baseDAQFXSFolder.Data(),
1362                 daqFileName,
1363                 fgkShuttleTempDir,
1364                 localFileName);
1365
1366         AliDebug(2, Form("%s",command.Data()));
1367
1368         UInt_t nRetries = 0;
1369         UInt_t maxRetries = 3;
1370
1371         // copy!! if successful TSystem::Exec returns 0
1372         while(nRetries++ < maxRetries) {
1373                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
1374                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
1375         }
1376
1377         return kFALSE;
1378
1379 }
1380
1381 //______________________________________________________________________________________________
1382 TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id)
1383 {
1384 // Retrieves a file from the DCS FXS.
1385
1386         // check connection, in case connect
1387         if(!Connect(kDAQ)){
1388                 Log(detector, "GetDAQFileSources - Couldn't connect to DAQ FXS database");
1389                 return 0;
1390         }
1391
1392         // Query preparation
1393         TString sqlQueryStart = Form("select DAQsource from %s where", fConfig->GetFXSdbTable(kDAQ));
1394         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
1395                                 GetCurrentRun(), detector, id);
1396         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1397
1398         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1399
1400         // Query execution
1401         TSQLResult* aResult;
1402         aResult = fServer[kDAQ]->Query(sqlQuery);
1403         if (!aResult) {
1404                 Log(detector, Form("GetDAQFileSources - Can't execute SQL query for id: %s", id));
1405                 return 0;
1406         }
1407
1408         if (aResult->GetRowCount() == 0) {
1409                 Log(detector,
1410                         Form("GetDAQFileSources - No entry in FXS table for id: %s", id));
1411                 delete aResult;
1412                 return 0;
1413         }
1414
1415         TSQLRow* aRow;
1416         TList *list = new TList();
1417         list->SetOwner(1);
1418
1419         while((aRow = aResult->Next())){
1420
1421                 TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
1422                 AliDebug(2, Form("daqSource = %s", daqSource.Data()));
1423                 list->Add(new TObjString(daqSource));
1424                 delete aRow;
1425         }
1426         delete aResult;
1427
1428         return list;
1429
1430 }
1431
1432 //______________________________________________________________________________________________
1433 const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1434 // Retrieves a file from the DCS FXS.
1435
1436 return "You're in DCS";
1437
1438 }
1439
1440 //______________________________________________________________________________________________
1441 TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
1442 // Retrieves file sources from the DCS FXS.
1443
1444 return NULL;
1445
1446 }
1447
1448 //______________________________________________________________________________________________
1449 const char* AliShuttle::GetHLTFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1450 // Retrieves a file from the HLT FXS.
1451
1452 return "You're in HLT";
1453
1454 }
1455
1456 //______________________________________________________________________________________________
1457 TList* AliShuttle::GetHLTFileSources(const char* /*detector*/, const char* /*id*/){
1458 // Retrieves file sources from the HLT FXS.
1459
1460 return NULL;
1461
1462 }
1463
1464 //______________________________________________________________________________________________
1465 Bool_t AliShuttle::UpdateDAQTable()
1466 {
1467 // Update DAQ table filling time_processed field in all rows corresponding to current run and detector
1468
1469         // check connection, in case connect
1470         if(!Connect(kDAQ)){
1471                 Log(fCurrentDetector, "UpdateDAQTable - Couldn't connect to DAQ FXS database");
1472                 return kFALSE;
1473         }
1474
1475         TTimeStamp now; // now
1476
1477         // Loop on FXS list entries
1478         TIter iter(&fFXSlist[kDAQ]);
1479         TObjString *aFXSentry=0;
1480         while((aFXSentry = dynamic_cast<TObjString*> (iter.Next()))){
1481                 TString aFXSentrystr = aFXSentry->String();
1482                 TObjArray *aFXSarray = aFXSentrystr.Tokenize("_!?!_");
1483                 if(!aFXSarray || aFXSarray->GetEntries() != 2 ) {
1484                         Log(fCurrentDetector, Form("UpdateDAQTable - error updating FXS entry. Check string: <%s>",
1485                                 aFXSentrystr.Data()));
1486                         if(aFXSarray) delete aFXSarray;
1487                         return kFALSE;
1488                 }
1489                 const char* fileId = ((TObjString*) aFXSarray->At(0))->GetName();
1490                 const char* daqSource = ((TObjString*) aFXSarray->At(1))->GetName();
1491                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
1492                         GetCurrentRun(), fCurrentDetector.Data(), fileId, daqSource);
1493
1494                 delete aFXSarray;
1495
1496                 TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(kDAQ),
1497                                                         now.GetSec(), whereClause.Data());
1498
1499                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1500
1501                 // Query execution
1502                 TSQLResult* aResult;
1503                 aResult = dynamic_cast<TSQLResult*> (fServer[3]->Query(sqlQuery));
1504                 if (!aResult) {
1505                         Log(fCurrentDetector, Form("UpdateDAQTable - Can't execute SQL query <%s>", sqlQuery.Data()));
1506                         return kFALSE;
1507                 }
1508                 delete aResult;
1509         }
1510
1511         return kTRUE;
1512 }
1513
1514
1515 //______________________________________________________________________________________________
1516 Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status)
1517 {
1518 // Update Shuttle logbook filling detector or shuttle_done column
1519 // ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
1520
1521         // check connection, in case connect
1522         if(!Connect(3)){
1523                 Log("SHUTTLE", "UpdateShuttleLogbook - Couldn't connect to DAQ Logbook.");
1524                 return kFALSE;
1525         }
1526
1527         TString detName(detector);
1528         TString setClause;
1529         if(detName == "shuttle_done") {
1530                 setClause = "set shuttle_done=1";
1531         } else {
1532                 TString statusStr(status);
1533                 if(statusStr.Contains("done", TString::kIgnoreCase) ||
1534                    statusStr.Contains("failed", TString::kIgnoreCase)){
1535                         setClause = Form("set %s=\"%s\"", detector, status);
1536                 } else {
1537                         Log("SHUTTLE",
1538                                 Form("UpdateShuttleLogbook - Invalid status <%s> for detector %s",
1539                                         status, detector));
1540                         return kFALSE;
1541                 }
1542         }
1543
1544         TString whereClause = Form("where run=%d", GetCurrentRun());
1545
1546         TString sqlQuery = Form("update logbook_shuttle %s %s",
1547                                         setClause.Data(), whereClause.Data());
1548
1549         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1550
1551         // Query execution
1552         TSQLResult* aResult;
1553         aResult = dynamic_cast<TSQLResult*> (fServer[3]->Query(sqlQuery));
1554         if (!aResult) {
1555                 Log("SHUTTLE", Form("UpdateShuttleLogbook - Can't execute query <%s>", sqlQuery.Data()));
1556                 return kFALSE;
1557         }
1558         delete aResult;
1559
1560         return kTRUE;
1561 }
1562
1563 //______________________________________________________________________________________________
1564 Int_t AliShuttle::GetCurrentRun() const
1565 {
1566 // Get current run from logbook entry
1567
1568         return fLogbookEntry ? fLogbookEntry->GetRun() : -1;
1569 }
1570
1571 //______________________________________________________________________________________________
1572 UInt_t AliShuttle::GetCurrentStartTime() const
1573 {
1574 // get current start time
1575
1576         return fLogbookEntry ? fLogbookEntry->GetStartTime() : 0;
1577 }
1578
1579 //______________________________________________________________________________________________
1580 UInt_t AliShuttle::GetCurrentEndTime() const
1581 {
1582 // get current end time from logbook entry
1583
1584         return fLogbookEntry ? fLogbookEntry->GetEndTime() : 0;
1585 }
1586
1587 //______________________________________________________________________________________________
1588 void AliShuttle::Log(const char* detector, const char* message)
1589 {
1590 // Fill log string with a message
1591
1592         void* dir = gSystem->OpenDirectory(fgkShuttleLogDir);
1593         if (dir == NULL) {
1594                 if (gSystem->mkdir(fgkShuttleLogDir, kTRUE)) {
1595                         AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
1596                         return;
1597                 }
1598
1599         } else {
1600                 gSystem->FreeDirectory(dir);
1601         }
1602
1603         TString toLog = Form("%s (%d): %s - ", TTimeStamp(time(0)).AsString("s"), getpid(), detector);
1604         if(GetCurrentRun()>=0 ) toLog += Form("run %d - ", GetCurrentRun());
1605         toLog += Form("%s", message);
1606
1607         AliInfo(toLog.Data());
1608
1609         TString fileName;
1610         fileName.Form("%s/%s.log", fgkShuttleLogDir, detector);
1611         gSystem->ExpandPathName(fileName);
1612
1613         ofstream logFile;
1614         logFile.open(fileName, ofstream::out | ofstream::app);
1615
1616         if (!logFile.is_open()) {
1617                 AliError(Form("Could not open file %s", fileName.Data()));
1618                 return;
1619         }
1620
1621         logFile << toLog.Data() << "\n";
1622
1623         logFile.close();
1624 }
1625
1626 //______________________________________________________________________________________________
1627 Bool_t AliShuttle::Collect(Int_t run)
1628 {
1629 //
1630 // Collects conditions data for all UNPROCESSED run written to DAQ LogBook in case of run = -1 (default)
1631 // If a dedicated run is given this run is processed
1632 //
1633 // In operational mode, this is the Shuttle function triggered by the EOR signal.
1634 //
1635
1636         if (run == -1)
1637                 Log("SHUTTLE","Collect - Shuttle called. Collecting conditions data for unprocessed runs");
1638         else
1639                 Log("SHUTTLE", Form("Collect - Shuttle called. Collecting conditions data for run %d", run));
1640
1641         SetLastAction("Starting");
1642
1643         TString whereClause("where shuttle_done=0");
1644         if (run != -1)
1645                 whereClause += Form(" and run=%d", run);
1646
1647         TObjArray shuttleLogbookEntries;
1648         if (!QueryShuttleLogbook(whereClause, shuttleLogbookEntries))
1649         {
1650                 Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
1651                 return kFALSE;
1652         }
1653
1654         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
1655                 fFirstUnprocessed[iDet] = kTRUE;
1656
1657         if (run != 1)
1658         {
1659                 // query Shuttle logbook for earlier runs, check if some detectors are unprocessed,
1660                 // flag them into fFirstUnprocessed array
1661                 TString whereClause(Form("where shuttle_done=0 and run < %d", run));
1662                 TObjArray tmpLogbookEntries;
1663                 if (!QueryShuttleLogbook(whereClause, tmpLogbookEntries))
1664                 {
1665                         Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
1666                         return kFALSE;
1667                 }
1668
1669                 TIter iter(&tmpLogbookEntries);
1670                 AliShuttleLogbookEntry* anEntry = 0;
1671                 while ((anEntry = dynamic_cast<AliShuttleLogbookEntry*> (iter.Next())))
1672                 {
1673                         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
1674                         {
1675                                 if (anEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
1676                                 {
1677                                         AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
1678                                                         anEntry->GetRun(), GetDetName(iDet)));
1679                                         fFirstUnprocessed[iDet] = kFALSE;
1680                                 }
1681                         }
1682
1683                 }
1684
1685         }
1686
1687         if (!RetrieveConditionsData(shuttleLogbookEntries))
1688         {
1689                 Log("SHUTTLE", "Collect - Process of at least one run failed");
1690                 return kFALSE;
1691         }
1692
1693         return kTRUE;
1694 }
1695
1696 //______________________________________________________________________________________________
1697 Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
1698 {
1699 // Retrieve conditions data for all runs that aren't processed yet
1700
1701         Bool_t hasError = kFALSE;
1702
1703         TIter iter(&dateEntries);
1704         AliShuttleLogbookEntry* anEntry;
1705
1706         while ((anEntry = (AliShuttleLogbookEntry*) iter.Next())){
1707                 if (!Process(anEntry)){
1708                         hasError = kTRUE;
1709                 }
1710         }
1711
1712         return hasError == kFALSE;
1713 }
1714
1715 //______________________________________________________________________________________________
1716 ULong_t AliShuttle::GetTimeOfLastAction() const
1717 {
1718         ULong_t tmp;
1719         
1720         fMonitoringMutex->Lock();
1721
1722         tmp = fLastActionTime;
1723         
1724         fMonitoringMutex->UnLock();
1725         
1726         return tmp;
1727 }
1728
1729 //______________________________________________________________________________________________
1730 const TString AliShuttle::GetLastAction() const
1731 {
1732         // returns a string description of the last action
1733
1734         TString tmp;
1735         
1736         fMonitoringMutex->Lock();
1737         
1738         tmp = fLastAction;
1739         
1740         fMonitoringMutex->UnLock();
1741
1742         return tmp;     
1743 }
1744
1745 //______________________________________________________________________________________________
1746 void AliShuttle::SetLastAction(const char* action)
1747 {
1748         // updates the monitoring variables
1749         
1750         fMonitoringMutex->Lock();
1751         
1752         fLastAction = action;
1753         fLastActionTime = time(0);
1754         
1755         fMonitoringMutex->UnLock();
1756 }
1757
1758 //______________________________________________________________________________________________
1759 const char* AliShuttle::GetRunParameter(const char* param)
1760 {
1761 // returns run parameter read from DAQ logbook
1762
1763         if(!fLogbookEntry) {
1764                 AliError("No logbook entry!");
1765                 return 0;
1766         }
1767
1768         return fLogbookEntry->GetRunParameter(param);
1769 }