Removed old ldif files, added TOF, MCH ldif files. Added some options in
[u/mrichter/AliRoot.git] / SHUTTLE / AliShuttle.cxx
1 /**************************************************************************
2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
3  *                                                                        *
4  * Author: The ALICE Off-line Project.                                    *
5  * Contributors are mentioned in the code where appropriate.              *
6  *                                                                        *
7  * Permission to use, copy, modify and distribute this software and its   *
8  * documentation strictly for non-commercial purposes is hereby granted   *
9  * without fee, provided that the above copyright notice appears in all   *
10  * copies and that both the copyright notice and this permission notice   *
11  * appear in the supporting documentation. The authors make no claims     *
12  * about the suitability of this software for any purpose. It is          *
13  * provided "as is" without express or implied warranty.                  *
14  **************************************************************************/
15
16 /*
17 $Log$
18 Revision 1.25  2007/01/15 19:13:52  acolla
19 Moved some AliInfo to AliDebug in SendMail function
20
21 Revision 1.21  2006/12/07 08:51:26  jgrosseo
22 update (alberto):
23 table, db names in ldap configuration
24 added GRP preprocessor
25 DCS data can also be retrieved by data point
26
27 Revision 1.20  2006/11/16 16:16:48  jgrosseo
28 introducing strict run ordering flag
29 removed giving preprocessor name to preprocessor, they have to know their name themselves ;-)
30
31 Revision 1.19  2006/11/06 14:23:04  jgrosseo
32 major update (Alberto)
33 o) reading of run parameters from the logbook
34 o) online offline naming conversion
35 o) standalone DCSclient package
36
37 Revision 1.18  2006/10/20 15:22:59  jgrosseo
38 o) Adding time out to the execution of the preprocessors: The Shuttle forks and the parent process monitors the child
39 o) Merging Collect, CollectAll, CollectNew function
40 o) Removing implementation of empty copy constructors (declaration still there!)
41
42 Revision 1.17  2006/10/05 16:20:55  jgrosseo
43 adapting to new CDB classes
44
45 Revision 1.16  2006/10/05 15:46:26  jgrosseo
46 applying to the new interface
47
48 Revision 1.15  2006/10/02 16:38:39  jgrosseo
49 update (alberto):
50 fixed memory leaks
51 storing of objects that failed to be stored to the grid before
52 interfacing of shuttle status table in daq system
53
54 Revision 1.14  2006/08/29 09:16:05  jgrosseo
55 small update
56
57 Revision 1.13  2006/08/15 10:50:00  jgrosseo
58 effc++ corrections (alberto)
59
60 Revision 1.12  2006/08/08 14:19:29  jgrosseo
61 Update to shuttle classes (Alberto)
62
63 - Possibility to set the full object's path in the Preprocessor's and
64 Shuttle's  Store functions
65 - Possibility to extend the object's run validity in the same classes
66 ("startValidity" and "validityInfinite" parameters)
67 - Implementation of the StoreReferenceData function to store reference
68 data in a dedicated CDB storage.
69
70 Revision 1.11  2006/07/21 07:37:20  jgrosseo
71 last run is stored after each run
72
73 Revision 1.10  2006/07/20 09:54:40  jgrosseo
74 introducing status management: The processing per subdetector is divided into several steps,
75 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
76 can keep track of the number of failures and skips further processing after a certain threshold is
77 exceeded. These thresholds can be configured in LDAP.
78
79 Revision 1.9  2006/07/19 10:09:55  jgrosseo
80 new configuration, accesst to DAQ FES (Alberto)
81
82 Revision 1.8  2006/07/11 12:44:36  jgrosseo
83 adding parameters for extended validity range of data produced by preprocessor
84
85 Revision 1.7  2006/07/10 14:37:09  jgrosseo
86 small fix + todo comment
87
88 Revision 1.6  2006/07/10 13:01:41  jgrosseo
89 enhanced storing of last sucessfully processed run (alberto)
90
91 Revision 1.5  2006/07/04 14:59:57  jgrosseo
92 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
93
94 Revision 1.4  2006/06/12 09:11:16  jgrosseo
95 coding conventions (Alberto)
96
97 Revision 1.3  2006/06/06 14:26:40  jgrosseo
98 o) removed files that were moved to STEER
99 o) shuttle updated to follow the new interface (Alberto)
100
101 Revision 1.2  2006/03/07 07:52:34  hristov
102 New version (B.Yordanov)
103
104 Revision 1.6  2005/11/19 17:19:14  byordano
105 RetrieveDATEEntries and RetrieveConditionsData added
106
107 Revision 1.5  2005/11/19 11:09:27  byordano
108 AliShuttle declaration added
109
110 Revision 1.4  2005/11/17 17:47:34  byordano
111 TList changed to TObjArray
112
113 Revision 1.3  2005/11/17 14:43:23  byordano
114 import to local CVS
115
116 Revision 1.1.1.1  2005/10/28 07:33:58  hristov
117 Initial import as subdirectory in AliRoot
118
119 Revision 1.2  2005/09/13 08:41:15  byordano
120 default startTime endTime added
121
122 Revision 1.4  2005/08/30 09:13:02  byordano
123 some docs added
124
125 Revision 1.3  2005/08/29 21:15:47  byordano
126 some docs added
127
128 */
129
130 //
131 // This class is the main manager for AliShuttle. 
132 // It organizes the data retrieval from DCS and call the 
133 // interface methods of AliPreprocessor.
134 // For every detector in AliShuttleConfgi (see AliShuttleConfig),
135 // data for its set of aliases is retrieved. If there is registered
136 // AliPreprocessor for this detector then it will be used
137 // accroding to the schema (see AliPreprocessor).
138 // If there isn't registered AliPreprocessor than the retrieved
139 // data is stored automatically to the undelying AliCDBStorage.
140 // For detSpec is used the alias name.
141 //
142
143 #include "AliShuttle.h"
144
145 #include "AliCDBManager.h"
146 #include "AliCDBStorage.h"
147 #include "AliCDBId.h"
148 #include "AliCDBRunRange.h"
149 #include "AliCDBPath.h"
150 #include "AliCDBEntry.h"
151 #include "AliShuttleConfig.h"
152 #include "DCSClient/AliDCSClient.h"
153 #include "AliLog.h"
154 #include "AliPreprocessor.h"
155 #include "AliShuttleStatus.h"
156 #include "AliShuttleLogbookEntry.h"
157
158 #include <TSystem.h>
159 #include <TObject.h>
160 #include <TString.h>
161 #include <TTimeStamp.h>
162 #include <TObjString.h>
163 #include <TSQLServer.h>
164 #include <TSQLResult.h>
165 #include <TSQLRow.h>
166 #include <TMutex.h>
167
168 #include <fstream>
169
170 #include <sys/types.h>
171 #include <sys/wait.h>
172
173 ClassImp(AliShuttle)
174
175 TString AliShuttle::fgkMainCDB("alien://folder=ShuttleCDB");
176 TString AliShuttle::fgkLocalCDB("local://LocalShuttleCDB");
177 TString AliShuttle::fgkMainRefStorage("alien://folder=ShuttleReference");
178 TString AliShuttle::fgkLocalRefStorage("local://LocalReferenceStorage");
179
180 Bool_t AliShuttle::fgkProcessDCS(kTRUE); 
181
182 TString AliShuttle::fgkShuttleTempDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/temp");
183 TString AliShuttle::fgkShuttleLogDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/log");
184
185 //______________________________________________________________________________________________
186 AliShuttle::AliShuttle(const AliShuttleConfig* config,
187                 UInt_t timeout, Int_t retries):
188 fConfig(config),
189 fTimeout(timeout), fRetries(retries),
190 fPreprocessorMap(),
191 fLogbookEntry(0),
192 fCurrentDetector(),
193 fStatusEntry(0),
194 fGridError(kFALSE),
195 fMonitoringMutex(0),
196 fLastActionTime(0),
197 fLastAction()
198 {
199         //
200         // config: AliShuttleConfig used
201         // timeout: timeout used for AliDCSClient connection
202         // retries: the number of retries in case of connection error.
203         //
204
205         if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
206         for(int iSys=0;iSys<4;iSys++) {
207                 fServer[iSys]=0;
208                 if (iSys < 3)
209                         fFXSlist[iSys].SetOwner(kTRUE);
210         }
211         fPreprocessorMap.SetOwner(kTRUE);
212
213         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
214                 fFirstUnprocessed[iDet] = kFALSE;
215
216         fMonitoringMutex = new TMutex();
217 }
218
219 //______________________________________________________________________________________________
220 AliShuttle::~AliShuttle()
221 {
222 // destructor
223
224         fPreprocessorMap.DeleteAll();
225         for(int iSys=0;iSys<4;iSys++)
226                 if(fServer[iSys]) {
227                         fServer[iSys]->Close();
228                         delete fServer[iSys];
229                         fServer[iSys] = 0;
230                 }
231
232         if (fStatusEntry){
233                 delete fStatusEntry;
234                 fStatusEntry = 0;
235         }
236         
237         if (fMonitoringMutex) 
238         {
239                 delete fMonitoringMutex;
240                 fMonitoringMutex = 0;
241         }
242 }
243
244 //______________________________________________________________________________________________
245 void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
246 {
247         //
248         // Registers new AliPreprocessor.
249         // It uses GetName() for indentificator of the pre processor.
250         // The pre processor is registered it there isn't any other
251         // with the same identificator (GetName()).
252         //
253
254         const char* detName = preprocessor->GetName();
255         if(GetDetPos(detName) < 0)
256                 AliFatal(Form("********** !!!!! Invalid detector name: %s !!!!! **********", detName));
257
258         if (fPreprocessorMap.GetValue(detName)) {
259                 AliWarning(Form("AliPreprocessor %s is already registered!", detName));
260                 return;
261         }
262
263         fPreprocessorMap.Add(new TObjString(detName), preprocessor);
264 }
265 //______________________________________________________________________________________________
266 UInt_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
267                 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
268 {
269   // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
270   // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
271   // using this function. Use StoreReferenceData instead!
272   // It calls WriteToCDB function which perform actual storage
273
274         return WriteToCDB(fgkMainCDB, fgkLocalCDB, path, object,
275                                 metaData, validityStart, validityInfinite);
276
277 }
278
279 //______________________________________________________________________________________________
280 UInt_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData)
281 {
282   // Stores a CDB object in the storage for reference data. This objects will not be available during
283   // offline reconstrunction. Use this function for reference data only!
284   // It calls WriteToCDB function which perform actual storage
285
286         return WriteToCDB(fgkMainRefStorage, fgkLocalRefStorage, path, object, metaData);
287
288 }
289
290 //______________________________________________________________________________________________
291 UInt_t AliShuttle::WriteToCDB(const char* mainUri, const char* localUri,
292                         const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
293                         Int_t validityStart, Bool_t validityInfinite)
294 {
295   // write object into the CDB. Parameters are passed by Store and StoreReferenceData functions.
296   // The parameters are:
297   //   1) Uri of the main storage (Grid)
298   //   2) Uri of the backup storage (Local)
299   //   3) the object's path.
300   //   4) the object to be stored
301   //   5) the metaData to be associated with the object
302   //   6) the validity start run number w.r.t. the current run,
303   //      if the data is valid only for this run leave the default 0
304   //   7) specifies if the calibration data is valid for infinity (this means until updated),
305   //      typical for calibration runs, the default is kFALSE
306   //
307   // returns 0 if fail
308   //         1 if stored in main (Grid) storage
309   //         2 if stored in backup (Local) storage
310
311         const char* cdbType = (mainUri == fgkMainCDB) ? "CDB" : "Reference";
312
313         Int_t firstRun = GetCurrentRun() - validityStart;
314         if(firstRun < 0) {
315                 AliError("First valid run happens to be less than 0! Setting it to 0.");
316                 firstRun=0;
317         }
318
319         Int_t lastRun = -1;
320         if(validityInfinite) {
321                 lastRun = AliCDBRunRange::Infinity();
322         } else {
323                 lastRun = GetCurrentRun();
324         }
325
326         AliCDBId id(path, firstRun, lastRun, -1, -1);
327
328         if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
329                 TObjString runUsed = Form("%d", GetCurrentRun());
330                 metaData->SetProperty("RunUsed(TObjString)", runUsed.Clone());
331         }
332
333         UInt_t result = 0;
334
335         if (!(AliCDBManager::Instance()->GetStorage(mainUri))) {
336                 AliError(Form("WriteToCDB - Cannot activate main %s storage", cdbType));
337         } else {
338                 result = (UInt_t) AliCDBManager::Instance()->GetStorage(mainUri)
339                                         ->Put(object, id, metaData);
340         }
341
342         if(!result) {
343
344                 Log(fCurrentDetector,
345                         Form("WriteToCDB - Problem with main %s storage. Putting <%s> into backup storage",
346                                 cdbType, path.GetPath().Data()));
347
348                 // Set Grid version to current run number, to ease retrieval later
349                 id.SetVersion(GetCurrentRun());
350
351                 result = AliCDBManager::Instance()->GetStorage(localUri)
352                                         ->Put(object, id, metaData);
353
354                 if(result) {
355                         result = 2;
356                         fGridError = kTRUE;
357                 }else{
358                         Log(fCurrentDetector, "WriteToCDB - Can't store data!");
359                 }
360         }
361
362         return result;
363
364 }
365
366 //______________________________________________________________________________________________
367 AliShuttleStatus* AliShuttle::ReadShuttleStatus()
368 {
369 // Reads the AliShuttleStatus from the CDB
370
371         if (fStatusEntry){
372                 delete fStatusEntry;
373                 fStatusEntry = 0;
374         }
375
376         fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalCDB())
377                 ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), GetCurrentRun());
378
379         if (!fStatusEntry) return 0;
380         fStatusEntry->SetOwner(1);
381
382         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
383         if (!status) {
384                 AliError("Invalid object stored to CDB!");
385                 return 0;
386         }
387
388         return status;
389 }
390
391 //______________________________________________________________________________________________
392 Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
393 {
394 // writes the status for one subdetector
395
396         if (fStatusEntry){
397                 delete fStatusEntry;
398                 fStatusEntry = 0;
399         }
400
401         Int_t run = GetCurrentRun();
402
403         AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), run, run);
404
405         fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
406         fStatusEntry->SetOwner(1);
407
408         UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
409
410         if (!result) {
411                 AliError(Form("WriteShuttleStatus for %s, run %d failed", fCurrentDetector.Data(), run));
412                 return kFALSE;
413         }
414
415         return kTRUE;
416 }
417
418 //______________________________________________________________________________________________
419 void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
420 {
421   // changes the AliShuttleStatus for the given detector and run to the given status
422
423         if (!fStatusEntry){
424                 AliError("UNEXPECTED: fStatusEntry empty");
425                 return;
426         }
427
428         AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
429
430         if (!status){
431                 AliError("UNEXPECTED: status could not be read from current CDB entry");
432                 return;
433         }
434
435         TString actionStr = Form("UpdateShuttleStatus - %s: Changing state from %s to %s",
436                                 fCurrentDetector.Data(),
437                                 status->GetStatusName(),
438                                 status->GetStatusName(newStatus));
439         Log("SHUTTLE", actionStr);
440         SetLastAction(actionStr);
441
442         status->SetStatus(newStatus);
443         if (increaseCount) status->IncreaseCount();
444
445         AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
446 }
447 //______________________________________________________________________________________________
448 Bool_t AliShuttle::ContinueProcessing()
449 {
450 // this function reads the AliShuttleStatus information from CDB and
451 // checks if the processing should be continued
452 // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
453
454         if (!fConfig->HostProcessDetector(fCurrentDetector)) return kFALSE;
455
456         AliPreprocessor* aPreprocessor =
457                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
458         if (!aPreprocessor)
459         {
460                 AliInfo(Form("%s: no preprocessor registered", fCurrentDetector.Data()));
461                 return kFALSE;
462         }
463
464         AliShuttleLogbookEntry::Status entryStatus =
465                 fLogbookEntry->GetDetectorStatus(fCurrentDetector);
466
467         if(entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
468                 AliInfo(Form("ContinueProcessing - %s is %s",
469                                 fCurrentDetector.Data(),
470                                 fLogbookEntry->GetDetectorStatusName(entryStatus)));
471                 return kFALSE;
472         }
473
474         // if we get here, according to Shuttle logbook subdetector is in UNPROCESSED state
475
476         // check if current run is first unprocessed run for current detector
477         if (fConfig->StrictRunOrder(fCurrentDetector) &&
478                 !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
479         {
480                 Log("SHUTTLE", Form("ContinueProcessing - %s requires strict run ordering but this is not the first unprocessed run!"));
481                 return kFALSE;
482         }
483
484         AliShuttleStatus* status = ReadShuttleStatus();
485         if (!status) {
486                 // first time
487                 Log("SHUTTLE", Form("ContinueProcessing - %s: Processing first time",
488                                 fCurrentDetector.Data()));
489                 status = new AliShuttleStatus(AliShuttleStatus::kStarted);
490                 return WriteShuttleStatus(status);
491         }
492
493         // The following two cases shouldn't happen if Shuttle Logbook was correctly updated.
494         // If it happens it may mean Logbook updating failed... let's do it now!
495         if (status->GetStatus() == AliShuttleStatus::kDone ||
496             status->GetStatus() == AliShuttleStatus::kFailed){
497                 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s. Updating Shuttle Logbook",
498                                         fCurrentDetector.Data(),
499                                         status->GetStatusName(status->GetStatus())));
500                 UpdateShuttleLogbook(fCurrentDetector.Data(),
501                                         status->GetStatusName(status->GetStatus()));
502                 return kFALSE;
503         }
504
505         if (status->GetStatus() == AliShuttleStatus::kStoreFailed) {
506                 Log("SHUTTLE",
507                         Form("ContinueProcessing - %s: Grid storage of one or more objects failed. Trying again now",
508                                 fCurrentDetector.Data()));
509                 if(TryToStoreAgain()){
510                         Log(fCurrentDetector.Data(), "ContinueProcessing - All objects successfully stored into OCDB");
511                         UpdateShuttleStatus(AliShuttleStatus::kDone);
512                         UpdateShuttleLogbook(fCurrentDetector.Data(), "DONE");
513                 } else {
514                         Log("SHUTTLE",
515                                 Form("ContinueProcessing - %s: Grid storage failed again",
516                                         fCurrentDetector.Data()));
517                 }
518                 return kFALSE;
519         }
520
521         // if we get here, there is a restart
522         Bool_t cont = kFALSE;
523
524         // abort conditions
525         if (status->GetCount() >= fConfig->GetMaxRetries()) {
526                 Log("SHUTTLE", Form("ContinueProcessing - %s failed %d times in status %s - "
527                                 "Updating Shuttle Logbook", fCurrentDetector.Data(),
528                                 status->GetCount(), status->GetStatusName()));
529                 UpdateShuttleLogbook(fCurrentDetector.Data(), "FAILED");
530         } else {
531                 Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. "
532                                 "Aborted before with %s. Retry number %d.", fCurrentDetector.Data(),
533                                 status->GetStatusName(), status->GetCount()));
534                 UpdateShuttleStatus(AliShuttleStatus::kStarted, kTRUE);
535                 cont = kTRUE;
536         }
537
538         // Send mail to detector expert!
539         AliInfo(Form("Sending mail to %s expert...", fCurrentDetector.Data()));
540         if (!SendMail())
541                 Log("SHUTTLE", Form("ContinueProcessing - Could not send mail to %s expert",
542                                 fCurrentDetector.Data()));
543
544         return cont;
545 }
546
547 //______________________________________________________________________________________________
548 Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
549 {
550         //
551         // Makes data retrieval for all detectors in the configuration.
552         // entry: Shuttle logbook entry, contains run paramenters and status of detectors
553         // (Unprocessed, Inactive, Failed or Done).
554         // Returns kFALSE in case of error occured and kTRUE otherwise
555         //
556
557         if(!entry) return kFALSE;
558
559         fLogbookEntry = entry;
560
561         if(fLogbookEntry->IsDone()){
562                 Log("SHUTTLE","Process - Shuttle is already DONE. Updating logbook");
563                 UpdateShuttleLogbook("shuttle_done");
564                 fLogbookEntry = 0;
565                 return kTRUE;
566         }
567
568
569         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^* \n",
570                                         GetCurrentRun()));
571
572         fLogbookEntry->Print("all");
573
574         // Initialization
575         Bool_t hasError = kFALSE;
576         for(Int_t iSys=0;iSys<3;iSys++) fFXSCalled[iSys]=kFALSE;
577
578         AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
579         if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
580         AliCDBStorage *mainRefSto = AliCDBManager::Instance()->GetStorage(fgkMainRefStorage);
581         if(mainRefSto) mainRefSto->QueryCDB(GetCurrentRun());
582
583         // Loop on detectors in the configuration
584         TIter iter(fConfig->GetDetectors());
585         TObjString* aDetector = 0;
586
587         while ((aDetector = (TObjString*) iter.Next()))
588         {
589                 fCurrentDetector = aDetector->String();
590
591                 if (ContinueProcessing() == kFALSE) continue;
592
593                 AliInfo(Form("\n\n \t\t\t****** run %d - %s: START  ******",
594                                                 GetCurrentRun(), aDetector->GetName()));
595
596
597                 Int_t pid = fork();
598
599                 if (pid < 0)
600                 {
601                         Log("SHUTTLE", "ERROR: Forking failed");
602                 }
603                 else if (pid > 0)
604                 {
605                         // parent
606                         AliInfo(Form("In parent process of %d - %s: Starting monitoring",
607                                                         GetCurrentRun(), aDetector->GetName()));
608
609                         Long_t begin = time(0);
610
611                         int status; // to be used with waitpid, on purpose an int (not Int_t)!
612                         while (waitpid(pid, &status, WNOHANG) == 0)
613                         {
614                                 Long_t expiredTime = time(0) - begin;
615
616                                 if (expiredTime > fConfig->GetPPTimeOut())
617                                 {
618                                         Log("SHUTTLE", Form("Process time out. Run time: %d seconds. Killing...",
619                                                                 expiredTime));
620
621                                         kill(pid, 9);
622
623                                         hasError = kTRUE;
624
625                                         gSystem->Sleep(1000);
626                                 }
627                                 else
628                                 {
629                                         if (expiredTime % 60 == 0)
630                                         Log("SHUTTLE", Form("Checked process. Run time: %d seconds.",
631                                                                 expiredTime));
632                                         gSystem->Sleep(1000);
633                                 }
634                         }
635
636                         AliInfo(Form("In parent process of %d - %s: Client has terminated.",
637                                                                 GetCurrentRun(), aDetector->GetName()));
638
639                         if (WIFEXITED(status))
640                         {
641                                 Int_t returnCode = WEXITSTATUS(status);
642
643                                 Log("SHUTTLE", Form("The return code is %d", returnCode));
644
645                                 if (returnCode != 0)
646                                 hasError = kTRUE;
647                         }
648                 }
649                 else if (pid == 0)
650                 {
651                         // client
652                         AliInfo(Form("In client process of %d - %s", GetCurrentRun(), aDetector->GetName()));
653
654                         UInt_t result = ProcessCurrentDetector();
655
656                         Int_t returnCode = 0; // will be set to 1 in case of an error
657
658                         if (!result)
659                         {
660                                 returnCode = 1;
661                                 AliInfo(Form("\n \t\t\t****** run %d - %s: PREPROCESSOR ERROR ****** \n\n",
662                                                         GetCurrentRun(), aDetector->GetName()));
663                         }
664                         else if (result == 2)
665                         {
666                                 AliInfo(Form("\n \t\t\t****** run %d - %s: STORAGE ERROR ****** \n\n",
667                                                         GetCurrentRun(), aDetector->GetName()));
668                         } else
669                         {
670                                 AliInfo(Form("\n \t\t\t****** run %d - %s: DONE ****** \n\n",
671                                                         GetCurrentRun(), aDetector->GetName()));
672                         }
673
674                         if (result > 0)
675                         {
676                                 // Process successful: Update time_processed field in FXS logbooks!
677                                 if (fFXSCalled[kDAQ])
678                                 {
679                                         if (UpdateDAQTable() == kFALSE)
680                                         returnCode = 1;
681                                         fFXSlist[kDAQ].Clear();
682                                 }
683                                 //if(fFXSCalled[kDCS]) {
684                                 //  if (UpdateDCSTable(aDetector->GetName()) == kFALSE)
685                                 //    returnCode = 1;
686                                 //  fFXSlist[kDCS].Clear();
687                                 //}
688                                 if (fFXSCalled[kHLT])
689                                 {
690                                         if (UpdateHLTTable() == kFALSE)
691                                         returnCode = 1;
692                                         fFXSlist[kHLT].Clear();
693                                 }
694                         }
695
696                         AliInfo(Form("Client process of %d - %s is exiting now with %d.",
697                                                         GetCurrentRun(), aDetector->GetName(), returnCode));
698
699                         // the client exits here
700                         gSystem->Exit(returnCode);
701
702                         AliError("We should never get here!!!");
703                 }
704         }
705
706         AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^* \n",
707                                                         GetCurrentRun()));
708
709         //check if shuttle is done for this run, if so update logbook
710         TObjArray checkEntryArray;
711         checkEntryArray.SetOwner(1);
712         TString whereClause = Form("where run=%d", GetCurrentRun());
713         if (!QueryShuttleLogbook(whereClause.Data(), checkEntryArray) || checkEntryArray.GetEntries() == 0) {
714                 Log("SHUTTLE", Form("Process - Warning: Cannot check status of run %d on Shuttle logbook!",
715                                                 GetCurrentRun()));
716                 return hasError == kFALSE;
717         }
718
719         AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
720                                                 (checkEntryArray.At(0));
721
722         if (checkEntry)
723         {
724                 if (checkEntry->IsDone())
725                 {
726                         Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
727                         UpdateShuttleLogbook("shuttle_done");
728                 }
729                 else
730                 {
731                         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
732                         {
733                                 if (checkEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
734                                 {
735                                         AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
736                                                         checkEntry->GetRun(), GetDetName(iDet)));
737                                         fFirstUnprocessed[iDet] = kFALSE;
738                                 }
739                         }
740                 }
741         }
742
743         fLogbookEntry = 0;
744
745         return hasError == kFALSE;
746 }
747
748 //______________________________________________________________________________________________
749 UInt_t AliShuttle::ProcessCurrentDetector()
750 {
751         //
752         // Makes data retrieval just for a specific detector (fCurrentDetector).
753         // Threre should be a configuration for this detector.
754
755         AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), GetCurrentRun()));
756
757         UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
758
759         TMap dcsMap;
760         dcsMap.SetOwner(1);
761
762         Bool_t aDCSError = kFALSE;
763         fGridError = kFALSE;
764
765         // TODO Test only... I've added a flag that allows to
766         // exclude DCS archive DB query
767         if (!fgkProcessDCS)
768         {
769                 AliInfo("Skipping DCS processing!");
770                 aDCSError = kFALSE;
771         } else {
772                 TString host(fConfig->GetDCSHost(fCurrentDetector));
773                 Int_t port = fConfig->GetDCSPort(fCurrentDetector);
774
775                 // Retrieval of Aliases
776                 TObjString* anAlias = 0;
777                 Int_t iAlias = 1;
778                 Int_t nTotAliases= ((TMap*)fConfig->GetDCSAliases(fCurrentDetector))->GetEntries();
779                 TIter iterAliases(fConfig->GetDCSAliases(fCurrentDetector));
780                 while ((anAlias = (TObjString*) iterAliases.Next()))
781                 {
782                         TObjArray *valueSet = new TObjArray();
783                         valueSet->SetOwner(1);
784
785                         if (((iAlias-1) % 500) == 0 || iAlias == nTotAliases)
786                                 AliInfo(Form("Querying DCS archive: alias %s (%d of %d)",
787                                                 anAlias->GetName(), iAlias++, nTotAliases));
788                         aDCSError = (GetValueSet(host, port, anAlias->String(), valueSet, kAlias) == 0);
789
790                         if(!aDCSError)
791                         {
792                                 dcsMap.Add(anAlias->Clone(), valueSet);
793                         } else {
794                                 Log(fCurrentDetector,
795                                         Form("ProcessCurrentDetector - Error while retrieving alias %s",
796                                                 anAlias->GetName()));
797                                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
798                                 dcsMap.DeleteAll();
799                                 return 0;
800                         }
801                 }
802
803                 // Retrieval of Data Points
804                 TObjString* aDP = 0;
805                 Int_t iDP = 0;
806                 Int_t nTotDPs= ((TMap*)fConfig->GetDCSDataPoints(fCurrentDetector))->GetEntries();
807                 TIter iterDP(fConfig->GetDCSDataPoints(fCurrentDetector));
808                 while ((aDP = (TObjString*) iterDP.Next()))
809                 {
810                         TObjArray *valueSet = new TObjArray();
811                         valueSet->SetOwner(1);
812                         if (((iDP-1) % 500) == 0 || iDP == nTotDPs)
813                                 AliInfo(Form("Querying DCS archive: DP %s (%d of %d)",
814                                                 aDP->GetName(), iDP++, nTotDPs));
815                         aDCSError = (GetValueSet(host, port, aDP->String(), valueSet, kDP) == 0);
816
817                         if(!aDCSError)
818                         {
819                                 dcsMap.Add(aDP->Clone(), valueSet);
820                         } else {
821                                 Log(fCurrentDetector,
822                                         Form("ProcessCurrentDetector - Error while retrieving data point %s",
823                                                 aDP->GetName()));
824                                 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
825                                 dcsMap.DeleteAll();
826                                 return 0;
827                         }
828                 }
829         }
830
831         // DCS Archive DB processing successful. Call Preprocessor!
832         UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
833
834         AliPreprocessor* aPreprocessor =
835                 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
836
837         aPreprocessor->Initialize(GetCurrentRun(), GetCurrentStartTime(), GetCurrentEndTime());
838         UInt_t aPPResult = aPreprocessor->Process(&dcsMap);
839
840         UInt_t returnValue = 0;
841         if (aPPResult == 0) { // Preprocessor error
842                 UpdateShuttleStatus(AliShuttleStatus::kPPError);
843                 returnValue = 0;
844         } else if (fGridError == kFALSE) { // process and Grid storage ok!
845                 UpdateShuttleStatus(AliShuttleStatus::kDone);
846                 UpdateShuttleLogbook(fCurrentDetector, "DONE");
847                 Log(fCurrentDetector.Data(),
848                         "ProcessCurrentDetector - Preprocessor and Grid storage ended successfully");
849                 returnValue = 1;
850         } else { // Grid storage error (process ok, but object put in local storage)
851                 UpdateShuttleStatus(AliShuttleStatus::kStoreFailed);
852                 returnValue = 2;
853         }
854
855         dcsMap.DeleteAll();
856
857         return returnValue;
858 }
859
860 //______________________________________________________________________________________________
861 Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
862                 TObjArray& entries)
863 {
864 // Query DAQ's Shuttle logbook and fills detector status object.
865 // Call QueryRunParameters to query DAQ logbook for run parameters.
866
867         entries.SetOwner(1);
868
869         // check connection, in case connect
870         if(!Connect(3)) return kFALSE;
871
872         TString sqlQuery;
873         sqlQuery = Form("select * from logbook_shuttle %s order by run", whereClause);
874
875         TSQLResult* aResult = fServer[3]->Query(sqlQuery);
876         if (!aResult) {
877                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
878                 return kFALSE;
879         }
880
881         AliDebug(2,Form("Query = %s", sqlQuery.Data()));
882
883         if(aResult->GetRowCount() == 0) {
884 //              if(sqlQuery.EndsWith("where shuttle_done=0 order by run")){
885 //                      Log("SHUTTLE", "QueryShuttleLogbook - All runs in Shuttle Logbook are already DONE");
886 //                      delete aResult;
887 //                      return kTRUE;
888 //              } else {
889                         AliInfo("No entries in Shuttle Logbook match request");
890                         delete aResult;
891                         return kTRUE;
892 //              }
893         }
894
895         // TODO Check field count!
896         const UInt_t nCols = 22;
897         if (aResult->GetFieldCount() != (Int_t) nCols) {
898                 AliError("Invalid SQL result field number!");
899                 delete aResult;
900                 return kFALSE;
901         }
902
903         TSQLRow* aRow;
904         while ((aRow = aResult->Next())) {
905                 TString runString(aRow->GetField(0), aRow->GetFieldLength(0));
906                 Int_t run = runString.Atoi();
907
908                 AliShuttleLogbookEntry *entry = QueryRunParameters(run);
909                 if (!entry)
910                         continue;
911
912                 // loop on detectors
913                 for(UInt_t ii = 0; ii < nCols; ii++)
914                         entry->SetDetectorStatus(aResult->GetFieldName(ii), aRow->GetField(ii));
915
916                 entries.AddLast(entry);
917                 delete aRow;
918         }
919
920 //      if(sqlQuery.EndsWith("where shuttle_done=0 order by run"))
921 //              Log("SHUTTLE", Form("QueryShuttleLogbook - Found %d unprocessed runs in Shuttle Logbook",
922 //                                                      entries.GetEntriesFast()));
923         delete aResult;
924         return kTRUE;
925 }
926
927 //______________________________________________________________________________________________
928 AliShuttleLogbookEntry* AliShuttle::QueryRunParameters(Int_t run)
929 {
930         //
931         // Retrieve run parameters written in the DAQ logbook and sets them into AliShuttleLogbookEntry object
932         //
933
934         // check connection, in case connect
935         if (!Connect(3))
936                 return 0;
937
938         TString sqlQuery;
939         sqlQuery.Form("select * from %s where run=%d", fConfig->GetDAQlbTable(), run);
940
941         TSQLResult* aResult = fServer[3]->Query(sqlQuery);
942         if (!aResult) {
943                 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
944                 return 0;
945         }
946
947         if (aResult->GetRowCount() == 0) {
948                 Log("SHUTTLE", Form("QueryRunParameters - No entry in DAQ Logbook for run %d. Skipping", run));
949                 delete aResult;
950                 return 0;
951         }
952
953         if (aResult->GetRowCount() > 1) {
954                 AliError(Form("More than one entry in DAQ Logbook for run %d. Skipping", run));
955                 delete aResult;
956                 return 0;
957         }
958
959         TSQLRow* aRow = aResult->Next();
960         if (!aRow)
961         {
962                 AliError(Form("Could not retrieve row for run %d. Skipping", run));
963                 delete aResult;
964                 return 0;
965         }
966
967         AliShuttleLogbookEntry* entry = new AliShuttleLogbookEntry(run);
968
969         for (Int_t ii = 0; ii < aResult->GetFieldCount(); ii++)
970                 entry->SetRunParameter(aResult->GetFieldName(ii), aRow->GetField(ii));
971
972         UInt_t startTime = entry->GetStartTime();
973         UInt_t endTime = entry->GetEndTime();
974
975         if (!startTime || !endTime || startTime > endTime) {
976                 Log("SHUTTLE",
977                         Form("QueryRunParameters - Invalid parameters for Run %d: startTime = %d, endTime = %d",
978                                 run, startTime, endTime));
979                 delete entry;
980                 delete aRow;
981                 delete aResult;
982                 return 0;
983         }
984
985         delete aRow;
986         delete aResult;
987
988         return entry;
989 }
990
991 //______________________________________________________________________________________________
992 Bool_t AliShuttle::TryToStoreAgain()
993 {
994   // Called in case the detector failed to store the object in Grid OCDB
995   // It tries to store the object again, if it does not find more recent and overlapping objects
996   // Calls underlying TryToStoreAgain(const char*) function twice, for OCDB and Reference storage.
997
998         AliInfo("Trying to store OCDB data again...");
999         Bool_t resultCDB = TryToStoreAgain(fgkMainCDB);
1000
1001         AliInfo("Trying to store reference data again...");
1002         Bool_t resultRef = TryToStoreAgain(fgkMainRefStorage);
1003
1004         return resultCDB && resultRef;
1005 }
1006
1007 //______________________________________________________________________________________________
1008 Bool_t AliShuttle::TryToStoreAgain(TString& gridURI)
1009 {
1010   // Called by TryToStoreAgain(), performs actual storage retry
1011
1012         TObjArray* gridIds=0;
1013
1014         Bool_t result = kTRUE;
1015
1016         const char* type = 0;
1017         TString backupURI;
1018         if(gridURI == fgkMainCDB) {
1019                 type = "OCDB";
1020                 backupURI = fgkLocalCDB;
1021         } else if(gridURI == fgkMainRefStorage) {
1022                 type = "reference";
1023                 backupURI = fgkLocalRefStorage;
1024         } else {
1025                 AliError(Form("Invalid storage URI: %s", gridURI.Data()));
1026                 return kFALSE;
1027         }
1028
1029         AliCDBManager* man = AliCDBManager::Instance();
1030
1031         AliCDBStorage *gridSto = man->GetStorage(gridURI);
1032         if(!gridSto) {
1033                 Log(fCurrentDetector.Data(),
1034                         Form("TryToStoreAgain - cannot activate main %s storage", type));
1035                 return kFALSE;
1036         }
1037
1038         gridIds = gridSto->GetQueryCDBList();
1039
1040         // get objects previously stored in local CDB
1041         AliCDBStorage *backupSto = man->GetStorage(backupURI);
1042         AliCDBPath aPath(GetOfflineDetName(fCurrentDetector.Data()),"*","*");
1043         // Local objects were stored with current run as Grid version!
1044         TList* localEntries = backupSto->GetAll(aPath.GetPath(), GetCurrentRun(), GetCurrentRun());
1045         localEntries->SetOwner(1);
1046
1047         // loop on local stored objects
1048         TIter localIter(localEntries);
1049         AliCDBEntry *aLocEntry = 0;
1050         while((aLocEntry = dynamic_cast<AliCDBEntry*> (localIter.Next()))){
1051                 aLocEntry->SetOwner(1);
1052                 AliCDBId aLocId = aLocEntry->GetId();
1053                 aLocEntry->SetVersion(-1);
1054                 aLocEntry->SetSubVersion(-1);
1055
1056                 // loop on Grid valid Id's
1057                 Bool_t store = kTRUE;
1058                 TIter gridIter(gridIds);
1059                 AliCDBId* aGridId = 0;
1060                 while((aGridId = dynamic_cast<AliCDBId*> (gridIter.Next()))){
1061                         // If local object is valid up to infinity we store it only if it is
1062                         // the first unprocessed run!
1063                         if (aLocId.GetLastRun() == AliCDBRunRange::Infinity())
1064                         {
1065                                 if (!fFirstUnprocessed[GetDetPos(fCurrentDetector)])
1066                                 {
1067                                         Log(fCurrentDetector.Data(),
1068                                                 ("TryToStoreAgain - This object has validity infinite but "
1069                                                  "there are previous unprocessed runs!"));
1070                                         continue;
1071                                 } else {
1072                                         break;
1073                                 }
1074                         }
1075                         if(aGridId->GetPath() != aLocId.GetPath()) continue;
1076                         // skip all objects valid up to infinity
1077                         if(aGridId->GetLastRun() == AliCDBRunRange::Infinity()) continue;
1078                         // if we get here, it means there's already some more recent object stored on Grid!
1079                         store = kFALSE;
1080                         break;
1081                 }
1082
1083                 if(!store){
1084                         Log(fCurrentDetector.Data(),
1085                                 Form("TryToStoreAgain - A more recent object already exists in %s storage: <%s>",
1086                                         type, aGridId->ToString().Data()));
1087                         // removing local filename...
1088                         // TODO maybe it's better not to remove it, it was not copied to the Grid!
1089                         TString filename;
1090                         backupSto->IdToFilename(aLocId, filename);
1091                         AliInfo(Form("Removing local file %s", filename.Data()));
1092                         gSystem->Exec(Form("rm %s",filename.Data()));
1093                         continue;
1094                 }
1095
1096                 // If we get here, the file can be stored!
1097                 Bool_t storeOk = gridSto->Put(aLocEntry);
1098                 if(storeOk){
1099                         Log(fCurrentDetector.Data(),
1100                                 Form("TryToStoreAgain - Object <%s> successfully put into %s storage",
1101                                         aLocId.ToString().Data(), type));
1102
1103                         // removing local filename...
1104                         TString filename;
1105                         backupSto->IdToFilename(aLocId, filename);
1106                         AliInfo(Form("Removing local file %s", filename.Data()));
1107                         gSystem->Exec(Form("rm %s", filename.Data()));
1108                         continue;
1109                 } else  {
1110                         Log(fCurrentDetector.Data(),
1111                                 Form("TryToStoreAgain - Grid %s storage of object <%s> failed again",
1112                                         type, aLocId.ToString().Data()));
1113                         result = kFALSE;
1114                 }
1115         }
1116         localEntries->Clear();
1117
1118         return result;
1119 }
1120
1121 //______________________________________________________________________________________________
1122 Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* entry,
1123                                 TObjArray* valueSet, DCSType type)
1124 {
1125 // Retrieve all "entry" data points from the DCS server
1126 // host, port: TSocket connection parameters
1127 // entry: name of the alias or data point
1128 // valueSet: array of retrieved AliDCSValue's
1129 // type: kAlias or kDP
1130
1131         AliDCSClient client(host, port, fTimeout, fRetries);
1132         if (!client.IsConnected())
1133         {
1134                 return kFALSE;
1135         }
1136
1137         Int_t result=0;
1138
1139         if (type == kAlias)
1140         {
1141                 result = client.GetAliasValues(entry,
1142                         GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
1143         } else
1144         if (type == kDP)
1145         {
1146                 result = client.GetDPValues(entry,
1147                         GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
1148         }
1149
1150         if (result < 0)
1151         {
1152                 Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get '%s'! Reason: %s",
1153                         entry, AliDCSClient::GetErrorString(result)));
1154
1155                 if (result == AliDCSClient::fgkServerError)
1156                 {
1157                         Log(fCurrentDetector.Data(), Form("GetValueSet - Server error: %s",
1158                                 client.GetServerError().Data()));
1159                 }
1160
1161                 return kFALSE;
1162         }
1163
1164         return kTRUE;
1165 }
1166
1167 //______________________________________________________________________________________________
1168 const char* AliShuttle::GetFile(Int_t system, const char* detector,
1169                 const char* id, const char* source)
1170 {
1171 // Get calibration file from file exchange servers
1172 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
1173
1174         switch(system){
1175                 case kDAQ:
1176                         return GetDAQFileName(detector, id, source);
1177                         break;
1178                 case kDCS:
1179                         return GetDCSFileName(detector, id, source);
1180                         break;
1181                 case kHLT:
1182                         return GetHLTFileName(detector, id, source);
1183                         break;
1184                 default:
1185                         AliError(Form("No valid system index: %d",system));
1186         }
1187
1188         return 0;
1189 }
1190
1191 //______________________________________________________________________________________________
1192 TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
1193 {
1194 // Get sources producing the condition file Id from file exchange servers
1195 // calls specific getter according to system index (kDAQ, kDCS, kHLT)
1196
1197         switch(system){
1198                 case kDAQ:
1199                         return GetDAQFileSources(detector, id);
1200                         break;
1201                 case kDCS:
1202                         return GetDCSFileSources(detector, id);
1203                         break;
1204                 case kHLT:
1205                         return GetHLTFileSources(detector, id);
1206                         break;
1207                 default:
1208                         AliError(Form("No valid system index: %d",system));
1209         }
1210
1211         return NULL;
1212 }
1213
1214 //______________________________________________________________________________________________
1215 Bool_t AliShuttle::Connect(Int_t system)
1216 {
1217 // Connect to MySQL Server of the system's FXS MySQL databases
1218 // DAQ Logbook, Shuttle Logbook and DAQ FXS db are on the same host
1219
1220         // check connection: if already connected return
1221         if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
1222
1223         TString dbHost, dbUser, dbPass, dbName;
1224
1225         if (system < 3) // FXS db servers
1226         {
1227                 dbHost = Form("mysql://%s:%d", fConfig->GetFXSdbHost(system), fConfig->GetFXSdbPort(system));
1228                 dbUser = fConfig->GetFXSdbUser(system);
1229                 dbPass = fConfig->GetFXSdbPass(system);
1230                 dbName =   fConfig->GetFXSdbName(system);
1231         } else { // Run & Shuttle logbook servers
1232         // TODO Will the Shuttle logbook server be the same as the Run logbook server ???
1233                 dbHost = Form("mysql://%s:%d", fConfig->GetDAQlbHost(), fConfig->GetDAQlbPort());
1234                 dbUser = fConfig->GetDAQlbUser();
1235                 dbPass = fConfig->GetDAQlbPass();
1236                 dbName =   fConfig->GetDAQlbDB();
1237         }
1238
1239         fServer[system] = TSQLServer::Connect(dbHost.Data(), dbUser.Data(), dbPass.Data());
1240         if (!fServer[system] || !fServer[system]->IsConnected()) {
1241                 if(system < 3)
1242                 {
1243                 AliError(Form("Can't establish connection to FXS database for %s",
1244                                         AliShuttleInterface::GetSystemName(system)));
1245                 } else {
1246                 AliError("Can't establish connection to Run logbook.");
1247                 }
1248                 if(fServer[system]) delete fServer[system];
1249                 return kFALSE;
1250         }
1251
1252         // Get tables
1253         // TODO in the configuration should the table name be there too?
1254         TSQLResult* aResult=0;
1255         switch(system){
1256                 case kDAQ:
1257                         aResult = fServer[kDAQ]->GetTables(dbName.Data());
1258                         break;
1259                 case kDCS:
1260                         //aResult = fServer[kDCS]->GetTables(dbName.Data());
1261                         break;
1262                 case kHLT:
1263                         aResult = fServer[kHLT]->GetTables(dbName.Data());
1264                         break;
1265                 default:
1266                         aResult = fServer[3]->GetTables(dbName.Data());
1267                         break;
1268         }
1269
1270         delete aResult;
1271         return kTRUE;
1272 }
1273
1274 //______________________________________________________________________________________________
1275 const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source)
1276 {
1277 // Retrieves a file from the DAQ FXS.
1278 // First queris the DAQ FXS database for the DAQ file name, using the run, detector, id and source info
1279 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
1280 // run: current run being processed (given by Logbook entry fLogbookEntry)
1281 // detector: the Preprocessor name
1282 // id: provided as a parameter by the Preprocessor
1283 // source: provided by the Preprocessor through GetFileSources function
1284
1285         // check connection, in case connect
1286         if (!Connect(kDAQ))
1287         {
1288                 Log(detector, "GetDAQFileName - Couldn't connect to DAQ FXS database");
1289                 return 0;
1290         }
1291
1292         // Query preparation
1293         TString sqlQueryStart = Form("select filePath from %s where", fConfig->GetFXSdbTable(kDAQ));
1294         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
1295                                 GetCurrentRun(), detector, id, source);
1296         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1297
1298         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1299
1300         // Query execution
1301         TSQLResult* aResult = 0;
1302         aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1303         if (!aResult) {
1304                 Log(detector, Form("GetDAQFileName - Can't execute SQL query for: id = %s, source = %s",
1305                                 id, source));
1306                 return 0;
1307         }
1308
1309         if(aResult->GetRowCount() == 0)
1310         {
1311                 Log(detector,
1312                         Form("GetDAQFileName - No entry in FXS table for: id = %s, source = %s",
1313                                 id, source));
1314                 delete aResult;
1315                 return 0;
1316         }
1317
1318         if (aResult->GetRowCount() > 1) {
1319                 Log(detector,
1320                         Form("GetDAQFileName - More than one entry in FXS table for: id = %s, source = %s",
1321                                 id, source));
1322                 delete aResult;
1323                 return 0;
1324         }
1325
1326         TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
1327
1328         if (!aRow){
1329                 Log(detector, Form("GetDAQFileName - Empty set result from query: id = %s, source = %s",
1330                                 id, source));
1331                 delete aResult;
1332                 return 0;
1333         }
1334
1335         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
1336
1337         delete aResult;
1338         delete aRow;
1339
1340         AliDebug(2, Form("filePath = %s",filePath.Data()));
1341
1342         // retrieved file is renamed to make it unique
1343         TString localFileName = Form("DAQ_%s_%d_%s_%s.shuttle",
1344                                         detector, GetCurrentRun(), id, source);
1345
1346         // file retrieval from DAQ FXS
1347         Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
1348         if(!result) {
1349                 Log(detector, Form("GetDAQFileName - Copy of file %s from DAQ FXS failed", filePath.Data()));
1350                 return 0;
1351         } else {
1352                 AliInfo(Form("File %s copied from DAQ FXS into %s/%s",
1353                         filePath.Data(), GetShuttleTempDir(), localFileName.Data()));
1354         }
1355
1356         fFXSCalled[kDAQ]=kTRUE;
1357         TObjString *fileParams = new TObjString(Form("%s#!?!#%s", id, source));
1358         fFXSlist[kDAQ].Add(fileParams);
1359
1360         static TString fullLocalFileName;
1361         fullLocalFileName = TString::Format("%s/%s", GetShuttleTempDir(), localFileName.Data());
1362
1363         AliInfo(Form("fullLocalFileName = %s", fullLocalFileName.Data()));
1364
1365         return fullLocalFileName.Data();
1366
1367 }
1368
1369 //______________________________________________________________________________________________
1370 Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName)
1371 {
1372 // Copies file from DAQ FXS to local Shuttle machine
1373
1374         // check temp directory: trying to cd to temp; if it does not exist, create it
1375         AliDebug(2, Form("Copy file %s from DAQ FXS into %s/%s",
1376                         daqFileName, GetShuttleTempDir(), localFileName));
1377
1378         void* dir = gSystem->OpenDirectory(GetShuttleTempDir());
1379         if (dir == NULL) {
1380                 if (gSystem->mkdir(GetShuttleTempDir(), kTRUE)) {
1381                         AliError(Form("Can't open directory <%s>", GetShuttleTempDir()));
1382                         return kFALSE;
1383                 }
1384
1385         } else {
1386                 gSystem->FreeDirectory(dir);
1387         }
1388
1389         TString baseDAQFXSFolder = "FES";
1390         TString command = Form("scp -oPort=%d -2 %s@%s:%s/%s %s/%s",
1391                 fConfig->GetFXSPort(kDAQ),
1392                 fConfig->GetFXSUser(kDAQ),
1393                 fConfig->GetFXSHost(kDAQ),
1394                 baseDAQFXSFolder.Data(),
1395                 daqFileName,
1396                 GetShuttleTempDir(),
1397                 localFileName);
1398
1399         AliDebug(2, Form("%s",command.Data()));
1400
1401         UInt_t nRetries = 0;
1402         UInt_t maxRetries = 3;
1403
1404         // copy!! if successful TSystem::Exec returns 0
1405         while(nRetries++ < maxRetries) {
1406                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
1407                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
1408         }
1409
1410         return kFALSE;
1411
1412 }
1413
1414 //______________________________________________________________________________________________
1415 TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id)
1416 {
1417 // Retrieves list of DAQ sources of file Id
1418
1419         // check connection, in case connect
1420         if(!Connect(kDAQ)){
1421                 Log(detector, "GetDAQFileSources - Couldn't connect to DAQ FXS database");
1422                 return 0;
1423         }
1424
1425         // Query preparation
1426         TString sqlQueryStart = Form("select DAQsource from %s where", fConfig->GetFXSdbTable(kDAQ));
1427         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
1428                                 GetCurrentRun(), detector, id);
1429         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1430
1431         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1432
1433         // Query execution
1434         TSQLResult* aResult;
1435         aResult = fServer[kDAQ]->Query(sqlQuery);
1436         if (!aResult) {
1437                 Log(detector, Form("GetDAQFileSources - Can't execute SQL query for id: %s", id));
1438                 return 0;
1439         }
1440
1441         if (aResult->GetRowCount() == 0) {
1442                 Log(detector,
1443                         Form("GetDAQFileSources - No entry in FXS table for id: %s", id));
1444                 delete aResult;
1445                 return 0;
1446         }
1447
1448         TSQLRow* aRow;
1449         TList *list = new TList();
1450         list->SetOwner(1);
1451
1452         while((aRow = aResult->Next())){
1453
1454                 TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
1455                 AliDebug(2, Form("daqSource = %s", daqSource.Data()));
1456                 list->Add(new TObjString(daqSource));
1457                 delete aRow;
1458         }
1459         delete aResult;
1460
1461         return list;
1462
1463 }
1464
1465 //______________________________________________________________________________________________
1466 const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1467 // Retrieves a file from the DCS FXS.
1468
1469 return "You're in DCS";
1470
1471 }
1472
1473 //______________________________________________________________________________________________
1474 TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
1475 // Retrieves file sources from the DCS FXS.
1476
1477 return NULL;
1478
1479 }
1480
1481 //______________________________________________________________________________________________
1482 const char* AliShuttle::GetHLTFileName(const char* detector, const char* id, const char* source){
1483 // Retrieves a file from the HLT FXS.
1484 // First queris the HLT FXS database for the HLT file name, using the run, detector, id and source info
1485 // then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
1486 // run: current run being processed (given by Logbook entry fLogbookEntry)
1487 // detector: the Preprocessor name
1488 // id: provided as a parameter by the Preprocessor
1489 // source: provided by the Preprocessor through GetFileSources function
1490
1491         // check connection, in case connect
1492         if (!Connect(kHLT))
1493         {
1494                 Log(detector, "GetHLTFileName - Couldn't connect to HLT FXS database");
1495                 return 0;
1496         }
1497
1498         // Query preparation
1499         TString sqlQueryStart = Form("select filePath,fileSize,fileChecksum from %s where",
1500                                                                                 fConfig->GetFXSdbTable(kHLT));
1501         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DDLnumbers=\"%s\"",
1502                                 GetCurrentRun(), detector, id, source);
1503         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1504
1505         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1506
1507         // Query execution
1508         TSQLResult* aResult = 0;
1509         aResult = dynamic_cast<TSQLResult*> (fServer[kHLT]->Query(sqlQuery));
1510         if (!aResult) {
1511                 Log(detector, Form("GetHLTFileName - Can't execute SQL query for: id = %s, source = %s",
1512                                 id, source));
1513                 return 0;
1514         }
1515
1516         if(aResult->GetRowCount() == 0)
1517         {
1518                 Log(detector,
1519                         Form("GetHLTFileName - No entry in FXS table for: id = %s, source = %s",
1520                                 id, source));
1521                 delete aResult;
1522                 return 0;
1523         }
1524
1525         if (aResult->GetRowCount() > 1) {
1526                 Log(detector,
1527                         Form("GetHLTFileName - More than one entry in FXS table for: id = %s, source = %s",
1528                                 id, source));
1529                 delete aResult;
1530                 return 0;
1531         }
1532
1533         if (aResult->GetFieldCount() != 3) {
1534                 Log(detector,
1535                         Form("GetHLTFileName - Wrong field count in FXS table for: id = %s, source = %s",
1536                                 id, source));
1537                 delete aResult;
1538                 return 0;
1539         }
1540
1541         TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
1542
1543         if (!aRow){
1544                 Log(detector, Form("GetHLTFileName - Empty set result from query: id = %s, source = %s",
1545                                 id, source));
1546                 delete aResult;
1547                 return 0;
1548         }
1549
1550         TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
1551         TString fileSize(aRow->GetField(1), aRow->GetFieldLength(1));
1552         TString fileMd5Sum(aRow->GetField(2), aRow->GetFieldLength(2));
1553
1554         delete aResult;
1555         delete aRow;
1556
1557         AliDebug(2, Form("filePath = %s",filePath.Data()));
1558
1559         // The full file path in HLT FXS is runNb/DET/DDLnumber/filePath
1560 //      TString fullFilePath = Form("%d/%s/%s/%s", GetCurrentRun(), detector, source, filePath.Data());
1561
1562         // retrieved file is renamed to make it unique
1563         TString localFileName = Form("HLT_%s_%d_%s_%s.shuttle",
1564                                         detector, GetCurrentRun(), id, source);
1565
1566         // file retrieval from HLT FXS
1567         Bool_t result = RetrieveHLTFile(filePath.Data(), localFileName.Data());
1568         if(!result)
1569         {
1570                 Log(detector, Form("GetHLTFileName - Copy of file %s from HLT FXS failed", filePath.Data()));
1571                 return 0;
1572         } else {
1573                 AliInfo(Form("File %s copied from HLT FXS into %s/%s",
1574                         filePath.Data(), GetShuttleTempDir(), localFileName.Data()));
1575         }
1576
1577         // compare md5sum of local file with the one stored in the HLT DB
1578         Int_t md5Comp = gSystem->Exec(Form("md5sum %s/%s |grep %s 2>&1 > /dev/null",
1579                                                 GetShuttleTempDir(), localFileName.Data(), fileMd5Sum.Data()));
1580
1581         if (md5Comp != 0)
1582         {
1583                 Log(detector, Form("GetHLTFileName - md5sum of file %s does not match with local copy!", filePath.Data()));
1584                 return 0;
1585         }
1586
1587         fFXSCalled[kHLT]=kTRUE;
1588         TObjString *fileParams = new TObjString(Form("%s#!?!#%s", id, source));
1589         fFXSlist[kHLT].Add(fileParams);
1590
1591         static TString fullLocalFileName;
1592         fullLocalFileName = TString::Format("%s/%s", GetShuttleTempDir(), localFileName.Data());
1593
1594         AliInfo(Form("fullLocalFileName = %s", fullLocalFileName.Data()));
1595
1596         return fullLocalFileName.Data();
1597
1598 }
1599
1600 //______________________________________________________________________________________________
1601 Bool_t AliShuttle::RetrieveHLTFile(const char* hltFileName, const char* localFileName)
1602 {
1603 // Copies file from HLT FXS to local Shuttle machine
1604
1605         // check temp directory: trying to cd to temp; if it does not exist, create it
1606         AliDebug(2, Form("Copy file %s from HLT FXS into %s/%s",
1607                         hltFileName, GetShuttleTempDir(), localFileName));
1608
1609         void* dir = gSystem->OpenDirectory(GetShuttleTempDir());
1610         if (dir == NULL) {
1611                 if (gSystem->mkdir(GetShuttleTempDir(), kTRUE)) {
1612                         AliError(Form("Can't open directory <%s>", GetShuttleTempDir()));
1613                         return kFALSE;
1614                 }
1615
1616         } else {
1617                 gSystem->FreeDirectory(dir);
1618         }
1619
1620         TString baseHLTFXSFolder = "~";
1621         TString command = Form("scp -oPort=%d %s@%s:%s/%s %s/%s",
1622                 fConfig->GetFXSPort(kHLT),
1623                 fConfig->GetFXSUser(kHLT),
1624                 fConfig->GetFXSHost(kHLT),
1625                 baseHLTFXSFolder.Data(),
1626                 hltFileName,
1627                 GetShuttleTempDir(),
1628                 localFileName);
1629
1630         AliDebug(2, Form("%s",command.Data()));
1631
1632         UInt_t nRetries = 0;
1633         UInt_t maxRetries = 3;
1634
1635         // copy!! if successful TSystem::Exec returns 0
1636         while(nRetries++ < maxRetries) {
1637                 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
1638                 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
1639         }
1640
1641         return kFALSE;
1642
1643 }
1644
1645 //______________________________________________________________________________________________
1646 TList* AliShuttle::GetHLTFileSources(const char* detector, const char* id){
1647 // Retrieves list of HLT sources (DDLnumbers) of file Id
1648
1649         // check connection, in case connect
1650         if(!Connect(kHLT)){
1651                 Log(detector, "GetHLTFileSources - Couldn't connect to HLT FXS database");
1652                 return 0;
1653         }
1654
1655         // Query preparation
1656         TString sqlQueryStart = Form("select DDLnumbers from %s where", fConfig->GetFXSdbTable(kHLT));
1657         TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
1658                                 GetCurrentRun(), detector, id);
1659         TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
1660
1661         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1662
1663         // Query execution
1664         TSQLResult* aResult;
1665         aResult = fServer[kHLT]->Query(sqlQuery);
1666         if (!aResult) {
1667                 Log(detector, Form("GetHLTFileSources - Can't execute SQL query for id: %s", id));
1668                 return 0;
1669         }
1670
1671         if (aResult->GetRowCount() == 0) {
1672                 Log(detector,
1673                         Form("GetHLTFileSources - No entry in FXS table for id: %s", id));
1674                 delete aResult;
1675                 return 0;
1676         }
1677
1678         TSQLRow* aRow;
1679         TList *list = new TList();
1680         list->SetOwner(1);
1681
1682         while((aRow = aResult->Next())){
1683
1684                 TString ddlNumbers(aRow->GetField(0), aRow->GetFieldLength(0));
1685                 AliDebug(2, Form("DDLnumbers = %s", ddlNumbers.Data()));
1686                 list->Add(new TObjString(ddlNumbers));
1687                 delete aRow;
1688         }
1689         delete aResult;
1690
1691         return list;
1692
1693 }
1694
1695 //______________________________________________________________________________________________
1696 Bool_t AliShuttle::UpdateDAQTable()
1697 {
1698 // Update DAQ table filling time_processed field in all rows corresponding to current run and detector
1699
1700         // check connection, in case connect
1701         if(!Connect(kDAQ)){
1702                 Log(fCurrentDetector, "UpdateDAQTable - Couldn't connect to DAQ FXS database");
1703                 return kFALSE;
1704         }
1705
1706         TTimeStamp now; // now
1707
1708         // Loop on FXS list entries
1709         TIter iter(&fFXSlist[kDAQ]);
1710         TObjString *aFXSentry=0;
1711         while((aFXSentry = dynamic_cast<TObjString*> (iter.Next()))){
1712                 TString aFXSentrystr = aFXSentry->String();
1713                 TObjArray *aFXSarray = aFXSentrystr.Tokenize("#!?!#");
1714                 if(!aFXSarray || aFXSarray->GetEntries() != 2 ) {
1715                         Log(fCurrentDetector, Form("UpdateDAQTable - error updating FXS entry. Check string: <%s>",
1716                                 aFXSentrystr.Data()));
1717                         if(aFXSarray) delete aFXSarray;
1718                         return kFALSE;
1719                 }
1720                 const char* fileId = ((TObjString*) aFXSarray->At(0))->GetName();
1721                 const char* daqSource = ((TObjString*) aFXSarray->At(1))->GetName();
1722                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
1723                         GetCurrentRun(), fCurrentDetector.Data(), fileId, daqSource);
1724
1725                 delete aFXSarray;
1726
1727                 TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(kDAQ),
1728                                                         now.GetSec(), whereClause.Data());
1729
1730                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1731
1732                 // Query execution
1733                 TSQLResult* aResult;
1734                 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
1735                 if (!aResult) {
1736                         Log(fCurrentDetector, Form("UpdateDAQTable - Can't execute SQL query <%s>", sqlQuery.Data()));
1737                         return kFALSE;
1738                 }
1739                 delete aResult;
1740         }
1741
1742         return kTRUE;
1743 }
1744
1745 //______________________________________________________________________________________________
1746 Bool_t AliShuttle::UpdateHLTTable()
1747 {
1748 // Update HLT table filling time_processed field in all rows corresponding to current run and detector
1749
1750         // check connection, in case connect
1751         if(!Connect(kHLT)){
1752                 Log(fCurrentDetector, "UpdateHLTTable - Couldn't connect to HLT FXS database");
1753                 return kFALSE;
1754         }
1755
1756         TTimeStamp now; // now
1757
1758         // Loop on FXS list entries
1759         TIter iter(&fFXSlist[kHLT]);
1760         TObjString *aFXSentry=0;
1761         while((aFXSentry = dynamic_cast<TObjString*> (iter.Next()))){
1762                 TString aFXSentrystr = aFXSentry->String();
1763                 TObjArray *aFXSarray = aFXSentrystr.Tokenize("#!?!#");
1764                 if(!aFXSarray || aFXSarray->GetEntries() != 2 ) {
1765                         Log(fCurrentDetector, Form("UpdateHLTTable - error updating FXS entry. Check string: <%s>",
1766                                 aFXSentrystr.Data()));
1767                         if(aFXSarray) delete aFXSarray;
1768                         return kFALSE;
1769                 }
1770                 const char* fileId = ((TObjString*) aFXSarray->At(0))->GetName();
1771                 const char* hltSource = ((TObjString*) aFXSarray->At(1))->GetName();
1772                 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DDLnumbers=\"%s\";",
1773                         GetCurrentRun(), fCurrentDetector.Data(), fileId, hltSource);
1774
1775                 delete aFXSarray;
1776
1777                 TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(kHLT),
1778                                                         now.GetSec(), whereClause.Data());
1779
1780                 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1781
1782                 // Query execution
1783                 TSQLResult* aResult;
1784                 aResult = dynamic_cast<TSQLResult*> (fServer[kHLT]->Query(sqlQuery));
1785                 if (!aResult) {
1786                         Log(fCurrentDetector, Form("UpdateHLTTable - Can't execute SQL query <%s>", sqlQuery.Data()));
1787                         return kFALSE;
1788                 }
1789                 delete aResult;
1790         }
1791
1792         return kTRUE;
1793 }
1794
1795 //______________________________________________________________________________________________
1796 Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status)
1797 {
1798 // Update Shuttle logbook filling detector or shuttle_done column
1799 // ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
1800
1801         // check connection, in case connect
1802         if(!Connect(3)){
1803                 Log("SHUTTLE", "UpdateShuttleLogbook - Couldn't connect to DAQ Logbook.");
1804                 return kFALSE;
1805         }
1806
1807         TString detName(detector);
1808         TString setClause;
1809         if(detName == "shuttle_done") {
1810                 setClause = "set shuttle_done=1";
1811         } else {
1812                 TString statusStr(status);
1813                 if(statusStr.Contains("done", TString::kIgnoreCase) ||
1814                    statusStr.Contains("failed", TString::kIgnoreCase)){
1815                         setClause = Form("set %s=\"%s\"", detector, status);
1816                 } else {
1817                         Log("SHUTTLE",
1818                                 Form("UpdateShuttleLogbook - Invalid status <%s> for detector %s",
1819                                         status, detector));
1820                         return kFALSE;
1821                 }
1822         }
1823
1824         TString whereClause = Form("where run=%d", GetCurrentRun());
1825
1826         TString sqlQuery = Form("update logbook_shuttle %s %s",
1827                                         setClause.Data(), whereClause.Data());
1828
1829         AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
1830
1831         // Query execution
1832         TSQLResult* aResult;
1833         aResult = dynamic_cast<TSQLResult*> (fServer[3]->Query(sqlQuery));
1834         if (!aResult) {
1835                 Log("SHUTTLE", Form("UpdateShuttleLogbook - Can't execute query <%s>", sqlQuery.Data()));
1836                 return kFALSE;
1837         }
1838         delete aResult;
1839
1840         return kTRUE;
1841 }
1842
1843 //______________________________________________________________________________________________
1844 Int_t AliShuttle::GetCurrentRun() const
1845 {
1846 // Get current run from logbook entry
1847
1848         return fLogbookEntry ? fLogbookEntry->GetRun() : -1;
1849 }
1850
1851 //______________________________________________________________________________________________
1852 UInt_t AliShuttle::GetCurrentStartTime() const
1853 {
1854 // get current start time
1855
1856         return fLogbookEntry ? fLogbookEntry->GetStartTime() : 0;
1857 }
1858
1859 //______________________________________________________________________________________________
1860 UInt_t AliShuttle::GetCurrentEndTime() const
1861 {
1862 // get current end time from logbook entry
1863
1864         return fLogbookEntry ? fLogbookEntry->GetEndTime() : 0;
1865 }
1866
1867 //______________________________________________________________________________________________
1868 void AliShuttle::Log(const char* detector, const char* message)
1869 {
1870 // Fill log string with a message
1871
1872         void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
1873         if (dir == NULL) {
1874                 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE)) {
1875                         AliError(Form("Can't open directory <%s>", GetShuttleLogDir()));
1876                         return;
1877                 }
1878
1879         } else {
1880                 gSystem->FreeDirectory(dir);
1881         }
1882
1883         TString toLog = Form("%s (%d): %s - ", TTimeStamp(time(0)).AsString("s"), getpid(), detector);
1884         if(GetCurrentRun()>=0 ) toLog += Form("run %d - ", GetCurrentRun());
1885         toLog += Form("%s", message);
1886
1887         AliInfo(toLog.Data());
1888
1889         TString fileName;
1890         fileName.Form("%s/%s.log", GetShuttleLogDir(), detector);
1891         gSystem->ExpandPathName(fileName);
1892
1893         ofstream logFile;
1894         logFile.open(fileName, ofstream::out | ofstream::app);
1895
1896         if (!logFile.is_open()) {
1897                 AliError(Form("Could not open file %s", fileName.Data()));
1898                 return;
1899         }
1900
1901         logFile << toLog.Data() << "\n";
1902
1903         logFile.close();
1904 }
1905
1906 //______________________________________________________________________________________________
1907 Bool_t AliShuttle::Collect(Int_t run)
1908 {
1909 //
1910 // Collects conditions data for all UNPROCESSED run written to DAQ LogBook in case of run = -1 (default)
1911 // If a dedicated run is given this run is processed
1912 //
1913 // In operational mode, this is the Shuttle function triggered by the EOR signal.
1914 //
1915
1916         if (run == -1)
1917                 Log("SHUTTLE","Collect - Shuttle called. Collecting conditions data for unprocessed runs");
1918         else
1919                 Log("SHUTTLE", Form("Collect - Shuttle called. Collecting conditions data for run %d", run));
1920
1921         SetLastAction("Starting");
1922
1923         TString whereClause("where shuttle_done=0");
1924         if (run != -1)
1925                 whereClause += Form(" and run=%d", run);
1926
1927         TObjArray shuttleLogbookEntries;
1928         if (!QueryShuttleLogbook(whereClause, shuttleLogbookEntries))
1929         {
1930                 Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
1931                 return kFALSE;
1932         }
1933
1934         if (shuttleLogbookEntries.GetEntries() == 0)
1935         {
1936                 if (run == -1)
1937                         Log("SHUTTLE","Collect - Found no UNPROCESSED runs in Shuttle logbook");
1938                 else
1939                         Log("SHUTTLE", Form("Collect - Run %d is already DONE "
1940                                                 "or it does not exist in Shuttle logbook", run));
1941                 return kTRUE;
1942         }
1943
1944         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
1945                 fFirstUnprocessed[iDet] = kTRUE;
1946
1947         if (run != -1)
1948         {
1949                 // query Shuttle logbook for earlier runs, check if some detectors are unprocessed,
1950                 // flag them into fFirstUnprocessed array
1951                 TString whereClause(Form("where shuttle_done=0 and run < %d", run));
1952                 TObjArray tmpLogbookEntries;
1953                 if (!QueryShuttleLogbook(whereClause, tmpLogbookEntries))
1954                 {
1955                         Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
1956                         return kFALSE;
1957                 }
1958
1959                 TIter iter(&tmpLogbookEntries);
1960                 AliShuttleLogbookEntry* anEntry = 0;
1961                 while ((anEntry = dynamic_cast<AliShuttleLogbookEntry*> (iter.Next())))
1962                 {
1963                         for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
1964                         {
1965                                 if (anEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
1966                                 {
1967                                         AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
1968                                                         anEntry->GetRun(), GetDetName(iDet)));
1969                                         fFirstUnprocessed[iDet] = kFALSE;
1970                                 }
1971                         }
1972
1973                 }
1974
1975         }
1976
1977         if (!RetrieveConditionsData(shuttleLogbookEntries))
1978         {
1979                 Log("SHUTTLE", "Collect - Process of at least one run failed");
1980                 return kFALSE;
1981         }
1982
1983         Log("SHUTTLE", "Collect - Requested run(s) successfully processed");
1984         return kTRUE;
1985 }
1986
1987 //______________________________________________________________________________________________
1988 Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
1989 {
1990 // Retrieve conditions data for all runs that aren't processed yet
1991
1992         Bool_t hasError = kFALSE;
1993
1994         TIter iter(&dateEntries);
1995         AliShuttleLogbookEntry* anEntry;
1996
1997         while ((anEntry = (AliShuttleLogbookEntry*) iter.Next())){
1998                 if (!Process(anEntry)){
1999                         hasError = kTRUE;
2000                 }
2001         }
2002
2003         return hasError == kFALSE;
2004 }
2005
2006 //______________________________________________________________________________________________
2007 ULong_t AliShuttle::GetTimeOfLastAction() const
2008 {
2009         ULong_t tmp;
2010
2011         fMonitoringMutex->Lock();
2012
2013         tmp = fLastActionTime;
2014
2015         fMonitoringMutex->UnLock();
2016
2017         return tmp;
2018 }
2019
2020 //______________________________________________________________________________________________
2021 const TString AliShuttle::GetLastAction() const
2022 {
2023         // returns a string description of the last action
2024
2025         TString tmp;
2026
2027         fMonitoringMutex->Lock();
2028         
2029         tmp = fLastAction;
2030         
2031         fMonitoringMutex->UnLock();
2032
2033         return tmp;
2034 }
2035
2036 //______________________________________________________________________________________________
2037 void AliShuttle::SetLastAction(const char* action)
2038 {
2039         // updates the monitoring variables
2040
2041         fMonitoringMutex->Lock();
2042
2043         fLastAction = action;
2044         fLastActionTime = time(0);
2045         
2046         fMonitoringMutex->UnLock();
2047 }
2048
2049 //______________________________________________________________________________________________
2050 const char* AliShuttle::GetRunParameter(const char* param)
2051 {
2052 // returns run parameter read from DAQ logbook
2053
2054         if(!fLogbookEntry) {
2055                 AliError("No logbook entry!");
2056                 return 0;
2057         }
2058
2059         return fLogbookEntry->GetRunParameter(param);
2060 }
2061
2062 //______________________________________________________________________________________________
2063 Bool_t AliShuttle::SendMail()
2064 {
2065 // sends a mail to the subdetector expert in case of preprocessor error
2066
2067         void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
2068         if (dir == NULL)
2069         {
2070                 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE))
2071                 {
2072                         AliError(Form("Can't open directory <%s>", GetShuttleLogDir()));
2073                         return kFALSE;
2074                 }
2075
2076         } else {
2077                 gSystem->FreeDirectory(dir);
2078         }
2079
2080         TString bodyFileName;
2081         bodyFileName.Form("%s/mail.body", GetShuttleLogDir());
2082         gSystem->ExpandPathName(bodyFileName);
2083
2084         ofstream mailBody;
2085         mailBody.open(bodyFileName, ofstream::out);
2086
2087         if (!mailBody.is_open())
2088         {
2089                 AliError(Form("Could not open mail body file %s", bodyFileName.Data()));
2090                 return kFALSE;
2091         }
2092
2093         TString to="";
2094         TIter iterExperts(fConfig->GetResponsibles(fCurrentDetector));
2095         TObjString *anExpert=0;
2096         while ((anExpert = (TObjString*) iterExperts.Next()))
2097         {
2098                 to += Form("%s,", anExpert->GetName());
2099         }
2100         to.Remove(to.Length()-1);
2101         AliDebug(2, Form("to: %s",to.Data()));
2102
2103         // TODO this will be removed...
2104         if (to.Contains("not_yet_set")) {
2105                 AliInfo("List of detector responsibles not yet set!");
2106                 return kFALSE;
2107         }
2108
2109         TString cc="alberto.colla@cern.ch";
2110
2111         TString subject = Form("%s Shuttle preprocessor error in run %d !",
2112                                 fCurrentDetector.Data(), GetCurrentRun());
2113         AliDebug(2, Form("subject: %s", subject.Data()));
2114
2115         TString body = Form("Dear %s expert(s), \n\n", fCurrentDetector.Data());
2116         body += Form("SHUTTLE just detected that your preprocessor "
2117                         "exited with ERROR state in run %d!!\n\n", GetCurrentRun());
2118         body += Form("Please check %s status on the web page asap!\n\n", fCurrentDetector.Data());
2119         body += Form("The last 10 lines of %s log file are following:\n\n");
2120
2121         AliDebug(2, Form("Body begin: %s", body.Data()));
2122
2123         mailBody << body.Data();
2124         mailBody.close();
2125         mailBody.open(bodyFileName, ofstream::out | ofstream::app);
2126
2127         TString logFileName = Form("%s/%s.log", GetShuttleLogDir(), fCurrentDetector.Data());
2128         TString tailCommand = Form("tail -n 10 %s >> %s", logFileName.Data(), bodyFileName.Data());
2129         if (gSystem->Exec(tailCommand.Data()))
2130         {
2131                 mailBody << Form("%s log file not found ...\n\n", fCurrentDetector.Data());
2132         }
2133
2134         TString endBody = Form("------------------------------------------------------\n\n");
2135         endBody += Form("In case of problems please contact the SHUTTLE core team.\n\n");
2136         endBody += "Please do not answer this message directly, it is automatically generated.\n\n";
2137         endBody += "Sincerely yours,\n\n \t\t\tthe SHUTTLE\n";
2138
2139         AliDebug(2, Form("Body end: %s", endBody.Data()));
2140
2141         mailBody << endBody.Data();
2142
2143         mailBody.close();
2144
2145         // send mail!
2146         TString mailCommand = Form("mail -s \"%s\" -c %s %s < %s",
2147                                                 subject.Data(),
2148                                                 cc.Data(),
2149                                                 to.Data(),
2150                                                 bodyFileName.Data());
2151         AliDebug(2, Form("mail command: %s", mailCommand.Data()));
2152
2153         Bool_t result = gSystem->Exec(mailCommand.Data());
2154
2155         return result == 0;
2156 }
2157
2158 //______________________________________________________________________________________________
2159 void AliShuttle::SetShuttleTempDir(const char* tmpDir)
2160 {
2161 // sets Shuttle temp directory
2162
2163         fgkShuttleTempDir = gSystem->ExpandPathName(tmpDir);
2164 }
2165
2166 //______________________________________________________________________________________________
2167 void AliShuttle::SetShuttleLogDir(const char* logDir)
2168 {
2169 // sets Shuttle log directory
2170
2171         fgkShuttleLogDir = gSystem->ExpandPathName(logDir);
2172 }