Implement symbolic links
[u/mrichter/AliRoot.git] / SHUTTLE / AliShuttle.cxx
CommitLineData
73abe331 1/**************************************************************************
2 * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16/*
17$Log$
85a80aa9 18Revision 1.13 2006/08/15 10:50:00 jgrosseo
19effc++ corrections (alberto)
20
4f0ab988 21Revision 1.12 2006/08/08 14:19:29 jgrosseo
22Update to shuttle classes (Alberto)
23
24- Possibility to set the full object's path in the Preprocessor's and
25Shuttle's Store functions
26- Possibility to extend the object's run validity in the same classes
27("startValidity" and "validityInfinite" parameters)
28- Implementation of the StoreReferenceData function to store reference
29data in a dedicated CDB storage.
30
84090f85 31Revision 1.11 2006/07/21 07:37:20 jgrosseo
32last run is stored after each run
33
7bfb2090 34Revision 1.10 2006/07/20 09:54:40 jgrosseo
35introducing status management: The processing per subdetector is divided into several steps,
36after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
37can keep track of the number of failures and skips further processing after a certain threshold is
38exceeded. These thresholds can be configured in LDAP.
39
5164a766 40Revision 1.9 2006/07/19 10:09:55 jgrosseo
41new configuration, accesst to DAQ FES (Alberto)
42
57f50b3c 43Revision 1.8 2006/07/11 12:44:36 jgrosseo
44adding parameters for extended validity range of data produced by preprocessor
45
17111222 46Revision 1.7 2006/07/10 14:37:09 jgrosseo
47small fix + todo comment
48
e090413b 49Revision 1.6 2006/07/10 13:01:41 jgrosseo
50enhanced storing of last sucessfully processed run (alberto)
51
a7160fe9 52Revision 1.5 2006/07/04 14:59:57 jgrosseo
53revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
54
45a493ce 55Revision 1.4 2006/06/12 09:11:16 jgrosseo
56coding conventions (Alberto)
57
58bc3020 58Revision 1.3 2006/06/06 14:26:40 jgrosseo
59o) removed files that were moved to STEER
60o) shuttle updated to follow the new interface (Alberto)
61
b948db8d 62Revision 1.2 2006/03/07 07:52:34 hristov
63New version (B.Yordanov)
64
d477ad88 65Revision 1.6 2005/11/19 17:19:14 byordano
66RetrieveDATEEntries and RetrieveConditionsData added
67
68Revision 1.5 2005/11/19 11:09:27 byordano
69AliShuttle declaration added
70
71Revision 1.4 2005/11/17 17:47:34 byordano
72TList changed to TObjArray
73
74Revision 1.3 2005/11/17 14:43:23 byordano
75import to local CVS
76
77Revision 1.1.1.1 2005/10/28 07:33:58 hristov
78Initial import as subdirectory in AliRoot
79
73abe331 80Revision 1.2 2005/09/13 08:41:15 byordano
81default startTime endTime added
82
83Revision 1.4 2005/08/30 09:13:02 byordano
84some docs added
85
86Revision 1.3 2005/08/29 21:15:47 byordano
87some docs added
88
89*/
90
91//
92// This class is the main manager for AliShuttle.
93// It organizes the data retrieval from DCS and call the
b948db8d 94// interface methods of AliPreprocessor.
73abe331 95// For every detector in AliShuttleConfgi (see AliShuttleConfig),
96// data for its set of aliases is retrieved. If there is registered
b948db8d 97// AliPreprocessor for this detector then it will be used
98// accroding to the schema (see AliPreprocessor).
99// If there isn't registered AliPreprocessor than the retrieved
73abe331 100// data is stored automatically to the undelying AliCDBStorage.
101// For detSpec is used the alias name.
102//
103
104#include "AliShuttle.h"
105
106#include "AliCDBManager.h"
107#include "AliCDBStorage.h"
108#include "AliCDBId.h"
84090f85 109#include "AliCDBRunRange.h"
110#include "AliCDBPath.h"
5164a766 111#include "AliCDBEntry.h"
73abe331 112#include "AliShuttleConfig.h"
113#include "AliDCSClient.h"
114#include "AliLog.h"
b948db8d 115#include "AliPreprocessor.h"
5164a766 116#include "AliShuttleStatus.h"
73abe331 117
57f50b3c 118#include <TSystem.h>
58bc3020 119#include <TObject.h>
b948db8d 120#include <TString.h>
57f50b3c 121#include <TTimeStamp.h>
73abe331 122#include <TObjString.h>
57f50b3c 123#include <TSQLServer.h>
124#include <TSQLResult.h>
125#include <TSQLRow.h>
73abe331 126
5164a766 127#include <fstream>
128
73abe331 129ClassImp(AliShuttle)
130
4f0ab988 131TString AliShuttle::fgkMainCDB("alien://DBFolder=ShuttleCDB");
84090f85 132TString AliShuttle::fgkLocalCDB("local://LocalShuttleCDB");
133TString AliShuttle::fgkMainRefStorage("alien://DBFolder=ShuttleReference");
134TString AliShuttle::fgkLocalRefStorage("local://LocalReferenceStorage");
135
4f0ab988 136Bool_t AliShuttle::fgkProcessDCS(kTRUE);
137
138
84090f85 139const char* AliShuttle::fgkShuttleTempDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/temp");
140const char* AliShuttle::fgkShuttleLogDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/log");
57f50b3c 141
142const char* AliShuttle::fgkDetectorName[AliShuttle::fgkNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
143 "PHOS", "CPV", "RICH", "EMCAL", "MUON_TRK", "MUON_TRG", "FMD", "ZDC", "PMD", "START", "VZERO"};
144
145const char* AliShuttle::fgkDetectorCode[AliShuttle::fgkNDetectors] = {"SPD", "SDD", "SSD", "TPC", "TRD", "TOF",
146 "PHS", "CPV", "HMP", "EMC", "MCH", "MTR", "FMD", "ZDC", "PMD", "T00", "V00"};
b948db8d 147
148//______________________________________________________________________________________________
149AliShuttle::AliShuttle(const AliShuttleConfig* config,
150 UInt_t timeout, Int_t retries):
4f0ab988 151fConfig(config),
152fTimeout(timeout), fRetries(retries),
153fPreprocessorMap(),
154fCurrentRun(-1),
155fCurrentStartTime(0), fCurrentEndTime(0),
156fCurrentDetector(""),
85a80aa9 157fStatusEntry(0),
158fGridError(kFALSE)
73abe331 159{
160 //
161 // config: AliShuttleConfig used
73abe331 162 // timeout: timeout used for AliDCSClient connection
163 // retries: the number of retries in case of connection error.
164 //
165
57f50b3c 166 if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
167 for(int iSys=0;iSys<3;iSys++) {
168 fServer[iSys]=0;
169 fFESlist[iSys].SetOwner(kTRUE);
170 }
73abe331 171}
172
58bc3020 173//______________________________________________________________________
174AliShuttle::AliShuttle(const AliShuttle& /*other*/):
4f0ab988 175AliShuttleInterface(),
176fConfig(0),
177fTimeout(0), fRetries(0),
178fPreprocessorMap(),
179fCurrentRun(-1),
180fCurrentStartTime(0), fCurrentEndTime(0),
181fCurrentDetector(""),
85a80aa9 182fStatusEntry(0),
183fGridError(kFALSE)
58bc3020 184{
185// copy constructor (not implemented)
186
187}
188
189//______________________________________________________________________
190AliShuttle &AliShuttle::operator=(const AliShuttle& /*other*/)
191{
192// assignment operator (not implemented)
193
194return *this;
195}
196
b948db8d 197//______________________________________________________________________________________________
57f50b3c 198AliShuttle::~AliShuttle()
58bc3020 199{
200// destructor
201
b948db8d 202 fPreprocessorMap.DeleteAll();
57f50b3c 203 for(int iSys=0;iSys<3;iSys++)
204 if(fServer[iSys]) {
205 fServer[iSys]->Close();
206 delete fServer[iSys];
207 }
73abe331 208}
209
b948db8d 210//______________________________________________________________________________________________
57f50b3c 211void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
58bc3020 212{
73abe331 213 //
b948db8d 214 // Registers new AliPreprocessor.
73abe331 215 // It uses GetName() for indentificator of the pre processor.
216 // The pre processor is registered it there isn't any other
217 // with the same identificator (GetName()).
218 //
219
b948db8d 220 if (fPreprocessorMap.GetValue(preprocessor->GetName())) {
221 AliWarning(Form("AliPreprocessor %s is already registered!",
222 preprocessor->GetName()));
73abe331 223 return;
224 }
225
b948db8d 226 fPreprocessorMap.Add(new TObjString(preprocessor->GetName()), preprocessor);
73abe331 227}
228
b948db8d 229//______________________________________________________________________________________________
84090f85 230UInt_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
231 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
73abe331 232{
84090f85 233 // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
234 // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
235 // using this function. Use StoreReferenceData instead!
85a80aa9 236 // It calls WriteToCDB function which perform actual storage
b948db8d 237
85a80aa9 238 return WriteToCDB(fgkMainCDB, fgkLocalCDB, path, object,
239 metaData, validityStart, validityInfinite);
84090f85 240
241}
242
243//______________________________________________________________________________________________
244UInt_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object,
245 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
246{
247 // Stores a CDB object in the storage for reference data. This objects will not be available during
248 // offline reconstrunction. Use this function for reference data only!
85a80aa9 249 // It calls WriteToCDB function which perform actual storage
250
251 return WriteToCDB(fgkMainRefStorage, fgkLocalRefStorage, path, object,
252 metaData, validityStart, validityInfinite);
84090f85 253
85a80aa9 254}
255
256//______________________________________________________________________________________________
257UInt_t AliShuttle::WriteToCDB(const char* mainUri, const char* localUri,
258 const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
259 Int_t validityStart, Bool_t validityInfinite)
260{
261 // write object into the CDB. Parameters are passed by Store and StoreReferenceData functions.
262 // The parameters are:
263 // 1) Uri of the main storage (Grid)
264 // 2) Uri of the backup storage (Local)
265 // 3) the object's path.
266 // 4) the object to be stored
267 // 5) the metaData to be associated with the object
268 // 6) the validity start run number w.r.t. the current run,
84090f85 269 // if the data is valid only for this run leave the default 0
85a80aa9 270 // 7) specifies if the calibration data is valid for infinity (this means until updated),
84090f85 271 // typical for calibration runs, the default is kFALSE
272 //
84090f85 273 // returns 0 if fail
85a80aa9 274 // 1 if stored in main (Grid) storage
275 // 2 if stored in backup (Local) storage
84090f85 276
85a80aa9 277 const char* cdbType = (mainUri == fgkMainCDB) ? "CDB" : "Reference";
278
279 Int_t firstRun = GetCurrentRun() - validityStart;
84090f85 280 if(firstRun < 0) {
281 AliError("First valid run happens to be less than 0! Setting it to 0...");
282 firstRun=0;
283 }
284
285 Int_t lastRun = -1;
286 if(validityInfinite) {
287 lastRun = AliCDBRunRange::Infinity();
288 } else {
289 lastRun = GetCurrentRun();
290 }
291
85a80aa9 292 AliCDBId id(path, firstRun, lastRun);
84090f85 293
294 UInt_t result = 0;
295
85a80aa9 296 if (!(AliCDBManager::Instance()->GetStorage(mainUri))) {
297 Log(fCurrentDetector, Form("Cannot activate main %s storage!", cdbType));
84090f85 298 } else {
85a80aa9 299 result = (UInt_t) AliCDBManager::Instance()->GetStorage(mainUri)
84090f85 300 ->Put(object, id, metaData);
301 }
302
303 if(!result) {
304
305 Log(fCurrentDetector,
85a80aa9 306 Form("Problem with main %s storage. Object will go to local storage!", cdbType));
84090f85 307
85a80aa9 308 result = AliCDBManager::Instance()->GetStorage(localUri)
84090f85 309 ->Put(object, id, metaData);
310
311 if(result) {
312 result = 2;
85a80aa9 313 fGridError = kTRUE;
84090f85 314 }else{
85a80aa9 315 Log(fCurrentDetector, "Can't store data!");
b948db8d 316 }
317 }
318 return result;
319
73abe331 320}
321
b948db8d 322//______________________________________________________________________________________________
5164a766 323AliShuttleStatus* AliShuttle::ReadShuttleStatus()
324{
325 // Reads the AliShuttleStatus from the CDB
326
327 if (fStatusEntry)
328 {
329 delete fStatusEntry;
330 fStatusEntry = 0;
331 }
332
84090f85 333 fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalCDB())
5164a766 334 ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), fCurrentRun);
335
5164a766 336 if (!fStatusEntry)
337 return 0;
338
339 TObject* anObject = fStatusEntry->GetObject();
340 if (anObject == NULL || anObject->IsA() != AliShuttleStatus::Class())
341 {
342 AliError("Invalid object stored to CDB!");
343 return 0;
344 }
345
346 AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (anObject);
347 return status;
348}
349
350//______________________________________________________________________________________________
7bfb2090 351Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
5164a766 352{
353 // writes the status for one subdetector
354
355 if (fStatusEntry)
356 {
357 delete fStatusEntry;
358 fStatusEntry = 0;
359 }
360
361 AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), fCurrentRun, fCurrentRun);
362
363 fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
364
84090f85 365 UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
7bfb2090 366
367 if (!result)
368 {
369 AliError(Form("WriteShuttleStatus for %s, run %d failed", fCurrentDetector.Data(), fCurrentRun));
370 return kFALSE;
371 }
372
373 return kTRUE;
5164a766 374}
375
376//______________________________________________________________________________________________
377void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
378{
379 // changes the AliShuttleStatus for the given detector and run to the given status
380
381 if (!fStatusEntry)
382 {
383 AliError("UNEXPECTED: fStatusEntry empty");
384 return;
385 }
386
387 TObject* anObject = fStatusEntry->GetObject();
388 AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (anObject);
389
390 if (!status)
391 {
392 AliError("UNEXPECTED: status could not be read from current CDB entry");
393 return;
394 }
395
7bfb2090 396 Log("SHUTTLE", Form("%s: Changing state from %s to %s", fCurrentDetector.Data(),
397 status->GetStatusName(), status->GetStatusName(newStatus)));
5164a766 398
399 status->SetStatus(newStatus);
400 if (increaseCount)
401 status->IncreaseCount();
402
84090f85 403 AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
5164a766 404}
405
406//______________________________________________________________________________________________
407Bool_t AliShuttle::ContinueProcessing()
408{
409 // this function reads the AliShuttleStatus information from CDB and
410 // checks if the processing should be continued
411 // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
412
413 AliShuttleStatus* status = ReadShuttleStatus();
414 if (!status)
415 {
416 // first time
417
418 Log("SHUTTLE", Form("%s: Processing first time.", fCurrentDetector.Data()));
419 status = new AliShuttleStatus(AliShuttleStatus::kStarted);
7bfb2090 420 return WriteShuttleStatus(status);
5164a766 421 }
422
423 if (status->GetStatus() == AliShuttleStatus::kDone)
424 {
425 Log("SHUTTLE", Form("%s already done for run %d", fCurrentDetector.Data(), fCurrentRun));
426 return kFALSE;
427 }
428
429 if (status->GetStatus() == AliShuttleStatus::kFailed)
430 {
431 Log("SHUTTLE", Form("%s already in failed state for run %d", fCurrentDetector.Data(), fCurrentRun));
432 return kFALSE;
433 }
434
85a80aa9 435 // TODO what to do in case of storage error? currently it does not do anything
436 if (status->GetStatus() == AliShuttleStatus::kStoreFailed)
437 {
438 Log("SHUTTLE", Form("%s: Grid storage failed at least once for run %d", fCurrentDetector.Data(), fCurrentRun));
439 return kFALSE;
440 }
441
5164a766 442 // if we get here, there is a restart
443
444 // abort conditions
7bfb2090 445 if (status->GetStatus() == AliShuttleStatus::kPPStarted && status->GetCount() >= fConfig->GetMaxPPRetries() ||
446 status->GetCount() >= fConfig->GetMaxRetries())
5164a766 447 {
7bfb2090 448 Log("SHUTTLE", Form("%s, run %d failed too often, %d times, status %s. Skipping processing.",
449 fCurrentDetector.Data(), fCurrentRun, status->GetCount(), status->GetStatusName()));
5164a766 450
451 return kFALSE;
452 }
453
7bfb2090 454 Log("SHUTTLE", Form("Restart of %s, run %d. Got stuck before in %s, count %d",
455 fCurrentDetector.Data(), fCurrentRun, status->GetStatusName(), status->GetCount()));
5164a766 456
457 UpdateShuttleStatus(AliShuttleStatus::kStarted, kTRUE);
458
459 return kTRUE;
460}
461
462//______________________________________________________________________________________________
463Bool_t AliShuttle::Process(Int_t run, UInt_t startTime, UInt_t endTime)
58bc3020 464{
73abe331 465 //
b948db8d 466 // Makes data retrieval for all detectors in the configuration.
73abe331 467 // run: is the run number used
468 // startTime: is the run start time
469 // endTime: is the run end time
d477ad88 470 // Returns kFALSE in case of error occured and kTRUE otherwise
73abe331 471 //
472
85a80aa9 473 AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^* \n", run));
57f50b3c 474
475 // Initialization
d477ad88 476 Bool_t hasError = kFALSE;
57f50b3c 477 for(Int_t iSys=0;iSys<3;iSys++) fFESCalled[iSys]=kFALSE;
5164a766 478
7bfb2090 479 fCurrentRun = run;
57f50b3c 480 fCurrentStartTime = startTime;
481 fCurrentEndTime = endTime;
d477ad88 482
57f50b3c 483 // Loop on detectors in the configuration
b948db8d 484 TIter iter(fConfig->GetDetectors());
73abe331 485 TObjString* aDetector;
b948db8d 486
73abe331 487 while ((aDetector = (TObjString*) iter.Next())) {
7bfb2090 488 fCurrentDetector = aDetector->String();
5164a766 489
57f50b3c 490 Bool_t detectorError=kFALSE;
5164a766 491 if (!fConfig->HostProcessDetector(fCurrentDetector)) continue;
492
7bfb2090 493 if (ContinueProcessing() == kFALSE) continue;
5164a766 494
85a80aa9 495 AliInfo(Form("\n\n \t\t\t****** %s: START ******", aDetector->GetName()));
496
5164a766 497 if(!Process()) {
d477ad88 498 hasError = kTRUE;
57f50b3c 499 detectorError=kTRUE;
500 continue;
d477ad88 501 }
85a80aa9 502 AliInfo(Form("\n \t\t\t****** %s: FINISH ****** \n\n", aDetector->GetName()));
57f50b3c 503
504 // Process successful: Update time_processed field in FES logbooks!
505 if(fFESCalled[kDAQ]) {
5164a766 506 hasError = (UpdateDAQTable() == kFALSE);
57f50b3c 507 fFESlist[kDAQ].Clear();
508 }
509 //if(fFESCalled[kDCS]) {
510 // hasError = UpdateDCSTable(aDetector->GetName());
511 // fFESlist[kDCS].Clear();
512 //}
513 //if(fFESCalled[kHLT]) {
514 // hasError = UpdateHLTTable(aDetector->GetName());
515 // fFESlist[kHLT].Clear();
516 //}
d477ad88 517
85a80aa9 518 // UpdateShuttleStatus(AliShuttleStatus::kDone);
7bfb2090 519 }
5164a766 520
57f50b3c 521 fCurrentRun = -1;
522 fCurrentStartTime = 0;
523 fCurrentEndTime = 0;
b948db8d 524
85a80aa9 525 AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^* \n", run));
526
a7160fe9 527 return hasError == kFALSE;
73abe331 528}
529
b948db8d 530//______________________________________________________________________________________________
5164a766 531Bool_t AliShuttle::Process()
73abe331 532{
533 //
b948db8d 534 // Makes data retrieval just for one specific detector.
73abe331 535 // Threre should be a configuration for this detector.
536 // run: is the run number used
537 // startTime: is the run start time
538 // endTime: is the run end time
539 // detector: detector for which the retrieval will be made
d477ad88 540 // Returns kFALSE in case of error occured and kTRUE otherwise
73abe331 541 //
542
5164a766 543 AliInfo(Form("Retrieving values for %s, run %d", fCurrentDetector.Data(), fCurrentRun));
73abe331 544
5164a766 545 if (!fConfig->HasDetector(fCurrentDetector)) {
546 Log(fCurrentDetector, "There isn't any configuration for %s !");
7bfb2090 547 UpdateShuttleStatus(AliShuttleStatus::kFailed);
d477ad88 548 return kFALSE;
73abe331 549 }
550
7bfb2090 551 UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
73abe331 552
7bfb2090 553 TString host(fConfig->GetDCSHost(fCurrentDetector));
5164a766 554 Int_t port = fConfig->GetDCSPort(fCurrentDetector);
555
556 TIter iter(fConfig->GetDCSAliases(fCurrentDetector));
73abe331 557 TObjString* anAlias;
b948db8d 558 TMap aliasMap;
73abe331 559
85a80aa9 560 Bool_t aDCSError = kFALSE;
561 fGridError = kFALSE;
562 UInt_t aPPResult;
d477ad88 563
b948db8d 564 while ((anAlias = (TObjString*) iter.Next())) {
d477ad88 565 TObjArray valueSet;
4f0ab988 566 // TODO Test only... I've added a flag that allows to
567 // exclude DCS archive DB query
568 if(fgkProcessDCS){
569 AliInfo("Querying DCS archive DB data...");
85a80aa9 570 aDCSError = (GetValueSet(host, port, anAlias->String(), valueSet) == 0);
4f0ab988 571 } else {
572 AliInfo(Form("Skipping DCS processing. Port = %d",port));
85a80aa9 573 aDCSError = kFALSE;
4f0ab988 574 }
85a80aa9 575 if(!aDCSError) {
b948db8d 576 aliasMap.Add(anAlias->Clone(), valueSet.Clone());
577 }else{
57f50b3c 578 TString message = Form("Error while retrieving alias %s !",
b948db8d 579 anAlias->GetName());
5164a766 580 Log(fCurrentDetector, message.Data());
85a80aa9 581 aDCSError = kTRUE;
84090f85 582 break;
73abe331 583 }
584 }
b948db8d 585
85a80aa9 586 if (aDCSError)
7bfb2090 587 {
588 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
589 return kFALSE;
590 }
5164a766 591
85a80aa9 592 UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
a7160fe9 593
85a80aa9 594 AliPreprocessor* aPreprocessor =
5164a766 595 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
a7160fe9 596 if(aPreprocessor)
597 {
5164a766 598 aPreprocessor->Initialize(fCurrentRun, fCurrentStartTime, fCurrentEndTime);
85a80aa9 599 aPPResult = aPreprocessor->Process(&aliasMap);
a7160fe9 600 }else{
e090413b 601 // TODO default behaviour?
5164a766 602 AliInfo(Form("No Preprocessor for %s: storing TMap of DP arrays into CDB!", fCurrentDetector.Data()));
a7160fe9 603 AliCDBMetaData metaData;
5164a766 604 AliDCSValue dcsValue(fCurrentStartTime, fCurrentEndTime);
a7160fe9 605 metaData.SetResponsible(Form("Duck, Donald"));
57f50b3c 606 metaData.SetProperty("StartEndTime", &dcsValue);
a7160fe9 607 metaData.SetComment("Automatically stored by Shuttle!");
84090f85 608 AliCDBPath path(fCurrentDetector,"DCS","Data");
85a80aa9 609 aPPResult = Store(path, &aliasMap, &metaData);
b948db8d 610 }
611
85a80aa9 612 if (aPPResult == 0) { // Preprocessor error
613 UpdateShuttleStatus(AliShuttleStatus::kPPError);
614 } else if (fGridError == kFALSE) { // process and Grid storage ok!
615 UpdateShuttleStatus(AliShuttleStatus::kDone);
616 } else { // Grid storage error (process ok, but object put in local storage)
617 UpdateShuttleStatus(AliShuttleStatus::kStoreFailed);
618 }
b948db8d 619
85a80aa9 620 aliasMap.Delete();
b948db8d 621
85a80aa9 622 return (aPPResult > 0);
73abe331 623}
624
b948db8d 625//______________________________________________________________________________________________
73abe331 626Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* alias,
d477ad88 627 TObjArray& valueSet)
73abe331 628{
58bc3020 629// Retrieve all "alias" data points from the DCS server
630// host, port: TSocket connection parameters
631// alias: name of the alias
632// valueSet: array of retrieved AliDCSValue's
633
73abe331 634 AliDCSClient client(host, port, fTimeout, fRetries);
635 if (!client.IsConnected()) {
b948db8d 636 return kFALSE;
73abe331 637 }
638
57f50b3c 639 Int_t result = client.GetAliasValues(alias,
73abe331 640 GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
641
642 if (result < 0) {
643 AliError(Form("Can't get '%s'! Reason: %s",
644 alias, AliDCSClient::GetErrorString(result)));
645
646 if (result == AliDCSClient::fgkServerError) {
647 AliError(Form("Server error: %s",
648 client.GetServerError().Data()));
649 }
650
651 return kFALSE;
652 }
653
654 return kTRUE;
655}
b948db8d 656
657//______________________________________________________________________________________________
57f50b3c 658const char* AliShuttle::GetFile(Int_t system, const char* detector,
659 const char* id, const char* source)
b948db8d 660{
57f50b3c 661// Get calibration file from file exchange servers
662// calls specific getter according to system index (kDAQ, kDCS, kHLT)
663
664 switch(system){
665 case kDAQ:
666 return GetDAQFileName(detector, id, source);
667 break;
668 case kDCS:
669 return GetDCSFileName(detector, id, source);
670 break;
671 case kHLT:
672 return GetHLTFileName(detector, id, source);
673 break;
674 default:
675 AliError(Form("No valid system index: %d",system));
676 }
b948db8d 677
b948db8d 678 return 0;
679}
680
b948db8d 681//______________________________________________________________________________________________
57f50b3c 682TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
b948db8d 683{
57f50b3c 684// Get sources producing the condition file Id from file exchange servers
685// calls specific getter according to system index (kDAQ, kDCS, kHLT)
686
687 switch(system){
688 case kDAQ:
689 return GetDAQFileSources(detector, id);
690 break;
691 case kDCS:
692 return GetDCSFileSources(detector, id);
693 break;
694 case kHLT:
695 return GetHLTFileSources(detector, id);
696 break;
697 default:
698 AliError(Form("No valid system index: %d",system));
699 }
700
701 return NULL;
702}
703
704//______________________________________________________________________________________________
705Bool_t AliShuttle::Connect(Int_t system){
706// Connect to MySQL Server of the system's FES logbook
707
708 // check connection: if already connected return
709 if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
710
711 TString aFESlbHost= Form("mysql://%s", fConfig->GetFESlbHost(system));
712
713 fServer[system] = TSQLServer::Connect(aFESlbHost,
714 fConfig->GetFESlbUser(system),
715 fConfig->GetFESlbPass(system));
716 if (!fServer[system] || !fServer[system]->IsConnected()) {
717 AliError(Form("Can't establish connection to FES logbook for %s !",fkSystemNames[system]));
718 return kFALSE;
719 }
720
721 // Get tables
722 // TODO in the configuration should the table name be there too?
723 switch(system){
724 case kDAQ:
725 fServer[kDAQ]->GetTables("REFSYSLOG");
726 break;
727 case kDCS:
728 //fServer[kDCS]->GetTables("REFSYSLOG");
729 break;
730 case kHLT:
731 //fServer[kHLT]->GetTables("REFSYSLOG");
732 break;
733 default:
734 break;
735 }
736
737 return kTRUE;
738}
739
740//______________________________________________________________________________________________
741const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source){
742// Retrieves a file from the DAQ FES.
743// First queris the DAQ logbook_fs for the DAQ file name, using the run, detector, id and source info
744// then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
745// run: current run being processed (fCurrentRun)
746// detector: comes from the Preprocessor name (must be converted into detector code with GetDetCode)
747// id: provided as a parameter by the Preprocessor
748// source: provided by the Preprocessor through GetFileSources function
749
750 // check connection, in case connect
751 if(!Connect(kDAQ)){
752 Log(detector, "GetDAQFileName: Couldn't connect to DAQ Logbook !");
753 return 0;
754 }
755
756 // Query preparation
757 TString sqlQueryStart = "select filePath from logbook_fs where";
758 TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
759 fCurrentRun, GetDetCode(detector), id, source);
760 TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
761
84090f85 762 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
57f50b3c 763
764 // Query execution
765 TSQLResult* aResult;
766 aResult = fServer[kDAQ]->Query(sqlQuery);
767 if (!aResult) {
768 Log(detector, Form("Can't execute query <%s>!", sqlQuery.Data()));
769 return 0;
770 }
771
772 if (aResult->GetRowCount() == 0) {
773 Log(detector,
774 Form("GetDAQFileName: No result from SQL query <%s>!", sqlQuery.Data()));
775 delete aResult;
776 return 0;
777 }
778
779 if (aResult->GetRowCount() >1) {
780 Log(detector,
781 Form("GetDAQFileName: More than one row resulting from SQL query <%s>!", sqlQuery.Data()));
782 delete aResult;
783 return 0;
784 }
785
786 TSQLRow* aRow = aResult->Next();
787
788 if(!aRow){
789 Log(detector, Form("GetDAQFileName: Empty set result from query <%s>!", sqlQuery.Data()));
790 delete aResult;
791 return 0;
792 }
793
794 TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
795
796 delete aResult;
797
84090f85 798 AliDebug(2, Form("filePath = %s",filePath.Data()));
57f50b3c 799
800 // retrieved file is renamed to make it unique
801 TString localFileName = Form("%s_%d_%s_%s.shuttle",
802 detector, fCurrentRun, id, source);
803
804 // file retrieval from DAQ FES
805 Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
806 if(!result) {
807 Log(detector, Form("copying file %s from DAQ FES failed!", filePath.Data()));
808 return 0;
809 } else {
810 AliInfo(Form("File %s copied from DAQ FES into %s/%s !",
811 filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
812 }
813
814
815 fFESCalled[kDAQ]=kTRUE;
816 TObjString *fileParams = new TObjString(Form("%s_!?!_%s", id, source));
817 fFESlist[kDAQ].Add(fileParams);
818
819 return localFileName.Data();
820
821}
822
823//______________________________________________________________________________________________
824Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName){
825
826 // check temp directory: trying to cd to temp; if it does not exist, create it
84090f85 827 AliDebug(2, Form("Copy file %s from DAQ FES into folder %s and rename it as %s",
57f50b3c 828 daqFileName,fgkShuttleTempDir, localFileName));
829
830 void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
831 if (dir == NULL) {
832 if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
833 AliError(Form("Can't open directory <%s>!", fgkShuttleTempDir));
834 return kFALSE;
835 }
836
837 } else {
838 gSystem->FreeDirectory(dir);
839 }
840
841 TString baseDAQFESFolder = "DAQ";
842 TString command = Form("scp %s@%s:%s/%s %s/%s",
843 fConfig->GetFESUser(kDAQ),
844 fConfig->GetFESHost(kDAQ),
845 baseDAQFESFolder.Data(),
846 daqFileName,
847 fgkShuttleTempDir,
848 localFileName);
849
84090f85 850 AliDebug(2, Form("%s",command.Data()));
57f50b3c 851
852 UInt_t nRetries = 0;
853 UInt_t maxRetries = 3;
854
855 // copy!! if successful TSystem::Exec returns 0
856 while(nRetries++ < maxRetries) {
84090f85 857 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
57f50b3c 858 if(gSystem->Exec(command.Data()) == 0) return kTRUE;
859 }
860
861 return kFALSE;
862
863}
864
865//______________________________________________________________________________________________
866TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id){
867// Retrieves a file from the DCS FES.
868
869 // check connection, in case connect
870 if(!Connect(kDAQ)){
871 Log(detector, "GetDAQFileName: Couldn't connect to DAQ Logbook !");
872 return 0;
873 }
874
875 // Query preparation
876 TString sqlQueryStart = "select DAQsource from logbook_fs where";
877 TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
878 fCurrentRun, GetDetCode(detector), id);
879 TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
880
84090f85 881 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
57f50b3c 882
883 // Query execution
884 TSQLResult* aResult;
885 aResult = fServer[kDAQ]->Query(sqlQuery);
886 if (!aResult) {
887 Log(detector, Form("GetDAQFileSources: Can't execute query <%s>!", sqlQuery.Data()));
888 return 0;
889 }
890
891 if (aResult->GetRowCount() == 0) {
892 Log(detector,
893 Form("GetDAQFileSources: No result from SQL query <%s>!", sqlQuery.Data()));
894 delete aResult;
895 return 0;
896 }
897
898 TSQLRow* aRow;
899 TList *list = new TList();
900 list->SetOwner(1);
901
902 while((aRow = aResult->Next())){
903
904 TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
84090f85 905 AliDebug(2, Form("daqSource = %s", daqSource.Data()));
57f50b3c 906 list->Add(new TObjString(daqSource));
907 }
908 delete aResult;
909
910 return list;
911
912}
913
914//______________________________________________________________________________________________
5164a766 915Bool_t AliShuttle::UpdateDAQTable(){
57f50b3c 916// Update DAQ table filling time_processed field in all rows corresponding to current run and detector
917
918 // check connection, in case connect
919 if(!Connect(kDAQ)){
5164a766 920 Log(fCurrentDetector, "UpdateDAQTable: Couldn't connect to DAQ Logbook !");
57f50b3c 921 return kFALSE;
922 }
923
924 TTimeStamp now; // now
925
926 // Loop on FES list entries
927 TIter iter(&fFESlist[kDAQ]);
928 TObjString *aFESentry=0;
929 while((aFESentry = dynamic_cast<TObjString*> (iter.Next()))){
930 TString aFESentrystr = aFESentry->String();
931 TObjArray *aFESarray = aFESentrystr.Tokenize("_!?!_");
932 if(!aFESarray || aFESarray->GetEntries() != 2 ) {
5164a766 933 Log(fCurrentDetector,Form("UpdateDAQTable: error updating FES entry! string = %s",
57f50b3c 934 aFESentrystr.Data()));
935 if(aFESarray) delete aFESarray;
936 return kFALSE;
937 }
938 const char* fileId = ((TObjString*) aFESarray->At(0))->GetName();
939 const char* daqSource = ((TObjString*) aFESarray->At(1))->GetName();
940 TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
5164a766 941 fCurrentRun,GetDetCode(fCurrentDetector), fileId, daqSource);
57f50b3c 942
943 delete aFESarray;
944
945 TString sqlQuery = Form("update logbook_fs set time_processed=%d %s", now.GetSec(), whereClause.Data());
946
84090f85 947 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
57f50b3c 948
949 // Query execution
950 TSQLResult* aResult;
951 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
952 if (!aResult) {
5164a766 953 Log(fCurrentDetector, Form("UpdateDAQTable: Can't execute query <%s>!", sqlQuery.Data()));
57f50b3c 954 return kFALSE;
955 }
956 delete aResult;
957
958 // check result - TODO Is it necessary?
959 sqlQuery = Form("select time_processed from logbook_fs %s", whereClause.Data());
84090f85 960 AliDebug(2, Form(" CHECK - SQL query: \n%s",sqlQuery.Data()));
57f50b3c 961
962 aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
963 if (!aResult) {
964 AliWarning("Can't check result!");
965 continue;
966 }
967
968 if (aResult->GetRowCount() == 0) {
5164a766 969 Log(fCurrentDetector,
57f50b3c 970 Form("GetDAQFileName: No result from SQL query <%s>!", sqlQuery.Data()));
971 delete aResult;
972 //return 0;
973 }
974
975 if (aResult->GetRowCount() >1) {
5164a766 976 Log(fCurrentDetector,
57f50b3c 977 Form("GetDAQFileName: More than one row resulting from SQL query <%s>!", sqlQuery.Data()));
978 delete aResult;
979 //return 0;
980 }
981
982 TSQLRow *row = dynamic_cast<TSQLRow*> (aResult->Next());
983 TString processedTimeString(row->GetField(0), row->GetFieldLength(0));
984 Int_t processedTime = processedTimeString.Atoi();
985 if(processedTime != now.GetSec()){
5164a766 986 Log(fCurrentDetector, Form("UpdateDAQTable: Update table error: processed_time=%d, now=%d !",
57f50b3c 987 processedTime, now.GetSec()));
988 delete aResult;
989 return kFALSE;
990 }
991
992 delete aResult;
993
994 }
995
996 return kTRUE;
997}
998
999//______________________________________________________________________________________________
1000const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1001// Retrieves a file from the DCS FES.
1002
1003return "You're in DCS";
1004
1005}
1006
1007//______________________________________________________________________________________________
1008TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
1009// Retrieves a file from the DCS FES.
1010
1011return NULL;
1012
1013}
1014
1015//______________________________________________________________________________________________
1016const char* AliShuttle::GetHLTFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
1017// Retrieves a file from the HLT FES.
1018
1019return "You're in HLT";
1020
1021}
1022
1023//______________________________________________________________________________________________
1024TList* AliShuttle::GetHLTFileSources(const char* /*detector*/, const char* /*id*/){
1025// Retrieves a file from the HLT FES.
1026
1027return NULL;
1028
1029}
1030
1031//______________________________________________________________________________________________
1032const char* AliShuttle::GetDetCode(const char* detector){
1033// Return detector code
1034
1035 for(int iDet=0; iDet < fgkNDetectors; iDet++){
1036 if(!strcmp(fgkDetectorName[iDet], detector)) return fgkDetectorCode[iDet];
1037 }
b948db8d 1038
b948db8d 1039 return 0;
1040}
1041
1042//______________________________________________________________________________________________
1043void AliShuttle::Log(const char* detector, const char* message)
1044{
58bc3020 1045// Fill log string with a message
b948db8d 1046
84090f85 1047 void* dir = gSystem->OpenDirectory(fgkShuttleLogDir);
1048 if (dir == NULL) {
1049 if (gSystem->mkdir(fgkShuttleLogDir, kTRUE)) {
1050 AliError(Form("Can't open directory <%s>!", fgkShuttleTempDir));
1051 return;
1052 }
b948db8d 1053
84090f85 1054 } else {
1055 gSystem->FreeDirectory(dir);
1056 }
b948db8d 1057
85a80aa9 1058 TString toLog = Form("%s: %s, run %d - %s", TTimeStamp(time(0)).AsString("s"),
84090f85 1059 detector, GetCurrentRun(), message);
1060 AliInfo(toLog.Data());
b948db8d 1061
84090f85 1062 TString fileName;
1063 fileName.Form("%s/%s.log", fgkShuttleLogDir, detector);
1064 gSystem->ExpandPathName(fileName);
1065
1066 ofstream logFile;
1067 logFile.open(fileName, ofstream::out | ofstream::app);
1068
1069 if (!logFile.is_open()) {
1070 AliError(Form("Could not open file %s", fileName.Data()));
1071 return;
1072 }
7bfb2090 1073
84090f85 1074 logFile << toLog.Data() << "\n";
b948db8d 1075
84090f85 1076 logFile.close();
b948db8d 1077}