]> git.uio.no Git - u/mrichter/AliRoot.git/blame - SHUTTLE/AliShuttle.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Typo
[u/mrichter/AliRoot.git] / SHUTTLE / AliShuttle.cxx
CommitLineData
73abe331 1/**************************************************************************
2 * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16/*
17$Log$
1abfbb60 18Revision 1.68 2007/12/11 10:15:17 acolla
19Added marking SHUTTLE=DONE for invalid runs
20(invalid start time or end time) and runs with totalEvents < 1
21
b0e53b15 22Revision 1.67 2007/12/07 19:14:36 acolla
23in AliShuttleTrigger:
24
25Added automatic collection of new runs on a regular time basis (settable from the configuration)
26
27in AliShuttleConfig: new members
28
29- triggerWait: time to wait for DIM trigger (s) before starting automatic collection of new runs
30- mode: run mode (test, prod) -> used to build log folder (logs or logs_PROD)
31
32in AliShuttle:
33
34- logs now stored in logs/#RUN/DET_#RUN.log
35
7d4cf768 36Revision 1.66 2007/12/05 10:45:19 jgrosseo
37changed order of arguments to TMonaLisaWriter
38
468df1ce 39Revision 1.65 2007/11/26 16:58:37 acolla
40Monalisa configuration added: host and table name
41
b832ec02 42Revision 1.64 2007/11/13 16:15:47 acolla
43DCS map is stored in a file in the temp folder where the detector is processed.
44If the preprocessor fails, the temp folder is not removed. This will help the debugging of the problem.
45
b1d18693 46Revision 1.63 2007/11/02 10:53:16 acolla
47Protection added to AliShuttle::CopyFileLocally
48
7d43a416 49Revision 1.62 2007/10/31 18:23:13 acolla
50Furter developement on the Shuttle:
51
52- Shuttle now connects to the Grid as alidaq. The OCDB and Reference folders
53are now built from /alice/data, e.g.:
54/alice/data/2007/LHC07a/OCDB
55
56the year and LHC period are taken from the Shuttle.
57Raw metadata files are stored by GRP to:
58/alice/data/2007/LHC07a/<runNb>/Raw/RunMetadata.root
59
60- Shuttle sends a mail to DCS experts each time DP retrieval fails.
61
675f64cd 62Revision 1.61 2007/10/30 20:33:51 acolla
63Improved managing of temporary folders, which weren't correctly handled.
64Resolved bug introduced in StoreReferenceFile, which caused SPD preprocessor fail.
65
d524ade6 66Revision 1.60 2007/10/29 18:06:16 acolla
67
68New function StoreRunMetadataFile added to preprocessor and Shuttle interface
69This function can be used by GRP only. It stores raw data tags merged file to the
70raw data folder (e.g. /alice/data/2008/LHC08a/000099999/Raw).
71
72KNOWN ISSUES:
73
741. Shuttle cannot write to /alice/data/ because it belongs to alidaq. Tag file is stored in /alice/simulation/... for the time being.
752. Due to a bug in TAlien::Mkdir, the creation of a folder in recursive mode (-p option) does not work. The problem
76has been corrected in the root package on the Shuttle machine.
77
c88ad5db 78Revision 1.59 2007/10/05 12:40:55 acolla
79
80Result error code added to AliDCSClient data members (it was "lost" with the new implementation of TMap* GetAliasValues and GetDPValues).
81
1790d4b7 82Revision 1.58 2007/09/28 15:27:40 acolla
83
84AliDCSClient "multiSplit" option added in the DCS configuration
85in AliDCSMessage: variable MAX_BODY_SIZE set to 500000
86
542b6cc8 87Revision 1.57 2007/09/27 16:53:13 acolla
88Detectors can have more than one AMANDA server. SHUTTLE queries the servers sequentially,
89merges the dcs aliases/DPs in one TMap and sends it to the preprocessor.
90
1d172743 91Revision 1.56 2007/09/14 16:46:14 jgrosseo
921) Connect and Close are called before and after each query, so one can
93keep the same AliDCSClient object.
942) The splitting of a query is moved to GetDPValues/GetAliasValues.
953) Splitting interval can be specified in constructor
96
b41b252a 97Revision 1.55 2007/08/06 12:26:40 acolla
98Function Bool_t GetHLTStatus added to preprocessor. It returns the status of HLT
99read from the run logbook.
100
4859271b 101Revision 1.54 2007/07/12 09:51:25 jgrosseo
102removed duplicated log message in GetFile
103
4f0749a8 104Revision 1.53 2007/07/12 09:26:28 jgrosseo
105updating hlt fxs base path
106
42fde080 107Revision 1.52 2007/07/12 08:06:45 jgrosseo
108adding log messages in getfile... functions
109adding not implemented copy constructor in alishuttleconfigholder
110
1bcd28db 111Revision 1.51 2007/07/03 17:24:52 acolla
112root moved to v5-16-00. TFileMerger->Cp moved to TFile::Cp.
113
a986b218 114Revision 1.50 2007/07/02 17:19:32 acolla
115preprocessor is run in a temp directory that is removed when process is finished.
116
5bac2bde 117Revision 1.49 2007/06/29 10:45:06 acolla
118Number of columns in MySql Shuttle logbook increased by one (HLT added)
119
db99d43e 120Revision 1.48 2007/06/21 13:06:19 acolla
121GetFileSources returns dummy list with 1 source if system=DCS (better than
122returning error as it was)
123
6297b37d 124Revision 1.47 2007/06/19 17:28:56 acolla
125HLT updated; missing map bug removed.
126
dc25836b 127Revision 1.46 2007/06/09 13:01:09 jgrosseo
128Switching to retrieval of several DCS DPs at a time (multiDPrequest)
129
a038aa70 130Revision 1.45 2007/05/30 06:35:20 jgrosseo
131Adding functionality to the Shuttle/TestShuttle:
132o) Function to retrieve list of sources from a given system (GetFileSources with id=0)
133o) Function to retrieve list of IDs for a given source (GetFileIDs)
134These functions are needed for dealing with the tag files that are saved for the GRP preprocessor
135Example code has been added to the TestProcessor in TestShuttle
136
4a33bdd9 137Revision 1.44 2007/05/11 16:09:32 acolla
138Reference files for ITS, MUON and PHOS are now stored in OfflineDetName/OnlineDetName/run_...
139example: ITS/SPD/100_filename.root
140
2d9019b4 141Revision 1.43 2007/05/10 09:59:51 acolla
142Various bug fixes in StoreRefFilesToGrid; Cleaning of reference storage before processing detector (CleanReferenceStorage)
143
546242fb 144Revision 1.42 2007/05/03 08:01:39 jgrosseo
145typo in last commit :-(
146
8b739301 147Revision 1.41 2007/05/03 08:00:48 jgrosseo
148fixing log message when pp want to skip dcs value retrieval
149
651fdaab 150Revision 1.40 2007/04/27 07:06:48 jgrosseo
151GetFileSources returns empty list in case of no files, but successful query
152No mails sent in testmode
153
86aa42c3 154Revision 1.39 2007/04/17 12:43:57 acolla
155Correction in StoreOCDB; change of text in mail to detector expert
156
26758fce 157Revision 1.38 2007/04/12 08:26:18 jgrosseo
158updated comment
159
3c2a21c8 160Revision 1.37 2007/04/10 16:53:14 jgrosseo
161redirecting sub detector stdout, stderr to sub detector log file
162
3d8bc902 163Revision 1.35 2007/04/04 16:26:38 acolla
1641. Re-organization of function calls in TestPreprocessor to make it more meaningful.
1652. Added missing dependency in test preprocessors.
1663. in AliShuttle.cxx: processing time and memory consumption info on a single line.
167
886d60e6 168Revision 1.34 2007/04/04 10:33:36 jgrosseo
1691) Storing of files to the Grid is now done _after_ your preprocessors succeeded. This is transparent, which means that you can still use the same functions (Store, StoreReferenceData) to store files to the Grid. However, the Shuttle first stores them locally and transfers them after the preprocessor finished. The return code of these two functions has changed from UInt_t to Bool_t which gives you the success of the storing.
170In case of an error with the Grid, the Shuttle will retry the storing later, the preprocessor does not need to be run again.
171
1722) The meaning of the return code of the preprocessor has changed. 0 is now success and any other value means failure. This value is stored in the log and you can use it to keep details about the error condition.
173
1743) New function StoreReferenceFile to _directly_ store a file (without opening it) to the reference storage.
175
1764) The memory usage of the preprocessor is monitored. If it exceeds 2 GB it is terminated.
177
1785) New function AliPreprocessor::ProcessDCS(). If you do not need to have DCS data in all cases, you can skip the processing by implemting this function and returning kFALSE under certain conditions. E.g. if there is a certain run type.
179If you always need DCS data (like before), you do not need to implement it.
180
1816) The run type has been added to the monitoring page
182
9827400b 183Revision 1.33 2007/04/03 13:56:01 acolla
184Grid Storage at the end of preprocessing. Added virtual method to disable DCS query according to the
185run type.
186
3301427a 187Revision 1.32 2007/02/28 10:41:56 acolla
188Run type field added in SHUTTLE framework. Run type is read from "run type" logbook and retrieved by
189AliPreprocessor::GetRunType() function.
190Added some ldap definition files.
191
d386d623 192Revision 1.30 2007/02/13 11:23:21 acolla
193Moved getters and setters of Shuttle's main OCDB/Reference, local
194OCDB/Reference, temp and log folders to AliShuttleInterface
195
9d733021 196Revision 1.27 2007/01/30 17:52:42 jgrosseo
197adding monalisa monitoring
198
e7f62f16 199Revision 1.26 2007/01/23 19:20:03 acolla
200Removed old ldif files, added TOF, MCH ldif files. Added some options in
201AliShuttleConfig::Print. Added in Ali Shuttle: SetShuttleTempDir and
202SetShuttleLogDir
203
36c99a6a 204Revision 1.25 2007/01/15 19:13:52 acolla
205Moved some AliInfo to AliDebug in SendMail function
206
fc5a4708 207Revision 1.21 2006/12/07 08:51:26 jgrosseo
208update (alberto):
209table, db names in ldap configuration
210added GRP preprocessor
211DCS data can also be retrieved by data point
212
2c15234c 213Revision 1.20 2006/11/16 16:16:48 jgrosseo
214introducing strict run ordering flag
215removed giving preprocessor name to preprocessor, they have to know their name themselves ;-)
216
be48e3ea 217Revision 1.19 2006/11/06 14:23:04 jgrosseo
218major update (Alberto)
219o) reading of run parameters from the logbook
220o) online offline naming conversion
221o) standalone DCSclient package
222
eba76848 223Revision 1.18 2006/10/20 15:22:59 jgrosseo
224o) Adding time out to the execution of the preprocessors: The Shuttle forks and the parent process monitors the child
225o) Merging Collect, CollectAll, CollectNew function
226o) Removing implementation of empty copy constructors (declaration still there!)
227
cb343cfd 228Revision 1.17 2006/10/05 16:20:55 jgrosseo
229adapting to new CDB classes
230
6ec0e06c 231Revision 1.16 2006/10/05 15:46:26 jgrosseo
232applying to the new interface
233
481441a2 234Revision 1.15 2006/10/02 16:38:39 jgrosseo
235update (alberto):
236fixed memory leaks
237storing of objects that failed to be stored to the grid before
238interfacing of shuttle status table in daq system
239
2bb7b766 240Revision 1.14 2006/08/29 09:16:05 jgrosseo
241small update
242
85a80aa9 243Revision 1.13 2006/08/15 10:50:00 jgrosseo
244effc++ corrections (alberto)
245
4f0ab988 246Revision 1.12 2006/08/08 14:19:29 jgrosseo
247Update to shuttle classes (Alberto)
248
249- Possibility to set the full object's path in the Preprocessor's and
250Shuttle's Store functions
251- Possibility to extend the object's run validity in the same classes
252("startValidity" and "validityInfinite" parameters)
253- Implementation of the StoreReferenceData function to store reference
254data in a dedicated CDB storage.
255
84090f85 256Revision 1.11 2006/07/21 07:37:20 jgrosseo
257last run is stored after each run
258
7bfb2090 259Revision 1.10 2006/07/20 09:54:40 jgrosseo
260introducing status management: The processing per subdetector is divided into several steps,
261after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
262can keep track of the number of failures and skips further processing after a certain threshold is
263exceeded. These thresholds can be configured in LDAP.
264
5164a766 265Revision 1.9 2006/07/19 10:09:55 jgrosseo
266new configuration, accesst to DAQ FES (Alberto)
267
57f50b3c 268Revision 1.8 2006/07/11 12:44:36 jgrosseo
269adding parameters for extended validity range of data produced by preprocessor
270
17111222 271Revision 1.7 2006/07/10 14:37:09 jgrosseo
272small fix + todo comment
273
e090413b 274Revision 1.6 2006/07/10 13:01:41 jgrosseo
275enhanced storing of last sucessfully processed run (alberto)
276
a7160fe9 277Revision 1.5 2006/07/04 14:59:57 jgrosseo
278revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
279
45a493ce 280Revision 1.4 2006/06/12 09:11:16 jgrosseo
281coding conventions (Alberto)
282
58bc3020 283Revision 1.3 2006/06/06 14:26:40 jgrosseo
284o) removed files that were moved to STEER
285o) shuttle updated to follow the new interface (Alberto)
286
b948db8d 287Revision 1.2 2006/03/07 07:52:34 hristov
288New version (B.Yordanov)
289
d477ad88 290Revision 1.6 2005/11/19 17:19:14 byordano
291RetrieveDATEEntries and RetrieveConditionsData added
292
293Revision 1.5 2005/11/19 11:09:27 byordano
294AliShuttle declaration added
295
296Revision 1.4 2005/11/17 17:47:34 byordano
297TList changed to TObjArray
298
299Revision 1.3 2005/11/17 14:43:23 byordano
300import to local CVS
301
302Revision 1.1.1.1 2005/10/28 07:33:58 hristov
303Initial import as subdirectory in AliRoot
304
73abe331 305Revision 1.2 2005/09/13 08:41:15 byordano
306default startTime endTime added
307
308Revision 1.4 2005/08/30 09:13:02 byordano
309some docs added
310
311Revision 1.3 2005/08/29 21:15:47 byordano
312some docs added
313
314*/
315
316//
317// This class is the main manager for AliShuttle.
318// It organizes the data retrieval from DCS and call the
b948db8d 319// interface methods of AliPreprocessor.
73abe331 320// For every detector in AliShuttleConfgi (see AliShuttleConfig),
321// data for its set of aliases is retrieved. If there is registered
b948db8d 322// AliPreprocessor for this detector then it will be used
323// accroding to the schema (see AliPreprocessor).
324// If there isn't registered AliPreprocessor than the retrieved
73abe331 325// data is stored automatically to the undelying AliCDBStorage.
326// For detSpec is used the alias name.
327//
328
329#include "AliShuttle.h"
330
331#include "AliCDBManager.h"
332#include "AliCDBStorage.h"
333#include "AliCDBId.h"
84090f85 334#include "AliCDBRunRange.h"
335#include "AliCDBPath.h"
5164a766 336#include "AliCDBEntry.h"
73abe331 337#include "AliShuttleConfig.h"
eba76848 338#include "DCSClient/AliDCSClient.h"
73abe331 339#include "AliLog.h"
b948db8d 340#include "AliPreprocessor.h"
5164a766 341#include "AliShuttleStatus.h"
2bb7b766 342#include "AliShuttleLogbookEntry.h"
73abe331 343
57f50b3c 344#include <TSystem.h>
58bc3020 345#include <TObject.h>
b948db8d 346#include <TString.h>
57f50b3c 347#include <TTimeStamp.h>
73abe331 348#include <TObjString.h>
57f50b3c 349#include <TSQLServer.h>
350#include <TSQLResult.h>
351#include <TSQLRow.h>
cb343cfd 352#include <TMutex.h>
9827400b 353#include <TSystemDirectory.h>
354#include <TSystemFile.h>
a986b218 355#include <TFile.h>
9827400b 356#include <TGrid.h>
357#include <TGridResult.h>
73abe331 358
e7f62f16 359#include <TMonaLisaWriter.h>
360
5164a766 361#include <fstream>
362
cb343cfd 363#include <sys/types.h>
364#include <sys/wait.h>
365
73abe331 366ClassImp(AliShuttle)
367
b948db8d 368//______________________________________________________________________________________________
369AliShuttle::AliShuttle(const AliShuttleConfig* config,
370 UInt_t timeout, Int_t retries):
4f0ab988 371fConfig(config),
372fTimeout(timeout), fRetries(retries),
373fPreprocessorMap(),
2bb7b766 374fLogbookEntry(0),
eba76848 375fCurrentDetector(),
85a80aa9 376fStatusEntry(0),
cb343cfd 377fMonitoringMutex(0),
eba76848 378fLastActionTime(0),
e7f62f16 379fLastAction(),
9827400b 380fMonaLisa(0),
381fTestMode(kNone),
ffa29e93 382fReadTestMode(kFALSE),
383fOutputRedirected(kFALSE)
73abe331 384{
385 //
386 // config: AliShuttleConfig used
73abe331 387 // timeout: timeout used for AliDCSClient connection
388 // retries: the number of retries in case of connection error.
389 //
390
57f50b3c 391 if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
be48e3ea 392 for(int iSys=0;iSys<4;iSys++) {
57f50b3c 393 fServer[iSys]=0;
be48e3ea 394 if (iSys < 3)
2c15234c 395 fFXSlist[iSys].SetOwner(kTRUE);
57f50b3c 396 }
2bb7b766 397 fPreprocessorMap.SetOwner(kTRUE);
be48e3ea 398
399 for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
400 fFirstUnprocessed[iDet] = kFALSE;
401
cb343cfd 402 fMonitoringMutex = new TMutex();
58bc3020 403}
404
b948db8d 405//______________________________________________________________________________________________
57f50b3c 406AliShuttle::~AliShuttle()
58bc3020 407{
9827400b 408 //
409 // destructor
410 //
58bc3020 411
b948db8d 412 fPreprocessorMap.DeleteAll();
be48e3ea 413 for(int iSys=0;iSys<4;iSys++)
57f50b3c 414 if(fServer[iSys]) {
415 fServer[iSys]->Close();
416 delete fServer[iSys];
eba76848 417 fServer[iSys] = 0;
57f50b3c 418 }
2bb7b766 419
420 if (fStatusEntry){
421 delete fStatusEntry;
422 fStatusEntry = 0;
423 }
cb343cfd 424
425 if (fMonitoringMutex)
426 {
427 delete fMonitoringMutex;
428 fMonitoringMutex = 0;
429 }
73abe331 430}
431
b948db8d 432//______________________________________________________________________________________________
57f50b3c 433void AliShuttle::RegisterPreprocessor(AliPreprocessor* preprocessor)
58bc3020 434{
73abe331 435 //
b948db8d 436 // Registers new AliPreprocessor.
73abe331 437 // It uses GetName() for indentificator of the pre processor.
438 // The pre processor is registered it there isn't any other
439 // with the same identificator (GetName()).
440 //
441
eba76848 442 const char* detName = preprocessor->GetName();
443 if(GetDetPos(detName) < 0)
444 AliFatal(Form("********** !!!!! Invalid detector name: %s !!!!! **********", detName));
445
446 if (fPreprocessorMap.GetValue(detName)) {
447 AliWarning(Form("AliPreprocessor %s is already registered!", detName));
73abe331 448 return;
449 }
450
eba76848 451 fPreprocessorMap.Add(new TObjString(detName), preprocessor);
73abe331 452}
b948db8d 453//______________________________________________________________________________________________
3301427a 454Bool_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
84090f85 455 AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
73abe331 456{
9827400b 457 // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
458 // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
459 // using this function. Use StoreReferenceData instead!
460 // It calls StoreLocally function which temporarily stores the data locally; when the preprocessor
461 // finishes the data are transferred to the main storage (Grid).
b948db8d 462
3301427a 463 return StoreLocally(fgkLocalCDB, path, object, metaData, validityStart, validityInfinite);
84090f85 464}
465
466//______________________________________________________________________________________________
3301427a 467Bool_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData)
84090f85 468{
9827400b 469 // Stores a CDB object in the storage for reference data. This objects will not be available during
470 // offline reconstrunction. Use this function for reference data only!
471 // It calls StoreLocally function which temporarily stores the data locally; when the preprocessor
472 // finishes the data are transferred to the main storage (Grid).
85a80aa9 473
3301427a 474 return StoreLocally(fgkLocalRefStorage, path, object, metaData);
85a80aa9 475}
476
477//______________________________________________________________________________________________
3301427a 478Bool_t AliShuttle::StoreLocally(const TString& localUri,
85a80aa9 479 const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
480 Int_t validityStart, Bool_t validityInfinite)
481{
9827400b 482 // Store object temporarily in local storage. Parameters are passed by Store and StoreReferenceData functions.
483 // when the preprocessor finishes the data are transferred to the main storage (Grid).
484 // The parameters are:
485 // 1) Uri of the backup storage (Local)
486 // 2) the object's path.
487 // 3) the object to be stored
488 // 4) the metaData to be associated with the object
489 // 5) the validity start run number w.r.t. the current run,
490 // if the data is valid only for this run leave the default 0
491 // 6) specifies if the calibration data is valid for infinity (this means until updated),
492 // typical for calibration runs, the default is kFALSE
493 //
494 // returns 0 if fail, 1 otherwise
84090f85 495
9827400b 496 if (fTestMode & kErrorStorage)
497 {
498 Log(fCurrentDetector, "StoreLocally - In TESTMODE - Simulating error while storing locally");
499 return kFALSE;
500 }
501
3301427a 502 const char* cdbType = (localUri == fgkLocalCDB) ? "CDB" : "Reference";
2bb7b766 503
85a80aa9 504 Int_t firstRun = GetCurrentRun() - validityStart;
84090f85 505 if(firstRun < 0) {
9827400b 506 AliWarning("First valid run happens to be less than 0! Setting it to 0.");
84090f85 507 firstRun=0;
508 }
509
510 Int_t lastRun = -1;
511 if(validityInfinite) {
512 lastRun = AliCDBRunRange::Infinity();
513 } else {
514 lastRun = GetCurrentRun();
515 }
516
3301427a 517 // Version is set to current run, it will be used later to transfer data to Grid
518 AliCDBId id(path, firstRun, lastRun, GetCurrentRun(), -1);
2bb7b766 519
520 if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
521 TObjString runUsed = Form("%d", GetCurrentRun());
9e080f92 522 metaData->SetProperty("RunUsed(TObjString)", runUsed.Clone());
2bb7b766 523 }
84090f85 524
3301427a 525 Bool_t result = kFALSE;
84090f85 526
3301427a 527 if (!(AliCDBManager::Instance()->GetStorage(localUri))) {
528 Log("SHUTTLE", Form("StoreLocally - Cannot activate local %s storage", cdbType));
84090f85 529 } else {
3301427a 530 result = AliCDBManager::Instance()->GetStorage(localUri)
84090f85 531 ->Put(object, id, metaData);
532 }
533
534 if(!result) {
535
9827400b 536 Log(fCurrentDetector, Form("StoreLocally - Can't store object <%s>!", id.ToString().Data()));
3301427a 537 }
2bb7b766 538
3301427a 539 return result;
540}
84090f85 541
3301427a 542//______________________________________________________________________________________________
543Bool_t AliShuttle::StoreOCDB()
544{
9827400b 545 //
546 // Called when preprocessor ends successfully or when previous storage attempt failed (kStoreError status)
547 // Calls underlying StoreOCDB(const char*) function twice, for OCDB and Reference storage.
548 // Then calls StoreRefFilesToGrid to store reference files.
549 //
550
551 if (fTestMode & kErrorGrid)
552 {
553 Log("SHUTTLE", "StoreOCDB - In TESTMODE - Simulating error while storing in the Grid");
554 Log(fCurrentDetector, "StoreOCDB - In TESTMODE - Simulating error while storing in the Grid");
555 return kFALSE;
556 }
557
c88ad5db 558 Log("SHUTTLE","StoreOCDB - Storing OCDB data ...");
86aa42c3 559 Bool_t resultCDB = StoreOCDB(fgkMainCDB);
560
c88ad5db 561 Log("SHUTTLE","StoreOCDB - Storing reference data ...");
3301427a 562 Bool_t resultRef = StoreOCDB(fgkMainRefStorage);
9827400b 563
c88ad5db 564 Log("SHUTTLE","StoreOCDB - Storing reference files ...");
565 Bool_t resultRefFiles = CopyFilesToGrid("reference");
566
567 Bool_t resultMetadata = kTRUE;
568 if(fCurrentDetector == "GRP")
569 {
570 Log("StoreOCDB - SHUTTLE","Storing Run Metadata file ...");
571 resultMetadata = CopyFilesToGrid("metadata");
572 }
9827400b 573
c88ad5db 574 return resultCDB && resultRef && resultRefFiles && resultMetadata;
3301427a 575}
576
577//______________________________________________________________________________________________
578Bool_t AliShuttle::StoreOCDB(const TString& gridURI)
579{
580 //
581 // Called by StoreOCDB(), performs actual storage to the main OCDB and reference storages (Grid)
582 //
583
584 TObjArray* gridIds=0;
585
586 Bool_t result = kTRUE;
587
588 const char* type = 0;
589 TString localURI;
590 if(gridURI == fgkMainCDB) {
591 type = "OCDB";
592 localURI = fgkLocalCDB;
593 } else if(gridURI == fgkMainRefStorage) {
594 type = "reference";
595 localURI = fgkLocalRefStorage;
596 } else {
597 AliError(Form("Invalid storage URI: %s", gridURI.Data()));
598 return kFALSE;
599 }
600
601 AliCDBManager* man = AliCDBManager::Instance();
602
603 AliCDBStorage *gridSto = man->GetStorage(gridURI);
604 if(!gridSto) {
605 Log("SHUTTLE",
606 Form("StoreOCDB - cannot activate main %s storage", type));
607 return kFALSE;
608 }
609
610 gridIds = gridSto->GetQueryCDBList();
611
612 // get objects previously stored in local CDB
613 AliCDBStorage *localSto = man->GetStorage(localURI);
614 if(!localSto) {
615 Log("SHUTTLE",
616 Form("StoreOCDB - cannot activate local %s storage", type));
617 return kFALSE;
618 }
619 AliCDBPath aPath(GetOfflineDetName(fCurrentDetector.Data()),"*","*");
620 // Local objects were stored with current run as Grid version!
621 TList* localEntries = localSto->GetAll(aPath.GetPath(), GetCurrentRun(), GetCurrentRun());
622 localEntries->SetOwner(1);
623
624 // loop on local stored objects
625 TIter localIter(localEntries);
626 AliCDBEntry *aLocEntry = 0;
627 while((aLocEntry = dynamic_cast<AliCDBEntry*> (localIter.Next()))){
628 aLocEntry->SetOwner(1);
629 AliCDBId aLocId = aLocEntry->GetId();
630 aLocEntry->SetVersion(-1);
631 aLocEntry->SetSubVersion(-1);
632
633 // If local object is valid up to infinity we store it only if it is
634 // the first unprocessed run!
635 if (aLocId.GetLastRun() == AliCDBRunRange::Infinity() &&
636 !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
637 {
638 Log("SHUTTLE", Form("StoreOCDB - %s: object %s has validity infinite but "
639 "there are previous unprocessed runs!",
640 fCurrentDetector.Data(), aLocId.GetPath().Data()));
641 continue;
642 }
643
644 // loop on Grid valid Id's
645 Bool_t store = kTRUE;
646 TIter gridIter(gridIds);
647 AliCDBId* aGridId = 0;
648 while((aGridId = dynamic_cast<AliCDBId*> (gridIter.Next()))){
649 if(aGridId->GetPath() != aLocId.GetPath()) continue;
650 // skip all objects valid up to infinity
651 if(aGridId->GetLastRun() == AliCDBRunRange::Infinity()) continue;
652 // if we get here, it means there's already some more recent object stored on Grid!
653 store = kFALSE;
654 break;
655 }
656
657 // If we get here, the file can be stored!
658 Bool_t storeOk = gridSto->Put(aLocEntry);
659 if(!store || storeOk){
660
661 if (!store)
662 {
663 Log(fCurrentDetector.Data(),
664 Form("StoreOCDB - A more recent object already exists in %s storage: <%s>",
665 type, aGridId->ToString().Data()));
666 } else {
667 Log("SHUTTLE",
668 Form("StoreOCDB - Object <%s> successfully put into %s storage",
669 aLocId.ToString().Data(), type));
2d9019b4 670 Log(fCurrentDetector.Data(),
671 Form("StoreOCDB - Object <%s> successfully put into %s storage",
672 aLocId.ToString().Data(), type));
3301427a 673 }
84090f85 674
3301427a 675 // removing local filename...
676 TString filename;
677 localSto->IdToFilename(aLocId, filename);
c88ad5db 678 Log("SHUTTLE", Form("StoreOCDB - Removing local file %s", filename.Data()));
3301427a 679 RemoveFile(filename.Data());
680 continue;
681 } else {
682 Log("SHUTTLE",
683 Form("StoreOCDB - Grid %s storage of object <%s> failed",
684 type, aLocId.ToString().Data()));
2d9019b4 685 Log(fCurrentDetector.Data(),
686 Form("StoreOCDB - Grid %s storage of object <%s> failed",
687 type, aLocId.ToString().Data()));
3301427a 688 result = kFALSE;
b948db8d 689 }
690 }
3301427a 691 localEntries->Clear();
2bb7b766 692
b948db8d 693 return result;
3301427a 694}
695
546242fb 696//______________________________________________________________________________________________
697Bool_t AliShuttle::CleanReferenceStorage(const char* detector)
698{
2d9019b4 699 // clears the directory used to store reference files of a given subdetector
546242fb 700
701 AliCDBManager* man = AliCDBManager::Instance();
702 AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
2d9019b4 703 TString localBaseFolder = sto->GetBaseFolder();
704
705 TString targetDir = GetRefFilePrefix(localBaseFolder.Data(), detector);
706
d524ade6 707 Log("SHUTTLE", Form("CleanReferenceStorage - Cleaning %s", targetDir.Data()));
2d9019b4 708
709 TString begin;
710 begin.Form("%d_", GetCurrentRun());
711
712 TSystemDirectory* baseDir = new TSystemDirectory("/", targetDir);
713 if (!baseDir)
714 return kTRUE;
715
716 TList* dirList = baseDir->GetListOfFiles();
717 delete baseDir;
718
719 if (!dirList) return kTRUE;
720
721 if (dirList->GetEntries() < 3)
722 {
723 delete dirList;
724 return kTRUE;
725 }
726
727 Int_t nDirs = 0, nDel = 0;
728 TIter dirIter(dirList);
729 TSystemFile* entry = 0;
546242fb 730
2d9019b4 731 Bool_t success = kTRUE;
546242fb 732
2d9019b4 733 while ((entry = dynamic_cast<TSystemFile*> (dirIter.Next())))
734 {
735 if (entry->IsDirectory())
736 continue;
737
738 TString fileName(entry->GetName());
739 if (!fileName.BeginsWith(begin))
740 continue;
741
742 nDirs++;
743
744 // delete file
745 Int_t result = gSystem->Unlink(fileName.Data());
746
747 if (result)
748 {
d524ade6 749 Log("SHUTTLE", Form("CleanReferenceStorage - Could not delete file %s!", fileName.Data()));
2d9019b4 750 success = kFALSE;
751 } else {
752 nDel++;
753 }
754 }
755
756 if(nDirs > 0)
757 Log("SHUTTLE", Form("CleanReferenceStorage - %d (over %d) reference files in folder %s were deleted.",
758 nDel, nDirs, targetDir.Data()));
759
760
761 delete dirList;
762 return success;
763
764
765
766
767
546242fb 768
769 Int_t result = gSystem->GetPathInfo(targetDir, 0, (Long64_t*) 0, 0, 0);
770 if (result == 0)
771 {
772 // delete directory
d524ade6 773 result = gSystem->Exec(Form("rm -rf %s", targetDir.Data()));
546242fb 774 if (result != 0)
775 {
d524ade6 776 Log("SHUTTLE", Form("CleanReferenceStorage - Could not clean directory %s", targetDir.Data()));
546242fb 777 return kFALSE;
778 }
779 }
780
781 result = gSystem->mkdir(targetDir, kTRUE);
782 if (result != 0)
783 {
c88ad5db 784 Log("SHUTTLE", Form("CleanReferenceStorage - Error creating base directory %s", targetDir.Data()));
546242fb 785 return kFALSE;
786 }
787
788 return kTRUE;
789}
790
9827400b 791//______________________________________________________________________________________________
792Bool_t AliShuttle::StoreReferenceFile(const char* detector, const char* localFile, const char* gridFileName)
793{
794 //
3c2a21c8 795 // Stores reference file directly (without opening it). This function stores the file locally.
9827400b 796 //
3c2a21c8 797 // The file is stored under the following location:
798 // <base folder of local reference storage>/<DET>/<RUN#>_<gridFileName>
799 // where <gridFileName> is the second parameter given to the function
800 //
9827400b 801
802 if (fTestMode & kErrorStorage)
803 {
804 Log(fCurrentDetector, "StoreReferenceFile - In TESTMODE - Simulating error while storing locally");
805 return kFALSE;
806 }
807
808 AliCDBManager* man = AliCDBManager::Instance();
809 AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
810
811 TString localBaseFolder = sto->GetBaseFolder();
812
d524ade6 813 TString target = GetRefFilePrefix(localBaseFolder.Data(), detector);
814 target.Append(Form("/%d_%s", GetCurrentRun(), gridFileName));
9827400b 815
d524ade6 816 return CopyFileLocally(localFile, target);
c88ad5db 817}
818
819//______________________________________________________________________________________________
820Bool_t AliShuttle::StoreRunMetadataFile(const char* localFile, const char* gridFileName)
821{
822 //
823 // Stores Run metadata file to the Grid, in the run folder
824 //
825 // Only GRP can call this function.
826
827 if (fTestMode & kErrorStorage)
828 {
829 Log(fCurrentDetector, "StoreRunMetaDataFile - In TESTMODE - Simulating error while storing locally");
830 return kFALSE;
831 }
832
833 AliCDBManager* man = AliCDBManager::Instance();
834 AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
835
836 TString localBaseFolder = sto->GetBaseFolder();
837
838 // Build Run level folder
839 // folder = /alice/data/year/lhcPeriod/runNb/Raw
840
c88ad5db 841
675f64cd 842 TString lhcPeriod = GetLHCPeriod();
c88ad5db 843 if (lhcPeriod.Length() == 0)
844 {
845 Log("SHUTTLE","StoreRunMetaDataFile - LHCPeriod not found in logbook!");
846 return 0;
847 }
848
675f64cd 849 TString target = Form("%s/GRP/RunMetadata/alice/data/%d/%s/%09d/Raw/%s",
850 localBaseFolder.Data(), GetCurrentYear(),
d524ade6 851 lhcPeriod.Data(), GetCurrentRun(), gridFileName);
c88ad5db 852
d524ade6 853 return CopyFileLocally(localFile, target);
c88ad5db 854}
855
856//______________________________________________________________________________________________
d524ade6 857Bool_t AliShuttle::CopyFileLocally(const char* localFile, const TString& target)
c88ad5db 858{
859 //
860 // Stores file locally. Called by StoreReferenceFile and StoreRunMetadataFile
d524ade6 861 // Files are temporarily stored in the local reference storage. When the preprocessor
862 // finishes, the Shuttle calls CopyFilesToGrid to transfer the files to AliEn
863 // (in reference or run level folders)
c88ad5db 864 //
865
d524ade6 866 TString targetDir(target(0, target.Last('/')));
867
868 //try to open base dir folder, if it does not exist
2d9019b4 869 void* dir = gSystem->OpenDirectory(targetDir.Data());
870 if (dir == NULL) {
871 if (gSystem->mkdir(targetDir.Data(), kTRUE)) {
c88ad5db 872 Log("SHUTTLE", Form("StoreFileLocally - Can't open directory <%s>", targetDir.Data()));
2d9019b4 873 return kFALSE;
874 }
875
876 } else {
877 gSystem->FreeDirectory(dir);
878 }
9827400b 879
7d43a416 880 Int_t result = 0;
881
882 result = gSystem->GetPathInfo(localFile, 0, (Long64_t*) 0, 0, 0);
9827400b 883 if (result)
884 {
c88ad5db 885 Log("SHUTTLE", Form("StoreFileLocally - %s does not exist", localFile));
546242fb 886 return kFALSE;
9827400b 887 }
546242fb 888
7d43a416 889 result = gSystem->GetPathInfo(target, 0, (Long64_t*) 0, 0, 0);
890 if (!result)
891 {
892 Log("SHUTTLE", Form("StoreFileLocally - target file %s already exist, removing...", target.Data()));
893 if (gSystem->Unlink(target.Data()))
894 {
895 Log("SHUTTLE", Form("StoreFileLocally - Could not remove existing target file %s!", target.Data()));
896 return kFALSE;
897 }
898 }
899
9827400b 900 result = gSystem->CopyFile(localFile, target);
901
902 if (result == 0)
903 {
c88ad5db 904 Log("SHUTTLE", Form("StoreFileLocally - File %s stored locally to %s", localFile, target.Data()));
9827400b 905 return kTRUE;
906 }
907 else
908 {
d524ade6 909 Log("SHUTTLE", Form("StoreFileLocally - Could not store file %s to %s! Error code = %d",
546242fb 910 localFile, target.Data(), result));
9827400b 911 return kFALSE;
912 }
c88ad5db 913
914
915
9827400b 916}
917
918//______________________________________________________________________________________________
c88ad5db 919Bool_t AliShuttle::CopyFilesToGrid(const char* type)
9827400b 920{
921 //
c88ad5db 922 // Transfers local files to the Grid. Local files can be reference files
923 // or run metadata file (from GRP only).
9827400b 924 //
c88ad5db 925 // According to the type (ref, metadata) the files are stored under the following location:
926 // ref --> <base folder of reference storage>/<DET>/<RUN#>_<gridFileName>
927 // metadata --> <run data folder>/<MetadataFileName>
86aa42c3 928 //
c88ad5db 929
9827400b 930 AliCDBManager* man = AliCDBManager::Instance();
931 AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
932 if (!sto)
933 return kFALSE;
934 TString localBaseFolder = sto->GetBaseFolder();
9827400b 935
c88ad5db 936 TString dir;
937 TString alienDir;
9827400b 938 TString begin;
9827400b 939
c88ad5db 940 if (strcmp(type, "reference") == 0)
941 {
942 dir = GetRefFilePrefix(localBaseFolder.Data(), fCurrentDetector.Data());
943 AliCDBStorage* gridSto = man->GetStorage(fgkMainRefStorage);
944 if (!gridSto)
945 return kFALSE;
946 TString gridBaseFolder = gridSto->GetBaseFolder();
947 alienDir = GetRefFilePrefix(gridBaseFolder.Data(), fCurrentDetector.Data());
948 begin = Form("%d_", GetCurrentRun());
949 }
950 else if (strcmp(type, "metadata") == 0)
951 {
c88ad5db 952
675f64cd 953 TString lhcPeriod = GetLHCPeriod();
c88ad5db 954
955 if (lhcPeriod.Length() == 0)
956 {
957 Log("SHUTTLE","CopyFilesToGrid - LHCPeriod not found in logbook!");
958 return 0;
959 }
960
675f64cd 961 dir = Form("%s/GRP/RunMetadata/alice/data/%d/%s/%09d/Raw",
962 localBaseFolder.Data(), GetCurrentYear(),
c88ad5db 963 lhcPeriod.Data(), GetCurrentRun());
675f64cd 964 alienDir = dir(dir.Index("/alice/data/"), dir.Length());
965
c88ad5db 966 begin = "";
967 }
968 else
969 {
970 Log("SHUTTLE", "CopyFilesToGrid - Unexpected: type label must be reference or metadata!");
971 return kFALSE;
972 }
973
9827400b 974 TSystemDirectory* baseDir = new TSystemDirectory("/", dir);
3d8bc902 975 if (!baseDir)
976 return kTRUE;
977
2d9019b4 978 TList* dirList = baseDir->GetListOfFiles();
979 delete baseDir;
980
981 if (!dirList) return kTRUE;
982
983 if (dirList->GetEntries() < 3)
3d8bc902 984 {
2d9019b4 985 delete dirList;
9827400b 986 return kTRUE;
3d8bc902 987 }
2d9019b4 988
546242fb 989 if (!gGrid)
990 {
c88ad5db 991 Log("SHUTTLE", "CopyFilesToGrid - Connection to Grid failed: Cannot continue!");
2d9019b4 992 delete dirList;
546242fb 993 return kFALSE;
994 }
995
2d9019b4 996 Int_t nDirs = 0, nTransfer = 0;
997 TIter dirIter(dirList);
998 TSystemFile* entry = 0;
999
9827400b 1000 Bool_t success = kTRUE;
3d8bc902 1001 Bool_t first = kTRUE;
9827400b 1002
2d9019b4 1003 while ((entry = dynamic_cast<TSystemFile*> (dirIter.Next())))
1004 {
9827400b 1005 if (entry->IsDirectory())
1006 continue;
1007
1008 TString fileName(entry->GetName());
1009 if (!fileName.BeginsWith(begin))
1010 continue;
1011
2d9019b4 1012 nDirs++;
1013
3d8bc902 1014 if (first)
1015 {
1016 first = kFALSE;
c88ad5db 1017 // check that folder exists, otherwise create it
3d8bc902 1018 TGridResult* result = gGrid->Ls(alienDir.Data(), "a");
1019
1020 if (!result)
2d9019b4 1021 {
1022 delete dirList;
3d8bc902 1023 return kFALSE;
2d9019b4 1024 }
3d8bc902 1025
546242fb 1026 if (!result->GetFileName(1)) // TODO: It looks like element 0 is always 0!!
3d8bc902 1027 {
675f64cd 1028 // TODO It does not work currently! Bug in TAliEn::Mkdir
1029 // TODO Manually fixed in local root v5-16-00
c88ad5db 1030 if (!gGrid->Mkdir(alienDir.Data(),"-p",0))
3d8bc902 1031 {
c88ad5db 1032 Log("SHUTTLE", Form("CopyFilesToGrid - Cannot create directory %s",
3d8bc902 1033 alienDir.Data()));
2d9019b4 1034 delete dirList;
3d8bc902 1035 return kFALSE;
546242fb 1036 } else {
c88ad5db 1037 Log("SHUTTLE",Form("CopyFilesToGrid - Folder %s created", alienDir.Data()));
3d8bc902 1038 }
1039
546242fb 1040 } else {
c88ad5db 1041 Log("SHUTTLE",Form("CopyFilesToGrid - Folder %s found", alienDir.Data()));
3d8bc902 1042 }
1043 }
1044
9827400b 1045 TString fullLocalPath;
1046 fullLocalPath.Form("%s/%s", dir.Data(), fileName.Data());
1047
1048 TString fullGridPath;
1049 fullGridPath.Form("alien://%s/%s", alienDir.Data(), fileName.Data());
1050
a986b218 1051 Bool_t result = TFile::Cp(fullLocalPath, fullGridPath);
9827400b 1052
1053 if (result)
1054 {
c88ad5db 1055 Log("SHUTTLE", Form("CopyFilesToGrid - Copying local file %s to %s succeeded!",
1056 fullLocalPath.Data(), fullGridPath.Data()));
9827400b 1057 RemoveFile(fullLocalPath);
2d9019b4 1058 nTransfer++;
9827400b 1059 }
1060 else
1061 {
c88ad5db 1062 Log("SHUTTLE", Form("CopyFilesToGrid - Copying local file %s to %s FAILED!",
1063 fullLocalPath.Data(), fullGridPath.Data()));
9827400b 1064 success = kFALSE;
1065 }
1066 }
2d9019b4 1067
c88ad5db 1068 Log("SHUTTLE", Form("CopyFilesToGrid - %d (over %d) files in folder %s copied to Grid.",
1069 nTransfer, nDirs, dir.Data()));
2d9019b4 1070
1071
1072 delete dirList;
9827400b 1073 return success;
1074}
1075
2d9019b4 1076//______________________________________________________________________________________________
1077const char* AliShuttle::GetRefFilePrefix(const char* base, const char* detector)
1078{
1079 //
1080 // Get folder name of reference files
1081 //
1082
1083 TString offDetStr(GetOfflineDetName(detector));
1084 TString dir;
1085 if (offDetStr == "ITS" || offDetStr == "MUON" || offDetStr == "PHOS")
1086 {
1087 dir.Form("%s/%s/%s", base, offDetStr.Data(), detector);
1088 } else {
1089 dir.Form("%s/%s", base, offDetStr.Data());
1090 }
1091
1092 return dir.Data();
1093
1094
1095}
c88ad5db 1096
3301427a 1097//______________________________________________________________________________________________
1098void AliShuttle::CleanLocalStorage(const TString& uri)
1099{
9827400b 1100 //
1101 // Called in case the preprocessor is declared failed. Remove remaining objects from the local storages.
1102 //
3301427a 1103
1104 const char* type = 0;
1105 if(uri == fgkLocalCDB) {
1106 type = "OCDB";
1107 } else if(uri == fgkLocalRefStorage) {
546242fb 1108 type = "Reference";
3301427a 1109 } else {
1110 AliError(Form("Invalid storage URI: %s", uri.Data()));
1111 return;
1112 }
1113
1114 AliCDBManager* man = AliCDBManager::Instance();
b948db8d 1115
3301427a 1116 // open local storage
1117 AliCDBStorage *localSto = man->GetStorage(uri);
1118 if(!localSto) {
1119 Log("SHUTTLE",
1120 Form("CleanLocalStorage - cannot activate local %s storage", type));
1121 return;
1122 }
1123
1124 TString filename(Form("%s/%s/*/Run*_v%d_s*.root",
546242fb 1125 localSto->GetBaseFolder().Data(), GetOfflineDetName(fCurrentDetector.Data()), GetCurrentRun()));
3301427a 1126
c88ad5db 1127 AliDebug(2, Form("filename = %s", filename.Data()));
3301427a 1128
c88ad5db 1129 Log("SHUTTLE", Form("Removing remaining local files for run %d and detector %s ...",
3301427a 1130 GetCurrentRun(), fCurrentDetector.Data()));
1131
1132 RemoveFile(filename.Data());
1133
1134}
1135
1136//______________________________________________________________________________________________
1137void AliShuttle::RemoveFile(const char* filename)
1138{
9827400b 1139 //
1140 // removes local file
1141 //
3301427a 1142
1143 TString command(Form("rm -f %s", filename));
1144
1145 Int_t result = gSystem->Exec(command.Data());
1146 if(result != 0)
1147 {
1148 Log("SHUTTLE", Form("RemoveFile - %s: Cannot remove file %s!",
1149 fCurrentDetector.Data(), filename));
1150 }
73abe331 1151}
1152
b948db8d 1153//______________________________________________________________________________________________
5164a766 1154AliShuttleStatus* AliShuttle::ReadShuttleStatus()
1155{
9827400b 1156 //
1157 // Reads the AliShuttleStatus from the CDB
1158 //
5164a766 1159
2bb7b766 1160 if (fStatusEntry){
1161 delete fStatusEntry;
1162 fStatusEntry = 0;
1163 }
5164a766 1164
10a5a932 1165 fStatusEntry = AliCDBManager::Instance()->GetStorage(GetLocalCDB())
2bb7b766 1166 ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), GetCurrentRun());
5164a766 1167
2bb7b766 1168 if (!fStatusEntry) return 0;
1169 fStatusEntry->SetOwner(1);
5164a766 1170
2bb7b766 1171 AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
1172 if (!status) {
1173 AliError("Invalid object stored to CDB!");
1174 return 0;
1175 }
5164a766 1176
2bb7b766 1177 return status;
5164a766 1178}
1179
1180//______________________________________________________________________________________________
7bfb2090 1181Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
5164a766 1182{
9827400b 1183 //
1184 // writes the status for one subdetector
1185 //
2bb7b766 1186
1187 if (fStatusEntry){
1188 delete fStatusEntry;
1189 fStatusEntry = 0;
1190 }
5164a766 1191
2bb7b766 1192 Int_t run = GetCurrentRun();
5164a766 1193
2bb7b766 1194 AliCDBId id(AliCDBPath("SHUTTLE", "STATUS", fCurrentDetector), run, run);
5164a766 1195
2bb7b766 1196 fStatusEntry = new AliCDBEntry(status, id, new AliCDBMetaData);
1197 fStatusEntry->SetOwner(1);
5164a766 1198
2bb7b766 1199 UInt_t result = AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
7bfb2090 1200
2bb7b766 1201 if (!result) {
3301427a 1202 Log("SHUTTLE", Form("WriteShuttleStatus - Failed for %s, run %d",
1203 fCurrentDetector.Data(), run));
2bb7b766 1204 return kFALSE;
1205 }
e7f62f16 1206
1207 SendMLInfo();
7bfb2090 1208
2bb7b766 1209 return kTRUE;
5164a766 1210}
1211
1212//______________________________________________________________________________________________
1213void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t increaseCount)
1214{
9827400b 1215 //
1216 // changes the AliShuttleStatus for the given detector and run to the given status
1217 //
5164a766 1218
2bb7b766 1219 if (!fStatusEntry){
1220 AliError("UNEXPECTED: fStatusEntry empty");
1221 return;
1222 }
5164a766 1223
2bb7b766 1224 AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
5164a766 1225
2bb7b766 1226 if (!status){
c88ad5db 1227 Log("SHUTTLE", "UpdateShuttleStatus - UNEXPECTED: status could not be read from current CDB entry");
2bb7b766 1228 return;
1229 }
5164a766 1230
2c15234c 1231 TString actionStr = Form("UpdateShuttleStatus - %s: Changing state from %s to %s",
eba76848 1232 fCurrentDetector.Data(),
36c99a6a 1233 status->GetStatusName(),
eba76848 1234 status->GetStatusName(newStatus));
cb343cfd 1235 Log("SHUTTLE", actionStr);
1236 SetLastAction(actionStr);
5164a766 1237
2bb7b766 1238 status->SetStatus(newStatus);
1239 if (increaseCount) status->IncreaseCount();
5164a766 1240
2bb7b766 1241 AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
e7f62f16 1242
1243 SendMLInfo();
5164a766 1244}
e7f62f16 1245
1246//______________________________________________________________________________________________
1247void AliShuttle::SendMLInfo()
1248{
1249 //
1250 // sends ML information about the current status of the current detector being processed
1251 //
1252
1253 AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
1254
1255 if (!status){
3301427a 1256 Log("SHUTTLE", "SendMLInfo - UNEXPECTED: status could not be read from current CDB entry");
e7f62f16 1257 return;
1258 }
1259
1260 TMonaLisaText mlStatus(Form("%s_status", fCurrentDetector.Data()), status->GetStatusName());
1261 TMonaLisaValue mlRetryCount(Form("%s_count", fCurrentDetector.Data()), status->GetCount());
1262
1263 TList mlList;
1264 mlList.Add(&mlStatus);
1265 mlList.Add(&mlRetryCount);
1266
1267 fMonaLisa->SendParameters(&mlList);
1268}
1269
5164a766 1270//______________________________________________________________________________________________
1271Bool_t AliShuttle::ContinueProcessing()
1272{
9827400b 1273 // this function reads the AliShuttleStatus information from CDB and
1274 // checks if the processing should be continued
1275 // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
2bb7b766 1276
57c1a579 1277 if (!fConfig->HostProcessDetector(fCurrentDetector)) return kFALSE;
1278
1279 AliPreprocessor* aPreprocessor =
1280 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
1281 if (!aPreprocessor)
1282 {
c88ad5db 1283 Log("SHUTTLE", Form("ContinueProcessing - %s: no preprocessor registered", fCurrentDetector.Data()));
57c1a579 1284 return kFALSE;
1285 }
1286
2bb7b766 1287 AliShuttleLogbookEntry::Status entryStatus =
eba76848 1288 fLogbookEntry->GetDetectorStatus(fCurrentDetector);
2bb7b766 1289
1290 if(entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
c88ad5db 1291 Log("SHUTTLE", Form("ContinueProcessing - %s is %s",
2bb7b766 1292 fCurrentDetector.Data(),
1293 fLogbookEntry->GetDetectorStatusName(entryStatus)));
1294 return kFALSE;
1295 }
1296
1297 // if we get here, according to Shuttle logbook subdetector is in UNPROCESSED state
be48e3ea 1298
1299 // check if current run is first unprocessed run for current detector
1300 if (fConfig->StrictRunOrder(fCurrentDetector) &&
1301 !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
1302 {
86aa42c3 1303 if (fTestMode == kNone)
1304 {
c88ad5db 1305 Log("SHUTTLE", Form("ContinueProcessing - %s requires strict run ordering"
1306 " but this is not the first unprocessed run!"));
86aa42c3 1307 return kFALSE;
1308 }
1309 else
1310 {
c88ad5db 1311 Log("SHUTTLE", Form("ContinueProcessing - In TESTMODE - "
1312 "Although %s requires strict run ordering "
1313 "and this is not the first unprocessed run, "
1314 "the SHUTTLE continues"));
86aa42c3 1315 }
be48e3ea 1316 }
1317
2bb7b766 1318 AliShuttleStatus* status = ReadShuttleStatus();
1319 if (!status) {
1320 // first time
1321 Log("SHUTTLE", Form("ContinueProcessing - %s: Processing first time",
1322 fCurrentDetector.Data()));
1323 status = new AliShuttleStatus(AliShuttleStatus::kStarted);
1324 return WriteShuttleStatus(status);
1325 }
1326
1327 // The following two cases shouldn't happen if Shuttle Logbook was correctly updated.
1328 // If it happens it may mean Logbook updating failed... let's do it now!
1329 if (status->GetStatus() == AliShuttleStatus::kDone ||
1330 status->GetStatus() == AliShuttleStatus::kFailed){
1331 Log("SHUTTLE", Form("ContinueProcessing - %s is already %s. Updating Shuttle Logbook",
1332 fCurrentDetector.Data(),
1333 status->GetStatusName(status->GetStatus())));
1334 UpdateShuttleLogbook(fCurrentDetector.Data(),
1335 status->GetStatusName(status->GetStatus()));
1336 return kFALSE;
1337 }
1338
3301427a 1339 if (status->GetStatus() == AliShuttleStatus::kStoreError) {
2bb7b766 1340 Log("SHUTTLE",
c88ad5db 1341 Form("ContinueProcessing - %s: Grid storage of one or more "
1342 "objects failed. Trying again now",
2bb7b766 1343 fCurrentDetector.Data()));
9827400b 1344 UpdateShuttleStatus(AliShuttleStatus::kStoreStarted);
1345 if (StoreOCDB()){
c88ad5db 1346 Log("SHUTTLE", Form("ContinueProcessing - %s: all objects "
1347 "successfully stored into main storage",
3301427a 1348 fCurrentDetector.Data()));
2bb7b766 1349 } else {
1350 Log("SHUTTLE",
1351 Form("ContinueProcessing - %s: Grid storage failed again",
1352 fCurrentDetector.Data()));
9827400b 1353 UpdateShuttleStatus(AliShuttleStatus::kStoreError);
2bb7b766 1354 }
1355 return kFALSE;
1356 }
1357
1358 // if we get here, there is a restart
57c1a579 1359 Bool_t cont = kFALSE;
2bb7b766 1360
1361 // abort conditions
cb343cfd 1362 if (status->GetCount() >= fConfig->GetMaxRetries()) {
57c1a579 1363 Log("SHUTTLE", Form("ContinueProcessing - %s failed %d times in status %s - "
1364 "Updating Shuttle Logbook", fCurrentDetector.Data(),
2bb7b766 1365 status->GetCount(), status->GetStatusName()));
1366 UpdateShuttleLogbook(fCurrentDetector.Data(), "FAILED");
e7f62f16 1367 UpdateShuttleStatus(AliShuttleStatus::kFailed);
3301427a 1368
1369 // there may still be objects in local OCDB and reference storage
1370 // and FXS databases may be not updated: do it now!
9827400b 1371
1372 // TODO Currently disabled, we want to keep files in case of failure!
1373 // CleanLocalStorage(fgkLocalCDB);
1374 // CleanLocalStorage(fgkLocalRefStorage);
1375 // UpdateTableFailCase();
1376
1377 // Send mail to detector expert!
c88ad5db 1378 Log("SHUTTLE", Form("ContinueProcessing - Sending mail to %s expert...",
1379 fCurrentDetector.Data()));
9827400b 1380 if (!SendMail())
1381 Log("SHUTTLE", Form("ContinueProcessing - Could not send mail to %s expert",
1382 fCurrentDetector.Data()));
3301427a 1383
57c1a579 1384 } else {
1385 Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. "
1386 "Aborted before with %s. Retry number %d.", fCurrentDetector.Data(),
1387 status->GetStatusName(), status->GetCount()));
9827400b 1388 Bool_t increaseCount = kTRUE;
c88ad5db 1389 if (status->GetStatus() == AliShuttleStatus::kDCSError ||
1390 status->GetStatus() == AliShuttleStatus::kDCSStarted)
1391 increaseCount = kFALSE;
675f64cd 1392
9827400b 1393 UpdateShuttleStatus(AliShuttleStatus::kStarted, increaseCount);
57c1a579 1394 cont = kTRUE;
2bb7b766 1395 }
1396
57c1a579 1397 return cont;
5164a766 1398}
1399
1400//______________________________________________________________________________________________
2bb7b766 1401Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
58bc3020 1402{
73abe331 1403 //
b948db8d 1404 // Makes data retrieval for all detectors in the configuration.
2bb7b766 1405 // entry: Shuttle logbook entry, contains run paramenters and status of detectors
1406 // (Unprocessed, Inactive, Failed or Done).
d477ad88 1407 // Returns kFALSE in case of error occured and kTRUE otherwise
73abe331 1408 //
1409
9827400b 1410 if (!entry) return kFALSE;
2bb7b766 1411
1412 fLogbookEntry = entry;
1413
c88ad5db 1414 Log("SHUTTLE", Form("\t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^*",
9827400b 1415 GetCurrentRun()));
2bb7b766 1416
e7f62f16 1417 // create ML instance that monitors this run
468df1ce 1418 fMonaLisa = new TMonaLisaWriter(fConfig->GetMonitorHost(), fConfig->GetMonitorTable(), Form("%d", GetCurrentRun()));
2bb7b766 1419
e7f62f16 1420 // Send the information to ML
1421 TMonaLisaText mlStatus("SHUTTLE_status", "Processing");
9827400b 1422 TMonaLisaText mlRunType("SHUTTLE_runtype", Form("%s (%s)", entry->GetRunType(), entry->GetRunParameter("log")));
e7f62f16 1423
1424 TList mlList;
1425 mlList.Add(&mlStatus);
9827400b 1426 mlList.Add(&mlRunType);
e7f62f16 1427
1428 fMonaLisa->SendParameters(&mlList);
3301427a 1429
9827400b 1430 if (fLogbookEntry->IsDone())
1431 {
1432 Log("SHUTTLE","Process - Shuttle is already DONE. Updating logbook");
1433 UpdateShuttleLogbook("shuttle_done");
1434 fLogbookEntry = 0;
1435 return kTRUE;
1436 }
1437
1438 // read test mode if flag is set
1439 if (fReadTestMode)
1440 {
3d8bc902 1441 fTestMode = kNone;
9827400b 1442 TString logEntry(entry->GetRunParameter("log"));
1443 //printf("log entry = %s\n", logEntry.Data());
1444 TString searchStr("Testmode: ");
1445 Int_t pos = logEntry.Index(searchStr.Data());
1446 //printf("%d\n", pos);
1447 if (pos >= 0)
1448 {
1449 TSubString subStr = logEntry(pos + searchStr.Length(), logEntry.Length());
1450 //printf("%s\n", subStr.String().Data());
1451 TString newStr(subStr.Data());
1452 TObjArray* token = newStr.Tokenize(' ');
1453 if (token)
1454 {
1455 //token->Print();
1456 TObjString* tmpStr = dynamic_cast<TObjString*> (token->First());
1457 if (tmpStr)
1458 {
1459 Int_t testMode = tmpStr->String().Atoi();
1460 if (testMode > 0)
1461 {
c88ad5db 1462 Log("SHUTTLE", Form("Process - Enabling test mode %d", testMode));
9827400b 1463 SetTestMode((TestMode) testMode);
1464 }
1465 }
1466 delete token;
1467 }
1468 }
1469 }
c88ad5db 1470
eba76848 1471 fLogbookEntry->Print("all");
57f50b3c 1472
1473 // Initialization
d477ad88 1474 Bool_t hasError = kFALSE;
5164a766 1475
675f64cd 1476 // Set the CDB and Reference folders according to the year and LHC period
1477 TString lhcPeriod(GetLHCPeriod());
1478 if (lhcPeriod.Length() == 0)
1479 {
7d4cf768 1480 Log("SHUTTLE","Process - LHCPeriod not found in logbook!");
1481 return 0;
675f64cd 1482 }
1483
1484 if (fgkMainCDB.Length() == 0)
1485 fgkMainCDB = Form("alien://folder=/alice/data/%d/%s/OCDB?user=alidaq?cacheFold=/tmp/OCDBCache",
1486 GetCurrentYear(), lhcPeriod.Data());
1487
1488 if (fgkMainRefStorage.Length() == 0)
1489 fgkMainRefStorage = Form("alien://folder=/alice/data/%d/%s/Reference?user=alidaq?cacheFold=/tmp/OCDBCache",
1490 GetCurrentYear(), lhcPeriod.Data());
1491
2bb7b766 1492 AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
1493 if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
1494 AliCDBStorage *mainRefSto = AliCDBManager::Instance()->GetStorage(fgkMainRefStorage);
1495 if(mainRefSto) mainRefSto->QueryCDB(GetCurrentRun());
d477ad88 1496
57f50b3c 1497 // Loop on detectors in the configuration
b948db8d 1498 TIter iter(fConfig->GetDetectors());
2bb7b766 1499 TObjString* aDetector = 0;
b948db8d 1500
be48e3ea 1501 while ((aDetector = (TObjString*) iter.Next()))
1502 {
7bfb2090 1503 fCurrentDetector = aDetector->String();
5164a766 1504
9e080f92 1505 if (ContinueProcessing() == kFALSE) continue;
1506
c88ad5db 1507 Log("SHUTTLE", Form("\t\t\t****** run %d - %s: START ******",
2bb7b766 1508 GetCurrentRun(), aDetector->GetName()));
1509
9d733021 1510 for(Int_t iSys=0;iSys<3;iSys++) fFXSCalled[iSys]=kFALSE;
1511
c88ad5db 1512 Log(fCurrentDetector.Data(), "Process - Starting processing");
85a80aa9 1513
be48e3ea 1514 Int_t pid = fork();
1515
1516 if (pid < 0)
1517 {
c88ad5db 1518 Log("SHUTTLE", "Process - ERROR: Forking failed");
be48e3ea 1519 }
1520 else if (pid > 0)
1521 {
1522 // parent
c88ad5db 1523 Log("SHUTTLE", Form("Process - In parent process of %d - %s: Starting monitoring",
be48e3ea 1524 GetCurrentRun(), aDetector->GetName()));
1525
1526 Long_t begin = time(0);
1527
1528 int status; // to be used with waitpid, on purpose an int (not Int_t)!
1529 while (waitpid(pid, &status, WNOHANG) == 0)
1530 {
1531 Long_t expiredTime = time(0) - begin;
1532
1533 if (expiredTime > fConfig->GetPPTimeOut())
1534 {
9827400b 1535 TString tmp;
c88ad5db 1536 tmp.Form("Process - Process of %s time out. "
1537 "Run time: %d seconds. Killing...",
1538 fCurrentDetector.Data(), expiredTime);
9827400b 1539 Log("SHUTTLE", tmp);
1540 Log(fCurrentDetector, tmp);
be48e3ea 1541
1542 kill(pid, 9);
1543
3301427a 1544 UpdateShuttleStatus(AliShuttleStatus::kPPTimeOut);
be48e3ea 1545 hasError = kTRUE;
1546
1547 gSystem->Sleep(1000);
1548 }
1549 else
1550 {
be48e3ea 1551 gSystem->Sleep(1000);
9827400b 1552
1553 TString checkStr;
1554 checkStr.Form("ps -o vsize --pid %d | tail -n 1", pid);
1555 FILE* pipe = gSystem->OpenPipe(checkStr, "r");
1556 if (!pipe)
1557 {
c88ad5db 1558 Log("SHUTTLE", Form("Process - Error: "
1559 "Could not open pipe to %s", checkStr.Data()));
9827400b 1560 continue;
1561 }
1562
1563 char buffer[100];
1564 if (!fgets(buffer, 100, pipe))
1565 {
c88ad5db 1566 Log("SHUTTLE", "Process - Error: ps did not return anything");
9827400b 1567 gSystem->ClosePipe(pipe);
1568 continue;
1569 }
1570 gSystem->ClosePipe(pipe);
1571
1572 //Log("SHUTTLE", Form("ps returned %s", buffer));
1573
1574 Int_t mem = 0;
1575 if ((sscanf(buffer, "%d\n", &mem) != 1) || !mem)
1576 {
c88ad5db 1577 Log("SHUTTLE", "Process - Error: Could not parse output of ps");
9827400b 1578 continue;
1579 }
1580
1581 if (expiredTime % 60 == 0)
c88ad5db 1582 Log("SHUTTLE", Form("Process - %s: Checking process. "
1583 "Run time: %d seconds - Memory consumption: %d KB",
1584 fCurrentDetector.Data(), expiredTime, mem));
9827400b 1585
1586 if (mem > fConfig->GetPPMaxMem())
1587 {
1588 TString tmp;
c88ad5db 1589 tmp.Form("Process - Process exceeds maximum allowed memory "
1590 "(%d KB > %d KB). Killing...",
9827400b 1591 mem, fConfig->GetPPMaxMem());
1592 Log("SHUTTLE", tmp);
1593 Log(fCurrentDetector, tmp);
1594
1595 kill(pid, 9);
1596
1597 UpdateShuttleStatus(AliShuttleStatus::kPPOutOfMemory);
1598 hasError = kTRUE;
1599
1600 gSystem->Sleep(1000);
1601 }
be48e3ea 1602 }
1603 }
1604
c88ad5db 1605 Log("SHUTTLE", Form("Process - In parent process of %d - %s: Client has terminated.",
be48e3ea 1606 GetCurrentRun(), aDetector->GetName()));
1607
1608 if (WIFEXITED(status))
1609 {
1610 Int_t returnCode = WEXITSTATUS(status);
1611
c88ad5db 1612 Log("SHUTTLE", Form("Process - %s: the return code is %d", fCurrentDetector.Data(),
3301427a 1613 returnCode));
be48e3ea 1614
9827400b 1615 if (returnCode == 0) hasError = kTRUE;
be48e3ea 1616 }
1617 }
1618 else if (pid == 0)
1619 {
1620 // client
c88ad5db 1621 Log("SHUTTLE", Form("Process - In client process of %d - %s", GetCurrentRun(),
1622 aDetector->GetName()));
be48e3ea 1623
c88ad5db 1624 Log("SHUTTLE", Form("Process - Redirecting output to %s log",fCurrentDetector.Data()));
ffa29e93 1625
546242fb 1626 if ((freopen(GetLogFileName(fCurrentDetector), "a", stdout)) == 0)
ffa29e93 1627 {
c88ad5db 1628 Log("SHUTTLE", "Process - Could not freopen stdout");
ffa29e93 1629 }
1630 else
1631 {
1632 fOutputRedirected = kTRUE;
1633 if ((dup2(fileno(stdout), fileno(stderr))) < 0)
c88ad5db 1634 Log("SHUTTLE", "Process - Could not redirect stderr");
ffa29e93 1635
1636 }
1637
5bac2bde 1638 TString wd = gSystem->WorkingDirectory();
675f64cd 1639 TString tmpDir = Form("%s/%s_%d_process", GetShuttleTempDir(),
1640 fCurrentDetector.Data(), GetCurrentRun());
5bac2bde 1641
d524ade6 1642 Int_t result = gSystem->GetPathInfo(tmpDir.Data(), 0, (Long64_t*) 0, 0, 0);
1643 if (!result) // temp dir already exists!
1644 {
1645 Log(fCurrentDetector.Data(),
1646 Form("Process - %s dir already exists! Removing...", tmpDir.Data()));
1647 gSystem->Exec(Form("rm -rf %s",tmpDir.Data()));
675f64cd 1648 }
1649
1650 if (gSystem->mkdir(tmpDir.Data(), 1))
1651 {
1652 Log(fCurrentDetector.Data(), "Process - could not make temp directory!!");
1653 gSystem->Exit(1);
d524ade6 1654 }
1655
1656 if (!gSystem->ChangeDirectory(tmpDir.Data()))
1657 {
1658 Log(fCurrentDetector.Data(), "Process - could not change directory!!");
1659 gSystem->Exit(1);
1660 }
5bac2bde 1661
9827400b 1662 Bool_t success = ProcessCurrentDetector();
5bac2bde 1663
1664 gSystem->ChangeDirectory(wd.Data());
b1d18693 1665
9827400b 1666 if (success) // Preprocessor finished successfully!
1667 {
b1d18693 1668 // remove temporary folder
1669 gSystem->Exec(Form("rm -rf %s",tmpDir.Data()));
1670
3301427a 1671 // Update time_processed field in FXS DB
1672 if (UpdateTable() == kFALSE)
5bac2bde 1673 Log("SHUTTLE", Form("Process - %s: Could not update FXS databases!",
1674 fCurrentDetector.Data()));
3301427a 1675
1676 // Transfer the data from local storage to main storage (Grid)
1677 UpdateShuttleStatus(AliShuttleStatus::kStoreStarted);
1678 if (StoreOCDB() == kFALSE)
1679 {
c88ad5db 1680 Log("SHUTTLE",
1681 Form("\t\t\t****** run %d - %s: STORAGE ERROR ******",
3301427a 1682 GetCurrentRun(), aDetector->GetName()));
1683 UpdateShuttleStatus(AliShuttleStatus::kStoreError);
9827400b 1684 success = kFALSE;
3301427a 1685 } else {
c88ad5db 1686 Log("SHUTTLE",
1687 Form("\t\t\t****** run %d - %s: DONE ******",
3301427a 1688 GetCurrentRun(), aDetector->GetName()));
1689 UpdateShuttleStatus(AliShuttleStatus::kDone);
9827400b 1690 UpdateShuttleLogbook(fCurrentDetector, "DONE");
3301427a 1691 }
c88ad5db 1692 } else
1693 {
1694 Log("SHUTTLE",
1695 Form("\t\t\t****** run %d - %s: PP ERROR ******",
1696 GetCurrentRun(), aDetector->GetName()));
be48e3ea 1697 }
1698
4b95672b 1699 for (UInt_t iSys=0; iSys<3; iSys++)
1700 {
1701 if (fFXSCalled[iSys]) fFXSlist[iSys].Clear();
1702 }
1703
c88ad5db 1704 Log("SHUTTLE", Form("Process - Client process of %d - %s is exiting now with %d.",
9827400b 1705 GetCurrentRun(), aDetector->GetName(), success));
be48e3ea 1706
1707 // the client exits here
9827400b 1708 gSystem->Exit(success);
be48e3ea 1709
1710 AliError("We should never get here!!!");
1711 }
7bfb2090 1712 }
5164a766 1713
c88ad5db 1714 Log("SHUTTLE", Form("\t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^*",
2bb7b766 1715 GetCurrentRun()));
1716
1717 //check if shuttle is done for this run, if so update logbook
1718 TObjArray checkEntryArray;
1719 checkEntryArray.SetOwner(1);
9e080f92 1720 TString whereClause = Form("where run=%d", GetCurrentRun());
b0e53b15 1721 if (!QueryShuttleLogbook(whereClause.Data(), checkEntryArray) ||
1722 checkEntryArray.GetEntries() == 0) {
9e080f92 1723 Log("SHUTTLE", Form("Process - Warning: Cannot check status of run %d on Shuttle logbook!",
1724 GetCurrentRun()));
1725 return hasError == kFALSE;
1726 }
b948db8d 1727
9e080f92 1728 AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
1729 (checkEntryArray.At(0));
2bb7b766 1730
9e080f92 1731 if (checkEntry)
1732 {
1733 if (checkEntry->IsDone())
be48e3ea 1734 {
9e080f92 1735 Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
1736 UpdateShuttleLogbook("shuttle_done");
1737 }
1738 else
1739 {
1740 for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
be48e3ea 1741 {
9e080f92 1742 if (checkEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
be48e3ea 1743 {
9e080f92 1744 AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
1745 checkEntry->GetRun(), GetDetName(iDet)));
1746 fFirstUnprocessed[iDet] = kFALSE;
be48e3ea 1747 }
1748 }
2bb7b766 1749 }
1750 }
1751
e7f62f16 1752 // remove ML instance
1753 delete fMonaLisa;
1754 fMonaLisa = 0;
1755
2bb7b766 1756 fLogbookEntry = 0;
85a80aa9 1757
a7160fe9 1758 return hasError == kFALSE;
73abe331 1759}
1760
b948db8d 1761//______________________________________________________________________________________________
9827400b 1762Bool_t AliShuttle::ProcessCurrentDetector()
73abe331 1763{
1764 //
2bb7b766 1765 // Makes data retrieval just for a specific detector (fCurrentDetector).
73abe331 1766 // Threre should be a configuration for this detector.
73abe331 1767
1d172743 1768 Log("SHUTTLE", Form("ProcessCurrentDetector - Retrieving values for %s, run %d",
1769 fCurrentDetector.Data(), GetCurrentRun()));
73abe331 1770
d524ade6 1771 TString wd = gSystem->WorkingDirectory();
1772
2d9019b4 1773 if (!CleanReferenceStorage(fCurrentDetector.Data()))
546242fb 1774 return kFALSE;
d524ade6 1775
1776 gSystem->ChangeDirectory(wd.Data());
1777
1d172743 1778 TMap* dcsMap = new TMap();
3301427a 1779
1780 // call preprocessor
1781 AliPreprocessor* aPreprocessor =
1782 dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
1783
1784 aPreprocessor->Initialize(GetCurrentRun(), GetCurrentStartTime(), GetCurrentEndTime());
1785
1786 Bool_t processDCS = aPreprocessor->ProcessDCS();
d477ad88 1787
651fdaab 1788 if (!processDCS)
1789 {
1d172743 1790 Log(fCurrentDetector, "ProcessCurrentDetector -"
1791 " The preprocessor requested to skip the retrieval of DCS values");
651fdaab 1792 }
8b739301 1793 else if (fTestMode & kSkipDCS)
2c15234c 1794 {
1d172743 1795 Log(fCurrentDetector, "ProcessCurrentDetector - In TESTMODE: Skipping DCS processing");
9827400b 1796 }
1797 else if (fTestMode & kErrorDCS)
1798 {
1d172743 1799 Log(fCurrentDetector, "ProcessCurrentDetector - In TESTMODE: Simulating DCS error");
3d8bc902 1800 UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
9827400b 1801 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
1d172743 1802 delete dcsMap;
9827400b 1803 return kFALSE;
2c15234c 1804 } else {
3301427a 1805
1806 UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
1807
1d172743 1808 // Query DCS archive
1809 Int_t nServers = fConfig->GetNServers(fCurrentDetector);
a038aa70 1810
1d172743 1811 for (int iServ=0; iServ<nServers; iServ++)
2c15234c 1812 {
1d172743 1813
1814 TString host(fConfig->GetDCSHost(fCurrentDetector, iServ));
1815 Int_t port = fConfig->GetDCSPort(fCurrentDetector, iServ);
542b6cc8 1816 Int_t multiSplit = fConfig->GetMultiSplit(fCurrentDetector, iServ);
1817
1790d4b7 1818 Log(fCurrentDetector, Form("ProcessCurrentDetector -"
1819 " Querying DCS Amanda server %s:%d (%d of %d)",
1820 host.Data(), port, iServ+1, nServers));
1d172743 1821
1822 TMap* aliasMap = 0;
1823 TMap* dpMap = 0;
1824
1825 if (fConfig->GetDCSAliases(fCurrentDetector, iServ)->GetEntries() > 0)
2c15234c 1826 {
1d172743 1827 aliasMap = GetValueSet(host, port,
542b6cc8 1828 fConfig->GetDCSAliases(fCurrentDetector, iServ),
1829 kAlias, multiSplit);
1d172743 1830 if (!aliasMap)
1831 {
1832 Log(fCurrentDetector,
1833 Form("ProcessCurrentDetector -"
675f64cd 1834 " Error retrieving DCS aliases from server %s."
1835 " Sending mail to DCS experts!", host.Data()));
1d172743 1836 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
675f64cd 1837
1838 if (!SendMailToDCS())
1839 Log("SHUTTLE", Form("ProcessCurrentDetector - Could not send mail to DCS experts!"));
1840
a038aa70 1841 delete dcsMap;
1d172743 1842 return kFALSE;
1843 }
2c15234c 1844 }
a038aa70 1845
1d172743 1846 if (fConfig->GetDCSDataPoints(fCurrentDetector, iServ)->GetEntries() > 0)
a038aa70 1847 {
1d172743 1848 dpMap = GetValueSet(host, port,
542b6cc8 1849 fConfig->GetDCSDataPoints(fCurrentDetector, iServ),
1850 kDP, multiSplit);
1d172743 1851 if (!dpMap)
1852 {
1853 Log(fCurrentDetector,
1854 Form("ProcessCurrentDetector -"
675f64cd 1855 " Error retrieving DCS data points from server %s."
1856 " Sending mail to DCS experts!", host.Data()));
1d172743 1857 UpdateShuttleStatus(AliShuttleStatus::kDCSError);
675f64cd 1858
1859 if (!SendMailToDCS())
1860 Log("SHUTTLE", Form("ProcessCurrentDetector - Could not send mail to DCS experts!"));
1861
1d172743 1862 if (aliasMap) delete aliasMap;
1863 delete dcsMap;
1864 return kFALSE;
1865 }
a038aa70 1866 }
1d172743 1867
1868 // merge aliasMap and dpMap into dcsMap
1869 if(aliasMap) {
1870 TIter iter(aliasMap);
a038aa70 1871 TObjString* key = 0;
1872 while ((key = (TObjString*) iter.Next()))
1d172743 1873 dcsMap->Add(key, aliasMap->GetValue(key->String()));
1874
1875 aliasMap->SetOwner(kFALSE);
1876 delete aliasMap;
1877 }
1878
1879 if(dpMap) {
1880 TIter iter(dpMap);
1881 TObjString* key = 0;
1882 while ((key = (TObjString*) iter.Next()))
1883 dcsMap->Add(key, dpMap->GetValue(key->String()));
1884
1885 dpMap->SetOwner(kFALSE);
1886 delete dpMap;
a038aa70 1887 }
73abe331 1888 }
1889 }
dc25836b 1890
b1d18693 1891 // save map into file, to help debugging in case of preprocessor error
1892 TFile* f = TFile::Open("DCSMap.root","recreate");
1893 f->cd();
1894 dcsMap->Write("DCSMap", TObject::kSingleKey);
1895 f->Close();
1896 delete f;
1897
2bb7b766 1898 // DCS Archive DB processing successful. Call Preprocessor!
85a80aa9 1899 UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
a7160fe9 1900
a038aa70 1901 UInt_t returnValue = aPreprocessor->Process(dcsMap);
b948db8d 1902
3301427a 1903 if (returnValue > 0) // Preprocessor error!
1904 {
c88ad5db 1905 Log(fCurrentDetector, Form("ProcessCurrentDetector - "
1906 "Preprocessor failed. Process returned %d.", returnValue));
cb343cfd 1907 UpdateShuttleStatus(AliShuttleStatus::kPPError);
a038aa70 1908 dcsMap->DeleteAll();
1909 delete dcsMap;
9827400b 1910 return kFALSE;
1911 }
1912
1913 // preprocessor ok!
1914 UpdateShuttleStatus(AliShuttleStatus::kPPDone);
1915 Log(fCurrentDetector, Form("ProcessCurrentDetector - %s preprocessor returned success",
1916 fCurrentDetector.Data()));
b948db8d 1917
a038aa70 1918 dcsMap->DeleteAll();
1919 delete dcsMap;
b948db8d 1920
9827400b 1921 return kTRUE;
2bb7b766 1922}
1923
1924//______________________________________________________________________________________________
1925Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
1926 TObjArray& entries)
1927{
9827400b 1928 // Query DAQ's Shuttle logbook and fills detector status object.
1929 // Call QueryRunParameters to query DAQ logbook for run parameters.
1930 //
2bb7b766 1931
fc5a4708 1932 entries.SetOwner(1);
1933
2bb7b766 1934 // check connection, in case connect
be48e3ea 1935 if(!Connect(3)) return kFALSE;
2bb7b766 1936
1937 TString sqlQuery;
441b0e9c 1938 sqlQuery = Form("select * from %s %s order by run", fConfig->GetShuttlelbTable(), whereClause);
2bb7b766 1939
be48e3ea 1940 TSQLResult* aResult = fServer[3]->Query(sqlQuery);
2bb7b766 1941 if (!aResult) {
1942 AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
1943 return kFALSE;
1944 }
1945
fc5a4708 1946 AliDebug(2,Form("Query = %s", sqlQuery.Data()));
1947
2bb7b766 1948 if(aResult->GetRowCount() == 0) {
c88ad5db 1949 Log("SHUTTLE", "No entries in Shuttle Logbook match request");
9827400b 1950 delete aResult;
1951 return kTRUE;
2bb7b766 1952 }
1953
1954 // TODO Check field count!
db99d43e 1955 const UInt_t nCols = 23;
2bb7b766 1956 if (aResult->GetFieldCount() != (Int_t) nCols) {
c88ad5db 1957 Log("SHUTTLE", "Invalid SQL result field number!");
2bb7b766 1958 delete aResult;
1959 return kFALSE;
1960 }
1961
2bb7b766 1962 TSQLRow* aRow;
1963 while ((aRow = aResult->Next())) {
1964 TString runString(aRow->GetField(0), aRow->GetFieldLength(0));
1965 Int_t run = runString.Atoi();
1966
eba76848 1967 AliShuttleLogbookEntry *entry = QueryRunParameters(run);
1968 if (!entry)
1969 continue;
2bb7b766 1970
1971 // loop on detectors
eba76848 1972 for(UInt_t ii = 0; ii < nCols; ii++)
1973 entry->SetDetectorStatus(aResult->GetFieldName(ii), aRow->GetField(ii));
2bb7b766 1974
eba76848 1975 entries.AddLast(entry);
2bb7b766 1976 delete aRow;
1977 }
1978
2bb7b766 1979 delete aResult;
1980 return kTRUE;
1981}
1982
1983//______________________________________________________________________________________________
eba76848 1984AliShuttleLogbookEntry* AliShuttle::QueryRunParameters(Int_t run)
2bb7b766 1985{
eba76848 1986 //
1987 // Retrieve run parameters written in the DAQ logbook and sets them into AliShuttleLogbookEntry object
1988 //
2bb7b766 1989
1990 // check connection, in case connect
be48e3ea 1991 if (!Connect(3))
eba76848 1992 return 0;
2bb7b766 1993
1994 TString sqlQuery;
2c15234c 1995 sqlQuery.Form("select * from %s where run=%d", fConfig->GetDAQlbTable(), run);
2bb7b766 1996
be48e3ea 1997 TSQLResult* aResult = fServer[3]->Query(sqlQuery);
2bb7b766 1998 if (!aResult) {
c88ad5db 1999 Log("SHUTTLE", Form("Can't execute query <%s>!", sqlQuery.Data()));
eba76848 2000 return 0;
2bb7b766 2001 }
2002
eba76848 2003 if (aResult->GetRowCount() == 0) {
2bb7b766 2004 Log("SHUTTLE", Form("QueryRunParameters - No entry in DAQ Logbook for run %d. Skipping", run));
2005 delete aResult;
eba76848 2006 return 0;
2bb7b766 2007 }
2008
eba76848 2009 if (aResult->GetRowCount() > 1) {
c88ad5db 2010 Log("SHUTTLE", Form("QueryRunParameters - UNEXPECTED: "
2011 "more than one entry in DAQ Logbook for run %d!", run));
2bb7b766 2012 delete aResult;
eba76848 2013 return 0;
2bb7b766 2014 }
2015
eba76848 2016 TSQLRow* aRow = aResult->Next();
2017 if (!aRow)
2018 {
c88ad5db 2019 Log("SHUTTLE", Form("QueryRunParameters - Could not retrieve row for run %d. Skipping", run));
eba76848 2020 delete aResult;
2021 return 0;
2022 }
2bb7b766 2023
eba76848 2024 AliShuttleLogbookEntry* entry = new AliShuttleLogbookEntry(run);
2bb7b766 2025
eba76848 2026 for (Int_t ii = 0; ii < aResult->GetFieldCount(); ii++)
2027 entry->SetRunParameter(aResult->GetFieldName(ii), aRow->GetField(ii));
2bb7b766 2028
eba76848 2029 UInt_t startTime = entry->GetStartTime();
2030 UInt_t endTime = entry->GetEndTime();
2031
1abfbb60 2032// if (!startTime || !endTime || startTime > endTime)
2033// {
2034// Log("SHUTTLE",
2035// Form("QueryRunParameters - Invalid parameters for Run %d: startTime = %d, endTime = %d. Skipping!",
2036// run, startTime, endTime));
2037//
2038// Log("SHUTTLE", Form("Marking SHUTTLE done for run %d", run));
2039// fLogbookEntry = entry;
2040// if (!UpdateShuttleLogbook("shuttle_done"))
2041// {
2042// AliError(Form("Could not update logbook for run %d !", run));
2043// }
2044// fLogbookEntry = 0;
2045//
2046// delete entry;
2047// delete aRow;
2048// delete aResult;
2049// return 0;
2050// }
2051
2052 if (!startTime)
b0e53b15 2053 {
2054 Log("SHUTTLE",
1abfbb60 2055 Form("QueryRunParameters - Invalid parameters for Run %d: "
2056 "startTime = %d, endTime = %d. Skipping!",
2057 run, startTime, endTime));
b0e53b15 2058
2059 Log("SHUTTLE", Form("Marking SHUTTLE done for run %d", run));
2060 fLogbookEntry = entry;
2061 if (!UpdateShuttleLogbook("shuttle_done"))
2062 {
2063 AliError(Form("Could not update logbook for run %d !", run));
2064 }
2065 fLogbookEntry = 0;
2066
2067 delete entry;
2068 delete aRow;
2069 delete aResult;
2070 return 0;
2071 }
2072
1abfbb60 2073 if (startTime && !endTime)
2074 {
2075 // TODO Here we don't mark SHUTTLE done, because this may mean
2076 //the run is still ongoing!!
2077 Log("SHUTTLE",
2078 Form("QueryRunParameters - Invalid parameters for Run %d: "
2079 "startTime = %d, endTime = %d. Skipping (Shuttle won't be marked as DONE)!",
2080 run, startTime, endTime));
2081
2082 //Log("SHUTTLE", Form("Marking SHUTTLE done for run %d", run));
2083 //fLogbookEntry = entry;
2084 //if (!UpdateShuttleLogbook("shuttle_done"))
2085 //{
2086 // AliError(Form("Could not update logbook for run %d !", run));
2087 //}
2088 //fLogbookEntry = 0;
2089
2090 delete entry;
2091 delete aRow;
2092 delete aResult;
2093 return 0;
2094 }
2095
2096 if (startTime && endTime && (startTime > endTime))
2097 {
2098 Log("SHUTTLE",
2099 Form("QueryRunParameters - Invalid parameters for Run %d: "
2100 "startTime = %d, endTime = %d. Skipping!",
2101 run, startTime, endTime));
2102
2103 Log("SHUTTLE", Form("Marking SHUTTLE done for run %d", run));
2104 fLogbookEntry = entry;
2105 if (!UpdateShuttleLogbook("shuttle_done"))
2106 {
2107 AliError(Form("Could not update logbook for run %d !", run));
2108 }
2109 fLogbookEntry = 0;
2110
2111 delete entry;
2112 delete aRow;
2113 delete aResult;
2114 return 0;
2115 }
2116
b0e53b15 2117 TString totEventsStr = entry->GetRunParameter("totalEvents");
2118 Int_t totEvents = totEventsStr.Atoi();
2119 if (totEvents < 1)
2120 {
eba76848 2121 Log("SHUTTLE",
b0e53b15 2122 Form("QueryRunParameters - Run %d has 0 events - Skipping!", run));
2123
2124 Log("SHUTTLE", Form("Marking SHUTTLE done for run %d", run));
2125 fLogbookEntry = entry;
2126 if (!UpdateShuttleLogbook("shuttle_done"))
2127 {
2128 AliError(Form("Could not update logbook for run %d !", run));
2129 }
2130 fLogbookEntry = 0;
2131
eba76848 2132 delete entry;
2bb7b766 2133 delete aRow;
eba76848 2134 delete aResult;
2135 return 0;
2bb7b766 2136 }
2137
eba76848 2138 delete aRow;
2bb7b766 2139 delete aResult;
eba76848 2140
2141 return entry;
2bb7b766 2142}
2143
a038aa70 2144//______________________________________________________________________________________________
2145TMap* AliShuttle::GetValueSet(const char* host, Int_t port, const TSeqCollection* entries,
542b6cc8 2146 DCSType type, Int_t multiSplit)
a038aa70 2147{
2148 // Retrieve all "entry" data points from the DCS server
2149 // host, port: TSocket connection parameters
2150 // entries: list of name of the alias or data point
2151 // type: kAlias or kDP
2152 // returns TMap of values, 0 when failure
542b6cc8 2153
2154 AliDCSClient client(host, port, fTimeout, fRetries, multiSplit);
b41b252a 2155
a038aa70 2156 TMap* result = 0;
b41b252a 2157 if (type == kAlias)
a038aa70 2158 {
b41b252a 2159 result = client.GetAliasValues(entries, GetCurrentStartTime(),
2160 GetCurrentEndTime());
2161 }
2162 else if (type == kDP)
2163 {
2164 result = client.GetDPValues(entries, GetCurrentStartTime(),
2165 GetCurrentEndTime());
2166 }
a038aa70 2167
b41b252a 2168 if (result == 0)
2169 {
2170 Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get entries! Reason: %s",
1790d4b7 2171 client.GetErrorString(client.GetResultErrorCode())));
2172 if (client.GetResultErrorCode() == AliDCSClient::fgkServerError)
2173 Log(fCurrentDetector.Data(), Form("GetValueSet - Server error code: %s",
2174 client.GetServerError().Data()));
a038aa70 2175
b41b252a 2176 return 0;
a038aa70 2177 }
b41b252a 2178
a038aa70 2179 return result;
2180}
b41b252a 2181
b948db8d 2182//______________________________________________________________________________________________
57f50b3c 2183const char* AliShuttle::GetFile(Int_t system, const char* detector,
2184 const char* id, const char* source)
b948db8d 2185{
9827400b 2186 // Get calibration file from file exchange servers
2187 // First queris the FXS database for the file name, using the run, detector, id and source info
2188 // then calls RetrieveFile(filename) for actual copy to local disk
2189 // run: current run being processed (given by Logbook entry fLogbookEntry)
2190 // detector: the Preprocessor name
2191 // id: provided as a parameter by the Preprocessor
2192 // source: provided by the Preprocessor through GetFileSources function
2193
2194 // check if test mode should simulate a FXS error
2195 if (fTestMode & kErrorFXSFiles)
2196 {
2197 Log(detector, Form("GetFile - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
2198 return 0;
2199 }
2200
57f50b3c 2201 // check connection, in case connect
9d733021 2202 if (!Connect(system))
eba76848 2203 {
9d733021 2204 Log(detector, Form("GetFile - Couldn't connect to %s FXS database", GetSystemName(system)));
57f50b3c 2205 return 0;
2206 }
2207
2208 // Query preparation
9d733021 2209 TString sourceName(source);
d386d623 2210 Int_t nFields = 3;
2211 TString sqlQueryStart = Form("select filePath,size,fileChecksum from %s where",
2212 fConfig->GetFXSdbTable(system));
2213 TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
2214 GetCurrentRun(), detector, id);
2215
9d733021 2216 if (system == kDAQ)
2217 {
d386d623 2218 whereClause += Form(" and DAQsource=\"%s\"", source);
57f50b3c 2219 }
9d733021 2220 else if (system == kDCS)
eba76848 2221 {
9d733021 2222 sourceName="none";
57f50b3c 2223 }
9d733021 2224 else if (system == kHLT)
9e080f92 2225 {
d386d623 2226 whereClause += Form(" and DDLnumbers=\"%s\"", source);
9d733021 2227 nFields = 3;
9e080f92 2228 }
2229
9e080f92 2230 TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
2231
2232 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2233
2234 // Query execution
2235 TSQLResult* aResult = 0;
9d733021 2236 aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
9e080f92 2237 if (!aResult) {
9d733021 2238 Log(detector, Form("GetFileName - Can't execute SQL query to %s database for: id = %s, source = %s",
2239 GetSystemName(system), id, sourceName.Data()));
9e080f92 2240 return 0;
2241 }
2242
2243 if(aResult->GetRowCount() == 0)
2244 {
2245 Log(detector,
9d733021 2246 Form("GetFileName - No entry in %s FXS db for: id = %s, source = %s",
2247 GetSystemName(system), id, sourceName.Data()));
9e080f92 2248 delete aResult;
2249 return 0;
2250 }
2bb7b766 2251
9e080f92 2252 if (aResult->GetRowCount() > 1) {
2253 Log(detector,
9d733021 2254 Form("GetFileName - More than one entry in %s FXS db for: id = %s, source = %s",
2255 GetSystemName(system), id, sourceName.Data()));
9e080f92 2256 delete aResult;
2257 return 0;
2258 }
2259
9d733021 2260 if (aResult->GetFieldCount() != nFields) {
9e080f92 2261 Log(detector,
9d733021 2262 Form("GetFileName - Wrong field count in %s FXS db for: id = %s, source = %s",
2263 GetSystemName(system), id, sourceName.Data()));
9e080f92 2264 delete aResult;
2265 return 0;
2266 }
2267
2268 TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
2269
2270 if (!aRow){
9d733021 2271 Log(detector, Form("GetFileName - Empty set result in %s FXS db from query: id = %s, source = %s",
2272 GetSystemName(system), id, sourceName.Data()));
9e080f92 2273 delete aResult;
2274 return 0;
2275 }
2276
2277 TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
2278 TString fileSize(aRow->GetField(1), aRow->GetFieldLength(1));
d386d623 2279 TString fileChecksum(aRow->GetField(2), aRow->GetFieldLength(2));
9e080f92 2280
2281 delete aResult;
2282 delete aRow;
2283
d386d623 2284 AliDebug(2, Form("filePath = %s; size = %s, fileChecksum = %s",
2285 filePath.Data(), fileSize.Data(), fileChecksum.Data()));
9e080f92 2286
9e080f92 2287 // retrieved file is renamed to make it unique
675f64cd 2288 TString localFileName = Form("%s/%s_%d_process/%s_%s_%d_%s_%s.shuttle",
2289 GetShuttleTempDir(), detector, GetCurrentRun(),
d524ade6 2290 GetSystemName(system), detector, GetCurrentRun(),
2291 id, sourceName.Data());
9d733021 2292
9e080f92 2293
9d733021 2294 // file retrieval from FXS
4b95672b 2295 UInt_t nRetries = 0;
2296 UInt_t maxRetries = 3;
2297 Bool_t result = kFALSE;
2298
2299 // copy!! if successful TSystem::Exec returns 0
2300 while(nRetries++ < maxRetries) {
2301 AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
2302 result = RetrieveFile(system, filePath.Data(), localFileName.Data());
2303 if(!result)
2304 {
2305 Log(detector, Form("GetFileName - Copy of file %s from %s FXS failed",
9d733021 2306 filePath.Data(), GetSystemName(system)));
4b95672b 2307 continue;
4f0749a8 2308 }
9e080f92 2309
d386d623 2310 if (fileChecksum.Length()>0)
4b95672b 2311 {
2312 // compare md5sum of local file with the one stored in the FXS DB
d524ade6 2313 Int_t md5Comp = gSystem->Exec(Form("md5sum %s |grep %s 2>&1 > /dev/null",
2314 localFileName.Data(), fileChecksum.Data()));
9e080f92 2315
4b95672b 2316 if (md5Comp != 0)
2317 {
2318 Log(detector, Form("GetFileName - md5sum of file %s does not match with local copy!",
2319 filePath.Data()));
2320 result = kFALSE;
2321 continue;
2322 }
d386d623 2323 } else {
2324 Log(fCurrentDetector, Form("GetFile - md5sum of file %s not set in %s database, skipping comparison",
2325 filePath.Data(), GetSystemName(system)));
9d733021 2326 }
4b95672b 2327 if (result) break;
9e080f92 2328 }
2329
4b95672b 2330 if(!result) return 0;
2331
9d733021 2332 fFXSCalled[system]=kTRUE;
2333 TObjString *fileParams = new TObjString(Form("%s#!?!#%s", id, sourceName.Data()));
2334 fFXSlist[system].Add(fileParams);
9e080f92 2335
675f64cd 2336 static TString staticLocalFileName;
2337 staticLocalFileName.Form("%s", localFileName.Data());
2338
c88ad5db 2339 Log(fCurrentDetector, Form("GetFile - Retrieved file with id %s and "
2340 "source %s from %s to %s", id, source,
d524ade6 2341 GetSystemName(system), localFileName.Data()));
675f64cd 2342
d524ade6 2343 return staticLocalFileName.Data();
2bb7b766 2344}
2345
2346//______________________________________________________________________________________________
9d733021 2347Bool_t AliShuttle::RetrieveFile(UInt_t system, const char* fxsFileName, const char* localFileName)
9e080f92 2348{
9827400b 2349 //
2350 // Copies file from FXS to local Shuttle machine
2351 //
2bb7b766 2352
9e080f92 2353 // check temp directory: trying to cd to temp; if it does not exist, create it
d524ade6 2354 AliDebug(2, Form("Copy file %s from %s FXS into %s",
2355 GetSystemName(system), fxsFileName, localFileName));
2356
2357 TString tmpDir(localFileName);
2358
2359 tmpDir = tmpDir(0,tmpDir.Last('/'));
9e080f92 2360
d524ade6 2361 Int_t noDir = gSystem->GetPathInfo(tmpDir.Data(), 0, (Long64_t*) 0, 0, 0);
2362 if (noDir) // temp dir does not exists!
2363 {
2364 if (gSystem->mkdir(tmpDir.Data(), 1))
2365 {
2366 Log(fCurrentDetector.Data(), "RetrieveFile - could not make temp directory!!");
9e080f92 2367 return kFALSE;
2368 }
9e080f92 2369 }
2370
9d733021 2371 TString baseFXSFolder;
2372 if (system == kDAQ)
2373 {
2374 baseFXSFolder = "FES/";
2375 }
2376 else if (system == kDCS)
2377 {
2378 baseFXSFolder = "";
2379 }
2380 else if (system == kHLT)
2381 {
42fde080 2382 baseFXSFolder = "/opt/FXS/";
9d733021 2383 }
2384
2385
d524ade6 2386 TString command = Form("scp -oPort=%d -2 %s@%s:%s%s %s",
9d733021 2387 fConfig->GetFXSPort(system),
2388 fConfig->GetFXSUser(system),
2389 fConfig->GetFXSHost(system),
2390 baseFXSFolder.Data(),
2391 fxsFileName,
9e080f92 2392 localFileName);
2393
2394 AliDebug(2, Form("%s",command.Data()));
2395
4b95672b 2396 Bool_t result = (gSystem->Exec(command.Data()) == 0);
9e080f92 2397
4b95672b 2398 return result;
9e080f92 2399}
2400
2401//______________________________________________________________________________________________
9d733021 2402TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
2403{
9827400b 2404 //
2405 // Get sources producing the condition file Id from file exchange servers
4a33bdd9 2406 // if id is NULL all sources are returned (distinct)
9827400b 2407 //
1bcd28db 2408
2409 Log(detector, Form("GetFileSources - Retrieving sources with id %s from %s", id, GetSystemName(system)));
9827400b 2410
2411 // check if test mode should simulate a FXS error
2412 if (fTestMode & kErrorFXSSources)
2413 {
2414 Log(detector, Form("GetFileSources - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
2415 return 0;
2416 }
2417
9d733021 2418 if (system == kDCS)
2419 {
c88ad5db 2420 Log(detector, "GetFileSources - WARNING: DCS system has only one source of data!");
6297b37d 2421 TList *list = new TList();
2422 list->SetOwner(1);
2423 list->Add(new TObjString(" "));
2424 return list;
9d733021 2425 }
9e080f92 2426
2427 // check connection, in case connect
9d733021 2428 if (!Connect(system))
2429 {
4a33bdd9 2430 Log(detector, Form("GetFileSources - Couldn't connect to %s FXS database", GetSystemName(system)));
9d733021 2431 return NULL;
9e080f92 2432 }
2433
9d733021 2434 TString sourceName = 0;
2435 if (system == kDAQ)
2436 {
2437 sourceName = "DAQsource";
2438 } else if (system == kHLT)
2439 {
2440 sourceName = "DDLnumbers";
2441 }
2442
4a33bdd9 2443 TString sqlQueryStart = Form("select distinct %s from %s where", sourceName.Data(), fConfig->GetFXSdbTable(system));
2444 TString whereClause = Form("run=%d and detector=\"%s\"",
2445 GetCurrentRun(), detector);
2446 if (id)
2447 whereClause += Form(" and fileId=\"%s\"", id);
9e080f92 2448 TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
2449
2450 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2451
2452 // Query execution
2453 TSQLResult* aResult;
9d733021 2454 aResult = fServer[system]->Query(sqlQuery);
9e080f92 2455 if (!aResult) {
9d733021 2456 Log(detector, Form("GetFileSources - Can't execute SQL query to %s database for id: %s",
2457 GetSystemName(system), id));
9e080f92 2458 return 0;
2459 }
2460
86aa42c3 2461 TList *list = new TList();
2462 list->SetOwner(1);
2463
9d733021 2464 if (aResult->GetRowCount() == 0)
2465 {
9e080f92 2466 Log(detector,
9d733021 2467 Form("GetFileSources - No entry in %s FXS table for id: %s", GetSystemName(system), id));
9e080f92 2468 delete aResult;
86aa42c3 2469 return list;
9e080f92 2470 }
2471
1bcd28db 2472 Log(detector, Form("GetFileSources - Found %d sources", aResult->GetRowCount()));
9e080f92 2473
1bcd28db 2474 TSQLRow* aRow;
9d733021 2475 while ((aRow = aResult->Next()))
2476 {
9e080f92 2477
9d733021 2478 TString source(aRow->GetField(0), aRow->GetFieldLength(0));
2479 AliDebug(2, Form("%s = %s", sourceName.Data(), source.Data()));
2480 list->Add(new TObjString(source));
9e080f92 2481 delete aRow;
2482 }
9d733021 2483
9e080f92 2484 delete aResult;
2485
2486 return list;
2bb7b766 2487}
2488
4a33bdd9 2489//______________________________________________________________________________________________
2490TList* AliShuttle::GetFileIDs(Int_t system, const char* detector, const char* source)
2491{
2492 //
2493 // Get all ids of condition files produced by a given source from file exchange servers
2494 //
2495
1bcd28db 2496 Log(detector, Form("GetFileIDs - Retrieving ids with source %s with %s", source, GetSystemName(system)));
2497
4a33bdd9 2498 // check if test mode should simulate a FXS error
2499 if (fTestMode & kErrorFXSSources)
2500 {
2501 Log(detector, Form("GetFileIDs - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
2502 return 0;
2503 }
2504
2505 // check connection, in case connect
2506 if (!Connect(system))
2507 {
2508 Log(detector, Form("GetFileIDs - Couldn't connect to %s FXS database", GetSystemName(system)));
2509 return NULL;
2510 }
2511
2512 TString sourceName = 0;
2513 if (system == kDAQ)
2514 {
2515 sourceName = "DAQsource";
2516 } else if (system == kHLT)
2517 {
2518 sourceName = "DDLnumbers";
2519 }
2520
2521 TString sqlQueryStart = Form("select fileId from %s where", fConfig->GetFXSdbTable(system));
2522 TString whereClause = Form("run=%d and detector=\"%s\"",
2523 GetCurrentRun(), detector);
2524 if (sourceName.Length() > 0 && source)
2525 whereClause += Form(" and %s=\"%s\"", sourceName.Data(), source);
2526 TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
2527
2528 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2529
2530 // Query execution
2531 TSQLResult* aResult;
2532 aResult = fServer[system]->Query(sqlQuery);
2533 if (!aResult) {
2534 Log(detector, Form("GetFileIDs - Can't execute SQL query to %s database for source: %s",
2535 GetSystemName(system), source));
2536 return 0;
2537 }
2538
2539 TList *list = new TList();
2540 list->SetOwner(1);
2541
2542 if (aResult->GetRowCount() == 0)
2543 {
2544 Log(detector,
2545 Form("GetFileIDs - No entry in %s FXS table for source: %s", GetSystemName(system), source));
2546 delete aResult;
2547 return list;
2548 }
2549
1bcd28db 2550 Log(detector, Form("GetFileIDs - Found %d ids", aResult->GetRowCount()));
2551
4a33bdd9 2552 TSQLRow* aRow;
2553
2554 while ((aRow = aResult->Next()))
2555 {
2556
2557 TString id(aRow->GetField(0), aRow->GetFieldLength(0));
2558 AliDebug(2, Form("fileId = %s", id.Data()));
2559 list->Add(new TObjString(id));
2560 delete aRow;
2561 }
2562
2563 delete aResult;
2564
2565 return list;
2566}
2567
2bb7b766 2568//______________________________________________________________________________________________
9d733021 2569Bool_t AliShuttle::Connect(Int_t system)
2bb7b766 2570{
9827400b 2571 // Connect to MySQL Server of the system's FXS MySQL databases
2572 // DAQ Logbook, Shuttle Logbook and DAQ FXS db are on the same host
2573 //
57f50b3c 2574
9d733021 2575 // check connection: if already connected return
2576 if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
57f50b3c 2577
9d733021 2578 TString dbHost, dbUser, dbPass, dbName;
57f50b3c 2579
9d733021 2580 if (system < 3) // FXS db servers
2581 {
2582 dbHost = Form("mysql://%s:%d", fConfig->GetFXSdbHost(system), fConfig->GetFXSdbPort(system));
2583 dbUser = fConfig->GetFXSdbUser(system);
2584 dbPass = fConfig->GetFXSdbPass(system);
2585 dbName = fConfig->GetFXSdbName(system);
2586 } else { // Run & Shuttle logbook servers
2587 // TODO Will the Shuttle logbook server be the same as the Run logbook server ???
2588 dbHost = Form("mysql://%s:%d", fConfig->GetDAQlbHost(), fConfig->GetDAQlbPort());
2589 dbUser = fConfig->GetDAQlbUser();
2590 dbPass = fConfig->GetDAQlbPass();
2591 dbName = fConfig->GetDAQlbDB();
2592 }
57f50b3c 2593
9d733021 2594 fServer[system] = TSQLServer::Connect(dbHost.Data(), dbUser.Data(), dbPass.Data());
2595 if (!fServer[system] || !fServer[system]->IsConnected()) {
2596 if(system < 3)
2597 {
2598 AliError(Form("Can't establish connection to FXS database for %s",
2599 AliShuttleInterface::GetSystemName(system)));
2600 } else {
2601 AliError("Can't establish connection to Run logbook.");
57f50b3c 2602 }
9d733021 2603 if(fServer[system]) delete fServer[system];
2604 return kFALSE;
2bb7b766 2605 }
57f50b3c 2606
9d733021 2607 // Get tables
2608 TSQLResult* aResult=0;
2609 switch(system){
2610 case kDAQ:
2611 aResult = fServer[kDAQ]->GetTables(dbName.Data());
2612 break;
2613 case kDCS:
2614 aResult = fServer[kDCS]->GetTables(dbName.Data());
2615 break;
2616 case kHLT:
2617 aResult = fServer[kHLT]->GetTables(dbName.Data());
2618 break;
2619 default:
2620 aResult = fServer[3]->GetTables(dbName.Data());
2621 break;
2622 }
2623
2624 delete aResult;
2bb7b766 2625 return kTRUE;
2626}
57f50b3c 2627
9e080f92 2628//______________________________________________________________________________________________
9d733021 2629Bool_t AliShuttle::UpdateTable()
9e080f92 2630{
9827400b 2631 //
2632 // Update FXS table filling time_processed field in all rows corresponding to current run and detector
2633 //
9e080f92 2634
9d733021 2635 Bool_t result = kTRUE;
9e080f92 2636
9d733021 2637 for (UInt_t system=0; system<3; system++)
2638 {
2639 if(!fFXSCalled[system]) continue;
9e080f92 2640
9d733021 2641 // check connection, in case connect
2642 if (!Connect(system))
2643 {
2644 Log(fCurrentDetector, Form("UpdateTable - Couldn't connect to %s FXS database", GetSystemName(system)));
2645 result = kFALSE;
2646 continue;
9e080f92 2647 }
9e080f92 2648
9d733021 2649 TTimeStamp now; // now
2650
2651 // Loop on FXS list entries
2652 TIter iter(&fFXSlist[system]);
2653 TObjString *aFXSentry=0;
2654 while ((aFXSentry = dynamic_cast<TObjString*> (iter.Next())))
2655 {
2656 TString aFXSentrystr = aFXSentry->String();
2657 TObjArray *aFXSarray = aFXSentrystr.Tokenize("#!?!#");
2658 if (!aFXSarray || aFXSarray->GetEntries() != 2 )
2659 {
2660 Log(fCurrentDetector, Form("UpdateTable - error updating %s FXS entry. Check string: <%s>",
2661 GetSystemName(system), aFXSentrystr.Data()));
2662 if(aFXSarray) delete aFXSarray;
2663 result = kFALSE;
2664 continue;
2665 }
2666 const char* fileId = ((TObjString*) aFXSarray->At(0))->GetName();
2667 const char* source = ((TObjString*) aFXSarray->At(1))->GetName();
2668
2669 TString whereClause;
2670 if (system == kDAQ)
2671 {
2672 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
2673 GetCurrentRun(), fCurrentDetector.Data(), fileId, source);
2674 }
2675 else if (system == kDCS)
2676 {
2677 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\";",
2678 GetCurrentRun(), fCurrentDetector.Data(), fileId);
2679 }
2680 else if (system == kHLT)
2681 {
2682 whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DDLnumbers=\"%s\";",
2683 GetCurrentRun(), fCurrentDetector.Data(), fileId, source);
2684 }
2685
2686 delete aFXSarray;
9e080f92 2687
9d733021 2688 TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(system),
2689 now.GetSec(), whereClause.Data());
9e080f92 2690
9d733021 2691 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
9e080f92 2692
9d733021 2693 // Query execution
2694 TSQLResult* aResult;
2695 aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
2696 if (!aResult)
2697 {
2698 Log(fCurrentDetector, Form("UpdateTable - %s db: can't execute SQL query <%s>",
2699 GetSystemName(system), sqlQuery.Data()));
2700 result = kFALSE;
2701 continue;
2702 }
2703 delete aResult;
9e080f92 2704 }
9e080f92 2705 }
2706
9d733021 2707 return result;
9e080f92 2708}
57f50b3c 2709
3301427a 2710//______________________________________________________________________________________________
2711Bool_t AliShuttle::UpdateTableFailCase()
2712{
9827400b 2713 // Update FXS table filling time_processed field in all rows corresponding to current run and detector
2714 // this is called in case the preprocessor is declared failed for the current run, because
2715 // the fields are updated only in case of success
3301427a 2716
2717 Bool_t result = kTRUE;
2718
2719 for (UInt_t system=0; system<3; system++)
2720 {
2721 // check connection, in case connect
2722 if (!Connect(system))
2723 {
2724 Log(fCurrentDetector, Form("UpdateTableFailCase - Couldn't connect to %s FXS database",
2725 GetSystemName(system)));
2726 result = kFALSE;
2727 continue;
2728 }
2729
2730 TTimeStamp now; // now
2731
2732 // Loop on FXS list entries
2733
2734 TString whereClause = Form("where run=%d and detector=\"%s\";",
2735 GetCurrentRun(), fCurrentDetector.Data());
2736
2737
2738 TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(system),
2739 now.GetSec(), whereClause.Data());
2740
2741 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2742
2743 // Query execution
2744 TSQLResult* aResult;
2745 aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
2746 if (!aResult)
2747 {
2748 Log(fCurrentDetector, Form("UpdateTableFailCase - %s db: can't execute SQL query <%s>",
2749 GetSystemName(system), sqlQuery.Data()));
2750 result = kFALSE;
2751 continue;
2752 }
2753 delete aResult;
2754 }
2755
2756 return result;
2757}
2758
2bb7b766 2759//______________________________________________________________________________________________
2760Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status)
2761{
e7f62f16 2762 //
2763 // Update Shuttle logbook filling detector or shuttle_done column
2764 // ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
2765 //
57f50b3c 2766
2bb7b766 2767 // check connection, in case connect
be48e3ea 2768 if(!Connect(3)){
2bb7b766 2769 Log("SHUTTLE", "UpdateShuttleLogbook - Couldn't connect to DAQ Logbook.");
2770 return kFALSE;
57f50b3c 2771 }
2772
2bb7b766 2773 TString detName(detector);
2774 TString setClause;
e7f62f16 2775 if(detName == "shuttle_done")
2776 {
2bb7b766 2777 setClause = "set shuttle_done=1";
e7f62f16 2778
b0e53b15 2779 if (fMonaLisa)
2780 {
2781 // Send the information to ML
2782 TMonaLisaText mlStatus("SHUTTLE_status", "Done");
e7f62f16 2783
b0e53b15 2784 TList mlList;
2785 mlList.Add(&mlStatus);
2786
2787 fMonaLisa->SendParameters(&mlList);
2788 }
2bb7b766 2789 } else {
2bb7b766 2790 TString statusStr(status);
2791 if(statusStr.Contains("done", TString::kIgnoreCase) ||
2792 statusStr.Contains("failed", TString::kIgnoreCase)){
eba76848 2793 setClause = Form("set %s=\"%s\"", detector, status);
2bb7b766 2794 } else {
2795 Log("SHUTTLE",
2796 Form("UpdateShuttleLogbook - Invalid status <%s> for detector %s",
2797 status, detector));
2798 return kFALSE;
2799 }
2800 }
57f50b3c 2801
2bb7b766 2802 TString whereClause = Form("where run=%d", GetCurrentRun());
2803
441b0e9c 2804 TString sqlQuery = Form("update %s %s %s",
2805 fConfig->GetShuttlelbTable(), setClause.Data(), whereClause.Data());
57f50b3c 2806
2bb7b766 2807 AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
2808
2809 // Query execution
2810 TSQLResult* aResult;
be48e3ea 2811 aResult = dynamic_cast<TSQLResult*> (fServer[3]->Query(sqlQuery));
2bb7b766 2812 if (!aResult) {
2813 Log("SHUTTLE", Form("UpdateShuttleLogbook - Can't execute query <%s>", sqlQuery.Data()));
2814 return kFALSE;
57f50b3c 2815 }
2bb7b766 2816 delete aResult;
57f50b3c 2817
2818 return kTRUE;
2819}
2820
2821//______________________________________________________________________________________________
2bb7b766 2822Int_t AliShuttle::GetCurrentRun() const
2823{
9827400b 2824 //
2825 // Get current run from logbook entry
2826 //
57f50b3c 2827
2bb7b766 2828 return fLogbookEntry ? fLogbookEntry->GetRun() : -1;
57f50b3c 2829}
2830
2831//______________________________________________________________________________________________
2bb7b766 2832UInt_t AliShuttle::GetCurrentStartTime() const
2833{
9827400b 2834 //
2835 // get current start time
2836 //
57f50b3c 2837
2bb7b766 2838 return fLogbookEntry ? fLogbookEntry->GetStartTime() : 0;
57f50b3c 2839}
2840
2841//______________________________________________________________________________________________
2bb7b766 2842UInt_t AliShuttle::GetCurrentEndTime() const
2843{
9827400b 2844 //
2845 // get current end time from logbook entry
2846 //
57f50b3c 2847
2bb7b766 2848 return fLogbookEntry ? fLogbookEntry->GetEndTime() : 0;
57f50b3c 2849}
2850
675f64cd 2851//______________________________________________________________________________________________
2852UInt_t AliShuttle::GetCurrentYear() const
2853{
2854 //
2855 // Get current year from logbook entry
2856 //
2857
2858 if (!fLogbookEntry) return 0;
2859
2860 TTimeStamp startTime(GetCurrentStartTime());
2861 TString year = Form("%d",startTime.GetDate());
2862 year = year(0,4);
2863
2864 return year.Atoi();
2865}
2866
2867//______________________________________________________________________________________________
2868const char* AliShuttle::GetLHCPeriod() const
2869{
2870 //
2871 // Get current LHC period from logbook entry
2872 //
2873
2874 if (!fLogbookEntry) return 0;
2875
2876 return fLogbookEntry->GetRunParameter("LHCperiod");
2877}
2878
b948db8d 2879//______________________________________________________________________________________________
2880void AliShuttle::Log(const char* detector, const char* message)
2881{
9827400b 2882 //
2883 // Fill log string with a message
2884 //
b948db8d 2885
7d4cf768 2886 TString logRunDir = GetShuttleLogDir();
2887 if (GetCurrentRun() >=0)
2888 logRunDir += Form("/%d", GetCurrentRun());
2889
2890 void* dir = gSystem->OpenDirectory(logRunDir.Data());
84090f85 2891 if (dir == NULL) {
7d4cf768 2892 if (gSystem->mkdir(logRunDir.Data(), kTRUE)) {
36c99a6a 2893 AliError(Form("Can't open directory <%s>", GetShuttleLogDir()));
84090f85 2894 return;
2895 }
b948db8d 2896
84090f85 2897 } else {
2898 gSystem->FreeDirectory(dir);
2899 }
b948db8d 2900
cb343cfd 2901 TString toLog = Form("%s (%d): %s - ", TTimeStamp(time(0)).AsString("s"), getpid(), detector);
e7f62f16 2902 if (GetCurrentRun() >= 0)
2903 toLog += Form("run %d - ", GetCurrentRun());
2bb7b766 2904 toLog += Form("%s", message);
2905
84090f85 2906 AliInfo(toLog.Data());
ffa29e93 2907
2908 // if we redirect the log output already to the file, leave here
2909 if (fOutputRedirected && strcmp(detector, "SHUTTLE") != 0)
2910 return;
b948db8d 2911
ffa29e93 2912 TString fileName = GetLogFileName(detector);
e7f62f16 2913
84090f85 2914 gSystem->ExpandPathName(fileName);
2915
2916 ofstream logFile;
2917 logFile.open(fileName, ofstream::out | ofstream::app);
2918
2919 if (!logFile.is_open()) {
2920 AliError(Form("Could not open file %s", fileName.Data()));
2921 return;
2922 }
7bfb2090 2923
84090f85 2924 logFile << toLog.Data() << "\n";
b948db8d 2925
84090f85 2926 logFile.close();
b948db8d 2927}
2bb7b766 2928
ffa29e93 2929//______________________________________________________________________________________________
2930TString AliShuttle::GetLogFileName(const char* detector) const
2931{
2932 //
2933 // returns the name of the log file for a given sub detector
2934 //
2935
2936 TString fileName;
2937
2938 if (GetCurrentRun() >= 0)
7d4cf768 2939 {
2940 fileName.Form("%s/%d/%s_%d.log", GetShuttleLogDir(), GetCurrentRun(),
2941 detector, GetCurrentRun());
2942 } else {
ffa29e93 2943 fileName.Form("%s/%s.log", GetShuttleLogDir(), detector);
7d4cf768 2944 }
ffa29e93 2945
2946 return fileName;
2947}
2948
2bb7b766 2949//______________________________________________________________________________________________
2950Bool_t AliShuttle::Collect(Int_t run)
2951{
9827400b 2952 //
2953 // Collects conditions data for all UNPROCESSED run written to DAQ LogBook in case of run = -1 (default)
2954 // If a dedicated run is given this run is processed
2955 //
2956 // In operational mode, this is the Shuttle function triggered by the EOR signal.
2957 //
2bb7b766 2958
eba76848 2959 if (run == -1)
2960 Log("SHUTTLE","Collect - Shuttle called. Collecting conditions data for unprocessed runs");
2961 else
2962 Log("SHUTTLE", Form("Collect - Shuttle called. Collecting conditions data for run %d", run));
cb343cfd 2963
2964 SetLastAction("Starting");
2bb7b766 2965
2966 TString whereClause("where shuttle_done=0");
eba76848 2967 if (run != -1)
2968 whereClause += Form(" and run=%d", run);
2bb7b766 2969
2970 TObjArray shuttleLogbookEntries;
be48e3ea 2971 if (!QueryShuttleLogbook(whereClause, shuttleLogbookEntries))
2972 {
cb343cfd 2973 Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
2bb7b766 2974 return kFALSE;
2975 }
2976
9e080f92 2977 if (shuttleLogbookEntries.GetEntries() == 0)
2978 {
2979 if (run == -1)
2980 Log("SHUTTLE","Collect - Found no UNPROCESSED runs in Shuttle logbook");
2981 else
2982 Log("SHUTTLE", Form("Collect - Run %d is already DONE "
2983 "or it does not exist in Shuttle logbook", run));
2984 return kTRUE;
2985 }
2986
be48e3ea 2987 for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
2988 fFirstUnprocessed[iDet] = kTRUE;
2989
fc5a4708 2990 if (run != -1)
be48e3ea 2991 {
2992 // query Shuttle logbook for earlier runs, check if some detectors are unprocessed,
2993 // flag them into fFirstUnprocessed array
2994 TString whereClause(Form("where shuttle_done=0 and run < %d", run));
2995 TObjArray tmpLogbookEntries;
2996 if (!QueryShuttleLogbook(whereClause, tmpLogbookEntries))
2997 {
2998 Log("SHUTTLE", "Collect - Can't retrieve entries from Shuttle logbook");
2999 return kFALSE;
3000 }
3001
3002 TIter iter(&tmpLogbookEntries);
3003 AliShuttleLogbookEntry* anEntry = 0;
3004 while ((anEntry = dynamic_cast<AliShuttleLogbookEntry*> (iter.Next())))
3005 {
3006 for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
3007 {
3008 if (anEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
3009 {
3010 AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
3011 anEntry->GetRun(), GetDetName(iDet)));
3012 fFirstUnprocessed[iDet] = kFALSE;
3013 }
3014 }
3015
3016 }
3017
3018 }
3019
3020 if (!RetrieveConditionsData(shuttleLogbookEntries))
3021 {
cb343cfd 3022 Log("SHUTTLE", "Collect - Process of at least one run failed");
2bb7b766 3023 return kFALSE;
3024 }
3025
36c99a6a 3026 Log("SHUTTLE", "Collect - Requested run(s) successfully processed");
eba76848 3027 return kTRUE;
2bb7b766 3028}
3029
2bb7b766 3030//______________________________________________________________________________________________
3031Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
3032{
9827400b 3033 //
3034 // Retrieve conditions data for all runs that aren't processed yet
3035 //
2bb7b766 3036
3037 Bool_t hasError = kFALSE;
3038
3039 TIter iter(&dateEntries);
3040 AliShuttleLogbookEntry* anEntry;
3041
3042 while ((anEntry = (AliShuttleLogbookEntry*) iter.Next())){
3043 if (!Process(anEntry)){
3044 hasError = kTRUE;
3045 }
4b95672b 3046
3047 // clean SHUTTLE temp directory
d524ade6 3048 //TString filename = Form("%s/*.shuttle", GetShuttleTempDir());
3049 //RemoveFile(filename.Data());
2bb7b766 3050 }
3051
3052 return hasError == kFALSE;
3053}
cb343cfd 3054
3055//______________________________________________________________________________________________
3056ULong_t AliShuttle::GetTimeOfLastAction() const
3057{
9827400b 3058 //
3059 // Gets time of last action
3060 //
3061
cb343cfd 3062 ULong_t tmp;
36c99a6a 3063
cb343cfd 3064 fMonitoringMutex->Lock();
be48e3ea 3065
cb343cfd 3066 tmp = fLastActionTime;
36c99a6a 3067
cb343cfd 3068 fMonitoringMutex->UnLock();
36c99a6a 3069
cb343cfd 3070 return tmp;
3071}
3072
3073//______________________________________________________________________________________________
3074const TString AliShuttle::GetLastAction() const
3075{
9827400b 3076 //
cb343cfd 3077 // returns a string description of the last action
9827400b 3078 //
cb343cfd 3079
3080 TString tmp;
36c99a6a 3081
cb343cfd 3082 fMonitoringMutex->Lock();
3083
3084 tmp = fLastAction;
3085
3086 fMonitoringMutex->UnLock();
3087
36c99a6a 3088 return tmp;
cb343cfd 3089}
3090
3091//______________________________________________________________________________________________
3092void AliShuttle::SetLastAction(const char* action)
3093{
9827400b 3094 //
cb343cfd 3095 // updates the monitoring variables
9827400b 3096 //
36c99a6a 3097
cb343cfd 3098 fMonitoringMutex->Lock();
36c99a6a 3099
cb343cfd 3100 fLastAction = action;
3101 fLastActionTime = time(0);
3102
3103 fMonitoringMutex->UnLock();
3104}
eba76848 3105
3106//______________________________________________________________________________________________
3107const char* AliShuttle::GetRunParameter(const char* param)
3108{
9827400b 3109 //
3110 // returns run parameter read from DAQ logbook
3111 //
eba76848 3112
3113 if(!fLogbookEntry) {
3114 AliError("No logbook entry!");
3115 return 0;
3116 }
3117
3118 return fLogbookEntry->GetRunParameter(param);
3119}
57c1a579 3120
d386d623 3121//______________________________________________________________________________________________
9827400b 3122AliCDBEntry* AliShuttle::GetFromOCDB(const char* detector, const AliCDBPath& path)
d386d623 3123{
9827400b 3124 //
3125 // returns object from OCDB valid for current run
3126 //
d386d623 3127
9827400b 3128 if (fTestMode & kErrorOCDB)
3129 {
3130 Log(detector, "GetFromOCDB - In TESTMODE - Simulating error with OCDB");
3131 return 0;
3132 }
3133
d386d623 3134 AliCDBStorage *sto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
3135 if (!sto)
3136 {
9827400b 3137 Log(detector, "GetFromOCDB - Cannot activate main OCDB for query!");
d386d623 3138 return 0;
3139 }
3140
3141 return dynamic_cast<AliCDBEntry*> (sto->Get(path, GetCurrentRun()));
3142}
3143
57c1a579 3144//______________________________________________________________________________________________
3145Bool_t AliShuttle::SendMail()
3146{
9827400b 3147 //
3148 // sends a mail to the subdetector expert in case of preprocessor error
3149 //
3150
3151 if (fTestMode != kNone)
3152 return kTRUE;
57c1a579 3153
36c99a6a 3154 void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
57c1a579 3155 if (dir == NULL)
3156 {
36c99a6a 3157 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE))
57c1a579 3158 {
675f64cd 3159 Log("SHUTTLE", Form("SendMail - Can't open directory <%s>", GetShuttleLogDir()));
57c1a579 3160 return kFALSE;
3161 }
3162
3163 } else {
3164 gSystem->FreeDirectory(dir);
3165 }
3166
3167 TString bodyFileName;
36c99a6a 3168 bodyFileName.Form("%s/mail.body", GetShuttleLogDir());
57c1a579 3169 gSystem->ExpandPathName(bodyFileName);
3170
3171 ofstream mailBody;
3172 mailBody.open(bodyFileName, ofstream::out);
3173
3174 if (!mailBody.is_open())
3175 {
675f64cd 3176 Log("SHUTTLE", Form("Could not open mail body file %s", bodyFileName.Data()));
57c1a579 3177 return kFALSE;
3178 }
3179
3180 TString to="";
3181 TIter iterExperts(fConfig->GetResponsibles(fCurrentDetector));
3182 TObjString *anExpert=0;
3183 while ((anExpert = (TObjString*) iterExperts.Next()))
3184 {
3185 to += Form("%s,", anExpert->GetName());
3186 }
3187 to.Remove(to.Length()-1);
909732f7 3188 AliDebug(2, Form("to: %s",to.Data()));
57c1a579 3189
86aa42c3 3190 if (to.IsNull()) {
c88ad5db 3191 Log("SHUTTLE", "List of detector responsibles not yet set!");
36c99a6a 3192 return kFALSE;
3193 }
3194
57c1a579 3195 TString cc="alberto.colla@cern.ch";
3196
546242fb 3197 TString subject = Form("%s Shuttle preprocessor FAILED in run %d !",
57c1a579 3198 fCurrentDetector.Data(), GetCurrentRun());
909732f7 3199 AliDebug(2, Form("subject: %s", subject.Data()));
57c1a579 3200
3201 TString body = Form("Dear %s expert(s), \n\n", fCurrentDetector.Data());
3202 body += Form("SHUTTLE just detected that your preprocessor "
546242fb 3203 "failed processing run %d!!\n\n", GetCurrentRun());
7d4cf768 3204 body += Form("Please check %s status on the SHUTTLE monitoring page: \n\n",
3205 fCurrentDetector.Data());
b0e53b15 3206 if (fConfig->GetRunMode() == AliShuttleConfig::kTest)
3207 {
3208 body += Form("\thttp://pcalimonitor.cern.ch:8889/shuttle.jsp?time=168 \n\n");
3209 } else {
3210 body += Form("\thttp://pcalimonitor.cern.ch/shuttle.jsp?instance=PROD?time=168 \n\n");
3211 }
3212
7d4cf768 3213
3214 TString logFolder = "logs";
3215 if (fConfig->GetRunMode() == AliShuttleConfig::kProd)
3216 logFolder += "_PROD";
3217
3218
546242fb 3219 body += Form("Find the %s log for the current run on \n\n"
7d4cf768 3220 "\thttp://pcalishuttle01.cern.ch:8880/%s/%d/%s_%d.log \n\n",
3221 fCurrentDetector.Data(), logFolder.Data(), GetCurrentRun(),
3222 fCurrentDetector.Data(), GetCurrentRun());
3223 body += Form("The last 10 lines of %s log file are following:\n\n", fCurrentDetector.Data());
57c1a579 3224
909732f7 3225 AliDebug(2, Form("Body begin: %s", body.Data()));
57c1a579 3226
3227 mailBody << body.Data();
3228 mailBody.close();
3229 mailBody.open(bodyFileName, ofstream::out | ofstream::app);
3230
7d4cf768 3231 TString logFileName = Form("%s/%d/%s_%d.log", GetShuttleLogDir(),
3232 GetCurrentRun(), fCurrentDetector.Data(), GetCurrentRun());
57c1a579 3233 TString tailCommand = Form("tail -n 10 %s >> %s", logFileName.Data(), bodyFileName.Data());
3234 if (gSystem->Exec(tailCommand.Data()))
3235 {
3236 mailBody << Form("%s log file not found ...\n\n", fCurrentDetector.Data());
3237 }
3238
3239 TString endBody = Form("------------------------------------------------------\n\n");
675f64cd 3240 endBody += Form("In case of problems please contact the SHUTTLE core team.\n\n");
3241 endBody += "Please do not answer this message directly, it is automatically generated.\n\n";
3242 endBody += "Greetings,\n\n \t\t\tthe SHUTTLE\n";
3243
3244 AliDebug(2, Form("Body end: %s", endBody.Data()));
3245
3246 mailBody << endBody.Data();
3247
3248 mailBody.close();
3249
3250 // send mail!
3251 TString mailCommand = Form("mail -s \"%s\" -c %s %s < %s",
3252 subject.Data(),
3253 cc.Data(),
3254 to.Data(),
3255 bodyFileName.Data());
3256 AliDebug(2, Form("mail command: %s", mailCommand.Data()));
3257
3258 Bool_t result = gSystem->Exec(mailCommand.Data());
3259
3260 return result == 0;
3261}
3262
3263//______________________________________________________________________________________________
3264Bool_t AliShuttle::SendMailToDCS()
3265{
3266 //
3267 // sends a mail to the DCS experts in case of DCS error
3268 //
3269
3270 if (fTestMode != kNone)
3271 return kTRUE;
3272
3273 void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
3274 if (dir == NULL)
3275 {
3276 if (gSystem->mkdir(GetShuttleLogDir(), kTRUE))
3277 {
3278 Log("SHUTTLE", Form("SendMailToDCS - Can't open directory <%s>", GetShuttleLogDir()));
3279 return kFALSE;
3280 }
3281
3282 } else {
3283 gSystem->FreeDirectory(dir);
3284 }
3285
3286 TString bodyFileName;
3287 bodyFileName.Form("%s/mail.body", GetShuttleLogDir());
3288 gSystem->ExpandPathName(bodyFileName);
3289
3290 ofstream mailBody;
3291 mailBody.open(bodyFileName, ofstream::out);
3292
3293 if (!mailBody.is_open())
3294 {
3295 Log("SHUTTLE", Form("SendMailToDCS - Could not open mail body file %s", bodyFileName.Data()));
3296 return kFALSE;
3297 }
3298
3299 TString to="Vladimir.Fekete@cern.ch, Svetozar.Kapusta@cern.ch";
3300 //TString to="alberto.colla@cern.ch";
3301 AliDebug(2, Form("to: %s",to.Data()));
3302
3303 if (to.IsNull()) {
3304 Log("SHUTTLE", "List of detector responsibles not yet set!");
3305 return kFALSE;
3306 }
3307
3308 TString cc="alberto.colla@cern.ch";
3309
3310 TString subject = Form("Retrieval of data points for %s FAILED in run %d !",
3311 fCurrentDetector.Data(), GetCurrentRun());
3312 AliDebug(2, Form("subject: %s", subject.Data()));
3313
3314 TString body = Form("Dear DCS experts, \n\n");
3315 body += Form("SHUTTLE couldn\'t retrieve the data points for detector %s "
3316 "in run %d!!\n\n", fCurrentDetector.Data(), GetCurrentRun());
7d4cf768 3317 body += Form("Please check %s status on the SHUTTLE monitoring page: \n\n",
3318 fCurrentDetector.Data());
b0e53b15 3319 if (fConfig->GetRunMode() == AliShuttleConfig::kTest)
3320 {
3321 body += Form("\thttp://pcalimonitor.cern.ch:8889/shuttle.jsp?time=168 \n\n");
3322 } else {
3323 body += Form("\thttp://pcalimonitor.cern.ch/shuttle.jsp?instance=PROD?time=168 \n\n");
3324 }
7d4cf768 3325
3326 TString logFolder = "logs";
3327 if (fConfig->GetRunMode() == AliShuttleConfig::kProd)
3328 logFolder += "_PROD";
3329
3330
675f64cd 3331 body += Form("Find the %s log for the current run on \n\n"
7d4cf768 3332 "\thttp://pcalishuttle01.cern.ch:8880/%s/%d/%s_%d.log \n\n",
3333 fCurrentDetector.Data(), logFolder.Data(), GetCurrentRun(),
3334 fCurrentDetector.Data(), GetCurrentRun());
3335 body += Form("The last 10 lines of %s log file are following:\n\n", fCurrentDetector.Data());
675f64cd 3336
3337 AliDebug(2, Form("Body begin: %s", body.Data()));
3338
3339 mailBody << body.Data();
3340 mailBody.close();
3341 mailBody.open(bodyFileName, ofstream::out | ofstream::app);
3342
7d4cf768 3343 TString logFileName = Form("%s/%d/%s_%d.log", GetShuttleLogDir(), GetCurrentRun(),
3344 fCurrentDetector.Data(), GetCurrentRun());
675f64cd 3345 TString tailCommand = Form("tail -n 10 %s >> %s", logFileName.Data(), bodyFileName.Data());
3346 if (gSystem->Exec(tailCommand.Data()))
3347 {
3348 mailBody << Form("%s log file not found ...\n\n", fCurrentDetector.Data());
3349 }
3350
3351 TString endBody = Form("------------------------------------------------------\n\n");
36c99a6a 3352 endBody += Form("In case of problems please contact the SHUTTLE core team.\n\n");
3353 endBody += "Please do not answer this message directly, it is automatically generated.\n\n";
546242fb 3354 endBody += "Greetings,\n\n \t\t\tthe SHUTTLE\n";
57c1a579 3355
909732f7 3356 AliDebug(2, Form("Body end: %s", endBody.Data()));
57c1a579 3357
3358 mailBody << endBody.Data();
3359
3360 mailBody.close();
3361
3362 // send mail!
3363 TString mailCommand = Form("mail -s \"%s\" -c %s %s < %s",
3364 subject.Data(),
3365 cc.Data(),
3366 to.Data(),
3367 bodyFileName.Data());
909732f7 3368 AliDebug(2, Form("mail command: %s", mailCommand.Data()));
57c1a579 3369
3370 Bool_t result = gSystem->Exec(mailCommand.Data());
3371
3372 return result == 0;
3373}
d386d623 3374
441b0e9c 3375//______________________________________________________________________________________________
9827400b 3376const char* AliShuttle::GetRunType()
441b0e9c 3377{
9827400b 3378 //
3379 // returns run type read from "run type" logbook
3380 //
441b0e9c 3381
3382 if(!fLogbookEntry) {
3383 AliError("No logbook entry!");
3384 return 0;
3385 }
3386
9827400b 3387 return fLogbookEntry->GetRunType();
441b0e9c 3388}
3389
4859271b 3390//______________________________________________________________________________________________
3391Bool_t AliShuttle::GetHLTStatus()
3392{
3393 // Return HLT status (ON=1 OFF=0)
3394 // Converts the HLT status from the status string read in the run logbook (not just a bool)
3395
3396 if(!fLogbookEntry) {
3397 AliError("No logbook entry!");
3398 return 0;
3399 }
3400
3401 // TODO implement when HLTStatus is inserted in run logbook
3402 //TString hltStatus = fLogbookEntry->GetRunParameter("HLTStatus");
3403 //if(hltStatus == "OFF") {return kFALSE};
3404
3405 return kTRUE;
3406}
3407
d386d623 3408//______________________________________________________________________________________________
3409void AliShuttle::SetShuttleTempDir(const char* tmpDir)
3410{
9827400b 3411 //
3412 // sets Shuttle temp directory
3413 //
d386d623 3414
3415 fgkShuttleTempDir = gSystem->ExpandPathName(tmpDir);
3416}
3417
3418//______________________________________________________________________________________________
3419void AliShuttle::SetShuttleLogDir(const char* logDir)
3420{
9827400b 3421 //
3422 // sets Shuttle log directory
3423 //
d386d623 3424
3425 fgkShuttleLogDir = gSystem->ExpandPathName(logDir);
3426}