1 /**************************************************************************
2 * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
18 Revision 1.15 2007/12/10 18:29:23 acolla
19 Some log added to the listen mode
21 Revision 1.14 2007/12/07 19:14:36 acolla
24 Added automatic collection of new runs on a regular time basis (settable from the configuration)
26 in AliShuttleConfig: new members
28 - triggerWait: time to wait for DIM trigger (s) before starting automatic collection of new runs
29 - mode: run mode (test, prod) -> used to build log folder (logs or logs_PROD)
33 - logs now stored in logs/#RUN/DET_#RUN.log
35 Revision 1.13 2006/11/16 16:16:48 jgrosseo
36 introducing strict run ordering flag
37 removed giving preprocessor name to preprocessor, they have to know their name themselves ;-)
39 Revision 1.12 2006/10/20 15:22:59 jgrosseo
40 o) Adding time out to the execution of the preprocessors: The Shuttle forks and the parent process monitors the child
41 o) Merging Collect, CollectAll, CollectNew function
42 o) Removing implementation of empty copy constructors (declaration still there!)
44 Revision 1.11 2006/10/02 16:38:39 jgrosseo
47 storing of objects that failed to be stored to the grid before
48 interfacing of shuttle status table in daq system
50 Revision 1.10 2006/08/15 10:50:00 jgrosseo
51 effc++ corrections (alberto)
53 Revision 1.9 2006/08/08 14:19:29 jgrosseo
54 Update to shuttle classes (Alberto)
56 - Possibility to set the full object's path in the Preprocessor's and
57 Shuttle's Store functions
58 - Possibility to extend the object's run validity in the same classes
59 ("startValidity" and "validityInfinite" parameters)
60 - Implementation of the StoreReferenceData function to store reference
61 data in a dedicated CDB storage.
63 Revision 1.8 2006/07/21 07:37:20 jgrosseo
64 last run is stored after each run
66 Revision 1.7 2006/07/20 09:54:40 jgrosseo
67 introducing status management: The processing per subdetector is divided into several steps,
68 after each step the status is stored on disk. If the system crashes in any of the steps the Shuttle
69 can keep track of the number of failures and skips further processing after a certain threshold is
70 exceeded. These thresholds can be configured in LDAP.
72 Revision 1.6 2006/07/19 10:09:55 jgrosseo
73 new configuration, accesst to DAQ FES (Alberto)
75 Revision 1.5 2006/07/10 13:01:41 jgrosseo
76 enhanced storing of last sucessfully processed run (alberto)
78 Revision 1.4 2006/07/04 14:59:57 jgrosseo
79 revision of AliDCSValue: Removed wrapper classes, reduced storage size per value by factor 2
81 Revision 1.3 2006/06/12 09:11:16 jgrosseo
82 coding conventions (Alberto)
84 Revision 1.2 2006/06/06 14:26:40 jgrosseo
85 o) removed files that were moved to STEER
86 o) shuttle updated to follow the new interface (Alberto)
88 Revision 1.1 2006/03/07 07:52:34 hristov
89 New version (B.Yordanov)
91 Revision 1.5 2005/11/21 09:03:48 byordano
94 Revision 1.4 2005/11/20 10:12:37 byordano
95 comments added to AliShuttleTrigger
101 // This class is to deal with DAQ LogBook and DAQ "end of run" notification.
102 // It has severeal two modes:
103 // 1) synchronized - Collect()
104 // 2) asynchronized - Run() - starts listening for DAQ "end of run"
105 // notification by DIM service.
108 #include "AliShuttleTrigger.h"
112 #include <TObjString.h>
115 #include "AliShuttleConfig.h"
116 #include "AliShuttle.h"
117 #include "DATENotifier.h"
121 ClassImp(TerminateSignalHandler)
122 ClassImp(AliShuttleTrigger)
124 //______________________________________________________________________________________________
125 Bool_t TerminateSignalHandler::Notify()
127 // Sentd terminate command to the Shuttle trigger
129 AliInfo("Terminate signal received ...");
130 fTrigger->Terminate();
135 //______________________________________________________________________________________________
136 AliShuttleTrigger::AliShuttleTrigger(const AliShuttleConfig* config):
137 fConfig(config), fShuttle(NULL),
138 fNotified(kFALSE), fTerminate(kFALSE),
139 fMutex(), fCondition(&fMutex),
140 fQuitSignalHandler(0),
141 fInterruptSignalHandler(0),
142 fLastMailDiskSpace(0)
145 // config - pointer to the AliShuttleConfig object which represents
147 // mainStorage - pointer to AliCDBStorage for the undelying CDBStorage
148 // localStorage (local) CDB storage to be used if mainStorage is unavailable
151 if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
152 UInt_t timeout = fConfig->GetDCSTimeOut();
153 Int_t retries = fConfig->GetDCSRetries();
154 fShuttle = new AliShuttle(config, timeout, retries);
156 fQuitSignalHandler = new TerminateSignalHandler(this, kSigQuit);
157 fInterruptSignalHandler = new TerminateSignalHandler(this, kSigInterrupt);
159 gSystem->AddSignalHandler(fQuitSignalHandler);
160 gSystem->AddSignalHandler(fInterruptSignalHandler);
164 //______________________________________________________________________________________________
165 AliShuttleTrigger::~AliShuttleTrigger()
169 gSystem->RemoveSignalHandler(fQuitSignalHandler);
170 gSystem->RemoveSignalHandler(fInterruptSignalHandler);
174 delete fQuitSignalHandler;
175 fQuitSignalHandler = 0;
177 delete fInterruptSignalHandler;
178 fInterruptSignalHandler = 0;
181 //______________________________________________________________________________________________
182 Bool_t AliShuttleTrigger::Notify() {
184 // Trigger Collect() methods in asynchronized (listen) mode.
185 // Usually called automaticly by DATENotifier on "end of run"
186 // notification event.
199 //______________________________________________________________________________________________
200 void AliShuttleTrigger::Terminate() {
202 // Stop triggers listen mode and exist from Run()
203 // Usually called automaticly by TerminateSignalHandler.
210 //______________________________________________________________________________________________
211 void AliShuttleTrigger::CheckTerminate()
214 // Checks if the Shuttle got an external terminate request by a created file
215 // This is an alternative to the signal which causes problems with the API libraries
218 if (strlen(fConfig->GetTerminateFilePath()) == 0)
221 if (gSystem->AccessPathName(fConfig->GetTerminateFilePath()) == kFALSE)
223 AliInfo("Terminate file exists. Terminating Shuttle...");
228 //______________________________________________________________________________________________
229 void AliShuttleTrigger::Run() {
231 // AliShuttleTrigger main loop for asynchronized (listen) mode.
232 // It spawns DIM service listener and waits for DAQ "end of run"
233 // notification. Calls Collect() on notification.
238 DATENotifier* notifier = new DATENotifier(this, "/LOGBOOK/SUBSCRIBE/ECS_EOR");
241 Int_t nMaxTry = fConfig->GetMaxRetries()+1;
244 AliInfo("Listening for ECS trigger");
250 while (!(fNotified || fTerminate)) {
251 for (Int_t iwait = 0; iwait < 10; iwait++){
252 received = fCondition.TimedWaitRelative(1000*fConfig->GetTriggerWait()/10); // to keep the connection to the server alive every minute while waiting for new runs
255 AliInfo(Form("Keeping the connection to the server alive while waiting for new runs - %d waited from last one", fConfig->GetTriggerWait()/10*iwait));
259 AliInfo("No gGrid initialized so far, we cannot keep the connection to the server alive while waiting for new runs");
263 //received=fCondition.TimedWaitRelative(1000*fConfig->GetTriggerWait());
265 if (received==1) break; // 1 = timeout
273 AliInfo("Terminated.");
279 AliInfo("Trigger from ECS received!");
280 } else if (received == 1) {
281 AliInfo(Form("Timeout (%d s) waiting for trigger. "
282 "Starting collection of new runs!",
283 fConfig->GetTriggerWait()));
285 AliInfo("Error receiving trigger from ECS!");
290 AliInfo(Form("Received %d triggers so far", nTry));
292 if (fConfig->GetRunMode() == AliShuttleConfig::kTest)
296 AliInfo(Form("Collect() ran more than %d times -> Exiting!",
309 //______________________________________________________________________________________________
310 Bool_t AliShuttleTrigger::Collect(Int_t run)
313 // this function creates a thread that runs the shuttle
314 // then it checks if the shuttle is still running by checking the monitoring functions of the shuttle
317 // first checking disk space
323 gSystem->GetFsInfo(fConfig->GetShuttleFileSystem(), &id, &bsize, &blocks, &bfree);
325 AliInfo(Form("n. of free blocks = %ld, total n. of blocks = %ld",bfree,blocks));
326 Int_t spaceFree = (Int_t)(((Float_t)bfree/(Float_t)blocks)*100);
328 if (spaceFree < fConfig->GetFreeDiskWarningThreshold()) {
329 AliWarning(Form("************** Free space left = %d%%, below the Warning Threshold (%d%%)",spaceFree,fConfig->GetFreeDiskWarningThreshold()));
330 if (TMath::Abs(time(0) - fLastMailDiskSpace) >= 86400){ // 86400 = n. of seconds in 1 d
331 SendMailDiskSpace(fConfig->GetFreeDiskWarningThreshold());
332 fLastMailDiskSpace = time(0); // resetting fLastMailDiskSpace to time(0) = now
334 if (spaceFree < fConfig->GetFreeDiskFatalThreshold()){
335 AliError(Form("*************** Free space left = %d%%, below the Fatal Threshold (%d%%), terminating....",spaceFree,fConfig->GetFreeDiskFatalThreshold()));
336 SendMailDiskSpace(fConfig->GetFreeDiskFatalThreshold());
337 fTerminate = kTRUE; // terminating....
345 return fShuttle->Collect(run);
347 //______________________________________________________________________________________________
348 Bool_t AliShuttleTrigger::SendMailDiskSpace(Short_t percentage)
351 // sends a mail to the shuttle experts in case of free disk space < theshold
355 AliInfo("******************* Sending the Mail!! *********************");
356 if (!fConfig->SendMail())
359 Int_t runMode = (Int_t)fConfig->GetRunMode();
361 if (runMode == 0) tmpStr = " Nightly Test:";
362 else tmpStr = " Data Taking:";
363 void* dir = gSystem->OpenDirectory(fShuttle->GetShuttleLogDir());
366 if (gSystem->mkdir(fShuttle->GetShuttleLogDir(), kTRUE))
368 AliWarning(Form("SendMail - Can't open directory <%s>", fShuttle->GetShuttleLogDir()));
373 gSystem->FreeDirectory(dir);
376 // SHUTTLE responsibles in to
378 TIter iterAdmins(fConfig->GetAdmins(AliShuttleConfig::kGlobal));
379 TObjString *anAdmin=0;
380 while ((anAdmin = (TObjString*) iterAdmins.Next()))
382 to += Form("%s,", anAdmin->GetName());
385 to.Remove(to.Length()-1);
386 AliDebug(2, Form("to: %s",to.Data()));
389 TString bodyFileName;
390 bodyFileName.Form("%s/mail.body", fShuttle->GetShuttleLogDir());
391 gSystem->ExpandPathName(bodyFileName);
394 mailBody.open(bodyFileName, ofstream::out);
396 if (!mailBody.is_open())
398 AliWarning(Form("Could not open mail body file %s", bodyFileName.Data()));
405 Int_t percentage_used = 100 - percentage;
406 subject = Form("%s CRITICAL Disk Space usage exceeds %d%c!",
407 tmpStr.Data(),percentage_used,'%');
408 AliDebug(2, Form("subject: %s", subject.Data()));
410 body = "Dear SHUTTLE experts, \n\n";
411 body += "The usage of the disk space on the shuttle machine has overcome \n";
412 body += Form("the threshold of %d%%. \n \n",percentage_used);
413 body += "Please check! \n \n";
414 body += "Please do not answer this message directly, it is automatically generated.\n\n";
415 body += "Greetings,\n\n \t\t\tthe SHUTTLE\n";
417 AliDebug(2, Form("Body : %s", body.Data()));
419 mailBody << body.Data();
423 TString mailCommand = Form("mail -s \"%s\" %s < %s",
426 bodyFileName.Data());
427 AliDebug(2, Form("mail command: %s", mailCommand.Data()));
429 Bool_t result = gSystem->Exec(mailCommand.Data());