fix for bug #66294
[u/mrichter/AliRoot.git] / SHUTTLE / AliShuttleTrigger.cxx
index 1716fd9..1f10af5 100644 (file)
 
 /*
  $Log$
+ Revision 1.15  2007/12/10 18:29:23  acolla
+ Some log added to the listen mode
+
+ Revision 1.14  2007/12/07 19:14:36  acolla
+ in AliShuttleTrigger:
+
+ Added automatic collection of new runs on a regular time basis (settable from the configuration)
+
+ in AliShuttleConfig: new members
+
+ - triggerWait: time to wait for DIM trigger (s) before starting automatic collection of new runs
+ - mode: run mode (test, prod) -> used to build log folder (logs or logs_PROD)
+
+ in AliShuttle:
+
+ - logs now stored in logs/#RUN/DET_#RUN.log
+
+ Revision 1.13  2006/11/16 16:16:48  jgrosseo
+ introducing strict run ordering flag
+ removed giving preprocessor name to preprocessor, they have to know their name themselves ;-)
+
+ Revision 1.12  2006/10/20 15:22:59  jgrosseo
+ o) Adding time out to the execution of the preprocessors: The Shuttle forks and the parent process monitors the child
+ o) Merging Collect, CollectAll, CollectNew function
+ o) Removing implementation of empty copy constructors (declaration still there!)
+
  Revision 1.11  2006/10/02 16:38:39  jgrosseo
  update (alberto):
  fixed memory leaks
 #include "AliShuttleTrigger.h"
 
 #include <TSystem.h>
+#include <TObjString.h>
 
 #include "AliLog.h"
 #include "AliShuttleConfig.h"
 #include "AliShuttle.h"
 #include "DATENotifier.h"
 
+#include <fstream>
+
 ClassImp(TerminateSignalHandler)
 ClassImp(AliShuttleTrigger)
 
@@ -103,13 +132,13 @@ Bool_t TerminateSignalHandler::Notify()
 }
 
 //______________________________________________________________________________________________
-AliShuttleTrigger::AliShuttleTrigger(const AliShuttleConfig* config,
-               UInt_t timeout, Int_t retries):
+AliShuttleTrigger::AliShuttleTrigger(const AliShuttleConfig* config):
        fConfig(config), fShuttle(NULL),
        fNotified(kFALSE), fTerminate(kFALSE),
        fMutex(), fCondition(&fMutex),
        fQuitSignalHandler(0),
-       fInterruptSignalHandler(0)
+       fInterruptSignalHandler(0),
+       fLastMailDiskSpace(0)
 {
        //
        // config - pointer to the AliShuttleConfig object which represents
@@ -118,10 +147,13 @@ AliShuttleTrigger::AliShuttleTrigger(const AliShuttleConfig* config,
        // localStorage (local) CDB storage to be used if mainStorage is unavailable
        //
 
+       if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
+       UInt_t timeout = fConfig->GetDCSTimeOut();
+       Int_t retries = fConfig->GetDCSRetries();
        fShuttle = new AliShuttle(config, timeout, retries);
 
-  TerminateSignalHandler* fQuitSignalHandler = new TerminateSignalHandler(this, kSigQuit);
-  TerminateSignalHandler* fInterruptSignalHandler = new TerminateSignalHandler(this, kSigInterrupt);
+       fQuitSignalHandler = new TerminateSignalHandler(this, kSigQuit);
+       fInterruptSignalHandler = new TerminateSignalHandler(this, kSigInterrupt);
 
        gSystem->AddSignalHandler(fQuitSignalHandler);
        gSystem->AddSignalHandler(fInterruptSignalHandler);
@@ -175,6 +207,24 @@ void AliShuttleTrigger::Terminate() {
 }
 
 //______________________________________________________________________________________________
+void AliShuttleTrigger::CheckTerminate() 
+{
+       // 
+       // Checks if the Shuttle got an external terminate request by a created file 
+       // This is an alternative to the signal which causes problems with the API libraries
+       //
+
+       if (strlen(fConfig->GetTerminateFilePath()) == 0)
+               return;
+
+       if (gSystem->AccessPathName(fConfig->GetTerminateFilePath()) == kFALSE)
+       {
+               AliInfo("Terminate file exists. Terminating Shuttle...");
+               fTerminate = kTRUE;
+       }
+}
+
+//______________________________________________________________________________________________
 void AliShuttleTrigger::Run() {
        //
        // AliShuttleTrigger main loop for asynchronized (listen) mode.
@@ -184,14 +234,22 @@ void AliShuttleTrigger::Run() {
 
        fTerminate = kFALSE;
 
-       DATENotifier* notifier = new DATENotifier(this, "/DATE/LOGBOOK/UPDATE");
+       DATENotifier* notifier = new DATENotifier(this, "/LOGBOOK/SUBSCRIBE/ECS_EOR");
 
+       Int_t nTry=0; 
+       Int_t nMaxTry = fConfig->GetMaxRetries()+1;
+       Int_t received=0;
+       
+       AliInfo("Listening for ECS trigger");
+       
        while (1) {
        
                fMutex.Lock();
 
                while (!(fNotified || fTerminate)) {
-                       fCondition.Wait();
+                       received=fCondition.TimedWaitRelative(1000*fConfig->GetTriggerWait());
+                       CheckTerminate();
+                       if (received==1) break; // 1 = timeout
                }
 
                fNotified = kFALSE;
@@ -202,8 +260,34 @@ void AliShuttleTrigger::Run() {
                        AliInfo("Terminated.");
                        break;          
                }
+               
+               if (received == 0)
+               {
+                       AliInfo("Trigger from ECS received!");
+               } else if (received == 1) {
+                       AliInfo(Form("Timeout (%d s) waiting for trigger. "
+                               "Starting collection of new runs!", 
+                                       fConfig->GetTriggerWait()));
+               } else {
+                       AliInfo("Error receiving trigger from ECS!");
+                       break;
+               }
+               
+               nTry++;
+               AliInfo(Form("Received %d triggers so far", nTry));
+               
+               if (fConfig->GetRunMode() == AliShuttleConfig::kTest)
+               {
+                       if(nTry>=nMaxTry)
+                       {
+                               AliInfo(Form("Collect() ran more than %d times -> Exiting!", 
+                                               nMaxTry));
+                               break;
+                       }
+               }
        
                Collect();
+               CheckTerminate();
        }
 
        delete notifier;
@@ -217,5 +301,119 @@ Bool_t AliShuttleTrigger::Collect(Int_t run)
        // then it checks if the shuttle is still running by checking the monitoring functions of the shuttle
        //
 
-  return fShuttle->Collect(run);
+       // first checking disk space
+       Long_t id = 0;
+       Long_t bsize = 0;
+       Long_t blocks = 0;
+       Long_t bfree = 0;
+
+       gSystem->GetFsInfo(fConfig->GetShuttleFileSystem(), &id, &bsize, &blocks, &bfree);
+
+       AliInfo(Form("n. of free blocks = %d, total n. of blocks = %d",bfree,blocks));
+       Int_t spaceFree = (Int_t)(((Float_t)bfree/(Float_t)blocks)*100);
+
+       if (spaceFree < fConfig->GetFreeDiskWarningThreshold()) {
+               AliWarning(Form("************** Free space left = %d%%, below the Warning Threshold (%d%%)",spaceFree,fConfig->GetFreeDiskWarningThreshold()));
+               if (TMath::Abs(time(0) - fLastMailDiskSpace) >= 86400){   // 86400 = n. of seconds in 1 d
+                       SendMailDiskSpace(fConfig->GetFreeDiskWarningThreshold());
+                       fLastMailDiskSpace = time(0);  // resetting fLastMailDiskSpace to time(0) = now
+               }
+               if (spaceFree < fConfig->GetFreeDiskFatalThreshold()){
+                       AliError(Form("*************** Free space left = %d%%, below the Fatal Threshold (%d%%), terminating....",spaceFree,fConfig->GetFreeDiskFatalThreshold()));
+                       SendMailDiskSpace(fConfig->GetFreeDiskFatalThreshold());
+                       fTerminate = kTRUE; // terminating....
+               }
+       }       
+
+       if (fTerminate) {
+               return kFALSE;
+       }
+
+       return fShuttle->Collect(run);
+}
+//______________________________________________________________________________________________
+Bool_t AliShuttleTrigger::SendMailDiskSpace(Short_t percentage)
+{
+       //
+       // sends a mail to the shuttle experts in case of free disk space < theshold
+       //
+       
+               
+       AliInfo("******************* Sending the Mail!! *********************");
+       if (!fConfig->SendMail()) 
+               return kTRUE;
+
+       Int_t runMode = (Int_t)fConfig->GetRunMode();
+       TString tmpStr;
+       if (runMode == 0) tmpStr = " Nightly Test:";
+       else tmpStr = " Data Taking:"; 
+       void* dir = gSystem->OpenDirectory(fShuttle->GetShuttleLogDir());
+       if (dir == NULL)
+       {
+               if (gSystem->mkdir(fShuttle->GetShuttleLogDir(), kTRUE))
+               {
+                       AliWarning(Form("SendMail - Can't open directory <%s>", fShuttle->GetShuttleLogDir()));
+                       return kFALSE;
+               }
+
+       } else {
+               gSystem->FreeDirectory(dir);
+       }
+
+       // SHUTTLE responsibles in to
+       TString to="";
+       TIter iterAdmins(fConfig->GetAdmins(AliShuttleConfig::kGlobal));
+       TObjString *anAdmin=0;
+       while ((anAdmin = (TObjString*) iterAdmins.Next()))
+       {
+               to += Form("%s,", anAdmin->GetName());
+       }
+       if (to.Length() > 0)
+         to.Remove(to.Length()-1);
+       AliDebug(2, Form("to: %s",to.Data()));
+
+       // mail body 
+       TString bodyFileName;
+       bodyFileName.Form("%s/mail.body", fShuttle->GetShuttleLogDir());
+       gSystem->ExpandPathName(bodyFileName);
+
+       ofstream mailBody;
+       mailBody.open(bodyFileName, ofstream::out);
+
+       if (!mailBody.is_open())
+       {
+               AliWarning(Form("Could not open mail body file %s", bodyFileName.Data()));
+               return kFALSE;
+       }
+
+       TString subject;
+       TString body;
+
+       Int_t percentage_used = 100 - percentage;       
+       subject = Form("%s CRITICAL Disk Space usage exceeds %d%c!",
+                      tmpStr.Data(),percentage_used,'%');
+       AliDebug(2, Form("subject: %s", subject.Data()));
+
+       body = "Dear SHUTTLE experts, \n\n";
+       body += "The usage of the disk space on the shuttle machine has overcome \n"; 
+       body += Form("the threshold of %d%%. \n \n",percentage_used);
+       body += "Please check! \n \n";
+       body += "Please do not answer this message directly, it is automatically generated.\n\n";
+       body += "Greetings,\n\n \t\t\tthe SHUTTLE\n";
+
+       AliDebug(2, Form("Body : %s", body.Data()));
+
+       mailBody << body.Data();
+       mailBody.close();
+
+       // send mail!
+       TString mailCommand = Form("mail -s \"%s\" %s < %s",
+                                               subject.Data(),
+                                               to.Data(),
+                                               bodyFileName.Data());
+       AliDebug(2, Form("mail command: %s", mailCommand.Data()));
+
+       Bool_t result = gSystem->Exec(mailCommand.Data());
+
+       return result == 0;
 }