X-Git-Url: http://git.uio.no/git/?p=u%2Fmrichter%2FAliRoot.git;a=blobdiff_plain;f=SHUTTLE%2FAliShuttleTrigger.cxx;h=149bef760d0335778d695fbba00337db1fe2f335;hp=1716fd9f6009b67d8fdfdcc72fe0ff619c4d2855;hb=03dec2c995153acc4cbb3a0f13b3b6a455d81ce1;hpb=cb343cfdc859a5a29405fd45f0d299831371553b diff --git a/SHUTTLE/AliShuttleTrigger.cxx b/SHUTTLE/AliShuttleTrigger.cxx index 1716fd9f600..149bef760d0 100644 --- a/SHUTTLE/AliShuttleTrigger.cxx +++ b/SHUTTLE/AliShuttleTrigger.cxx @@ -15,6 +15,32 @@ /* $Log$ + Revision 1.15 2007/12/10 18:29:23 acolla + Some log added to the listen mode + + Revision 1.14 2007/12/07 19:14:36 acolla + in AliShuttleTrigger: + + Added automatic collection of new runs on a regular time basis (settable from the configuration) + + in AliShuttleConfig: new members + + - triggerWait: time to wait for DIM trigger (s) before starting automatic collection of new runs + - mode: run mode (test, prod) -> used to build log folder (logs or logs_PROD) + + in AliShuttle: + + - logs now stored in logs/#RUN/DET_#RUN.log + + Revision 1.13 2006/11/16 16:16:48 jgrosseo + introducing strict run ordering flag + removed giving preprocessor name to preprocessor, they have to know their name themselves ;-) + + Revision 1.12 2006/10/20 15:22:59 jgrosseo + o) Adding time out to the execution of the preprocessors: The Shuttle forks and the parent process monitors the child + o) Merging Collect, CollectAll, CollectNew function + o) Removing implementation of empty copy constructors (declaration still there!) + Revision 1.11 2006/10/02 16:38:39 jgrosseo update (alberto): fixed memory leaks @@ -82,12 +108,15 @@ #include "AliShuttleTrigger.h" #include +#include #include "AliLog.h" #include "AliShuttleConfig.h" #include "AliShuttle.h" #include "DATENotifier.h" +#include + ClassImp(TerminateSignalHandler) ClassImp(AliShuttleTrigger) @@ -103,13 +132,13 @@ Bool_t TerminateSignalHandler::Notify() } //______________________________________________________________________________________________ -AliShuttleTrigger::AliShuttleTrigger(const AliShuttleConfig* config, - UInt_t timeout, Int_t retries): +AliShuttleTrigger::AliShuttleTrigger(const AliShuttleConfig* config): fConfig(config), fShuttle(NULL), fNotified(kFALSE), fTerminate(kFALSE), fMutex(), fCondition(&fMutex), fQuitSignalHandler(0), - fInterruptSignalHandler(0) + fInterruptSignalHandler(0), + fLastMailDiskSpace(0) { // // config - pointer to the AliShuttleConfig object which represents @@ -118,10 +147,13 @@ AliShuttleTrigger::AliShuttleTrigger(const AliShuttleConfig* config, // localStorage (local) CDB storage to be used if mainStorage is unavailable // + if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********"); + UInt_t timeout = fConfig->GetDCSTimeOut(); + Int_t retries = fConfig->GetDCSRetries(); fShuttle = new AliShuttle(config, timeout, retries); - TerminateSignalHandler* fQuitSignalHandler = new TerminateSignalHandler(this, kSigQuit); - TerminateSignalHandler* fInterruptSignalHandler = new TerminateSignalHandler(this, kSigInterrupt); + fQuitSignalHandler = new TerminateSignalHandler(this, kSigQuit); + fInterruptSignalHandler = new TerminateSignalHandler(this, kSigInterrupt); gSystem->AddSignalHandler(fQuitSignalHandler); gSystem->AddSignalHandler(fInterruptSignalHandler); @@ -184,14 +216,21 @@ void AliShuttleTrigger::Run() { fTerminate = kFALSE; - DATENotifier* notifier = new DATENotifier(this, "/DATE/LOGBOOK/UPDATE"); + DATENotifier* notifier = new DATENotifier(this, "/LOGBOOK/SUBSCRIBE/ECS_EOR"); + Int_t nTry=0; + Int_t nMaxTry = fConfig->GetMaxRetries()+1; + Int_t received=0; + + AliInfo("Listening for ECS trigger"); + while (1) { fMutex.Lock(); while (!(fNotified || fTerminate)) { - fCondition.Wait(); + received=fCondition.TimedWaitRelative(1000*fConfig->GetTriggerWait()); + if (received==1) break; // 1 = timeout } fNotified = kFALSE; @@ -202,6 +241,31 @@ void AliShuttleTrigger::Run() { AliInfo("Terminated."); break; } + + if (received == 0) + { + AliInfo("Trigger from ECS received!"); + } else if (received == 1) { + AliInfo(Form("Timeout (%d s) waiting for trigger. " + "Starting collection of new runs!", + fConfig->GetTriggerWait())); + } else { + AliInfo("Error receiving trigger from ECS!"); + break; + } + + nTry++; + AliInfo(Form("Received %d triggers so far", nTry)); + + if (fConfig->GetRunMode() == AliShuttleConfig::kTest) + { + if(nTry>=nMaxTry) + { + AliInfo(Form("Collect() ran more than %d times -> Exiting!", + nMaxTry)); + break; + } + } Collect(); } @@ -217,5 +281,119 @@ Bool_t AliShuttleTrigger::Collect(Int_t run) // then it checks if the shuttle is still running by checking the monitoring functions of the shuttle // - return fShuttle->Collect(run); + // first checking disk space + Long_t id = 0; + Long_t bsize = 0; + Long_t blocks = 0; + Long_t bfree = 0; + + gSystem->GetFsInfo(fConfig->GetShuttleFileSystem(), &id, &bsize, &blocks, &bfree); + + AliInfo(Form("n. of free blocks = %d, total n. of blocks = %d",bfree,blocks)); + Int_t spaceFree = (Int_t)(((Float_t)bfree/(Float_t)blocks)*100); + + if (spaceFree < fConfig->GetFreeDiskWarningThreshold()) { + AliWarning(Form("************** Free space left = %d%%, below the Warning Threshold (%d%%)",spaceFree,fConfig->GetFreeDiskWarningThreshold())); + if (TMath::Abs(time(0) - fLastMailDiskSpace) >= 86400){ // 86400 = n. of seconds in 1 d + SendMailDiskSpace(fConfig->GetFreeDiskWarningThreshold()); + fLastMailDiskSpace = time(0); // resetting fLastMailDiskSpace to time(0) = now + } + if (spaceFree < fConfig->GetFreeDiskFatalThreshold()){ + AliError(Form("*************** Free space left = %d%%, below the Fatal Threshold (%d%%), terminating....",spaceFree,fConfig->GetFreeDiskFatalThreshold())); + SendMailDiskSpace(fConfig->GetFreeDiskFatalThreshold()); + fTerminate = kTRUE; // terminating.... + } + } + + if (fTerminate) { + return kFALSE; + } + + return fShuttle->Collect(run); +} +//______________________________________________________________________________________________ +Bool_t AliShuttleTrigger::SendMailDiskSpace(Short_t percentage) +{ + // + // sends a mail to the shuttle experts in case of free disk space < theshold + // + + + AliInfo("******************* Sending the Mail!! *********************"); + if (!fConfig->SendMail()) + return kTRUE; + + Int_t runMode = (Int_t)fConfig->GetRunMode(); + TString tmpStr; + if (runMode == 0) tmpStr = " Nightly Test:"; + else tmpStr = " Data Taking:"; + void* dir = gSystem->OpenDirectory(fShuttle->GetShuttleLogDir()); + if (dir == NULL) + { + if (gSystem->mkdir(fShuttle->GetShuttleLogDir(), kTRUE)) + { + AliWarning(Form("SendMail - Can't open directory <%s>", fShuttle->GetShuttleLogDir())); + return kFALSE; + } + + } else { + gSystem->FreeDirectory(dir); + } + + // SHUTTLE responsibles in to + TString to=""; + TIter iterAdmins(fConfig->GetAdmins(AliShuttleConfig::kGlobal)); + TObjString *anAdmin=0; + while ((anAdmin = (TObjString*) iterAdmins.Next())) + { + to += Form("%s,", anAdmin->GetName()); + } + if (to.Length() > 0) + to.Remove(to.Length()-1); + AliDebug(2, Form("to: %s",to.Data())); + + // mail body + TString bodyFileName; + bodyFileName.Form("%s/mail.body", fShuttle->GetShuttleLogDir()); + gSystem->ExpandPathName(bodyFileName); + + ofstream mailBody; + mailBody.open(bodyFileName, ofstream::out); + + if (!mailBody.is_open()) + { + AliWarning(Form("Could not open mail body file %s", bodyFileName.Data())); + return kFALSE; + } + + TString subject; + TString body; + + subject = Form("%s CRITICAL Disk Space usage exceeds %d%c!", + tmpStr.Data(),percentage,'%'); + AliDebug(2, Form("subject: %s", subject.Data())); + Int_t percentage_used = 100 - percentage; + + body = "Dear SHUTTLE experts, \n\n"; + body += "The usage of the disk space on the shuttle machine has overcome \n"; + body += Form("the threshold of %d%%. \n \n",percentage_used); + body += "Please check! \n \n"; + body += "Please do not answer this message directly, it is automatically generated.\n\n"; + body += "Greetings,\n\n \t\t\tthe SHUTTLE\n"; + + AliDebug(2, Form("Body : %s", body.Data())); + + mailBody << body.Data(); + mailBody.close(); + + // send mail! + TString mailCommand = Form("mail -s \"%s\" %s < %s", + subject.Data(), + to.Data(), + bodyFileName.Data()); + AliDebug(2, Form("mail command: %s", mailCommand.Data())); + + Bool_t result = gSystem->Exec(mailCommand.Data()); + + return result == 0; }