/*
$Log$
+ Revision 1.15 2007/12/10 18:29:23 acolla
+ Some log added to the listen mode
+
+ Revision 1.14 2007/12/07 19:14:36 acolla
+ in AliShuttleTrigger:
+
+ Added automatic collection of new runs on a regular time basis (settable from the configuration)
+
+ in AliShuttleConfig: new members
+
+ - triggerWait: time to wait for DIM trigger (s) before starting automatic collection of new runs
+ - mode: run mode (test, prod) -> used to build log folder (logs or logs_PROD)
+
+ in AliShuttle:
+
+ - logs now stored in logs/#RUN/DET_#RUN.log
+
+ Revision 1.13 2006/11/16 16:16:48 jgrosseo
+ introducing strict run ordering flag
+ removed giving preprocessor name to preprocessor, they have to know their name themselves ;-)
+
+ Revision 1.12 2006/10/20 15:22:59 jgrosseo
+ o) Adding time out to the execution of the preprocessors: The Shuttle forks and the parent process monitors the child
+ o) Merging Collect, CollectAll, CollectNew function
+ o) Removing implementation of empty copy constructors (declaration still there!)
+
Revision 1.11 2006/10/02 16:38:39 jgrosseo
update (alberto):
fixed memory leaks
#include "AliShuttleTrigger.h"
#include <TSystem.h>
+#include <TGrid.h>
+#include <TObjString.h>
#include "AliLog.h"
#include "AliShuttleConfig.h"
#include "AliShuttle.h"
#include "DATENotifier.h"
+#include <fstream>
+
ClassImp(TerminateSignalHandler)
ClassImp(AliShuttleTrigger)
}
//______________________________________________________________________________________________
-AliShuttleTrigger::AliShuttleTrigger(const AliShuttleConfig* config,
- UInt_t timeout, Int_t retries):
+AliShuttleTrigger::AliShuttleTrigger(const AliShuttleConfig* config):
fConfig(config), fShuttle(NULL),
fNotified(kFALSE), fTerminate(kFALSE),
fMutex(), fCondition(&fMutex),
fQuitSignalHandler(0),
- fInterruptSignalHandler(0)
+ fInterruptSignalHandler(0),
+ fLastMailDiskSpace(0)
{
//
// config - pointer to the AliShuttleConfig object which represents
// localStorage (local) CDB storage to be used if mainStorage is unavailable
//
+ if (!fConfig->IsValid()) AliFatal("********** !!!!! Invalid configuration !!!!! **********");
+ UInt_t timeout = fConfig->GetDCSTimeOut();
+ Int_t retries = fConfig->GetDCSRetries();
fShuttle = new AliShuttle(config, timeout, retries);
- TerminateSignalHandler* fQuitSignalHandler = new TerminateSignalHandler(this, kSigQuit);
- TerminateSignalHandler* fInterruptSignalHandler = new TerminateSignalHandler(this, kSigInterrupt);
+ fQuitSignalHandler = new TerminateSignalHandler(this, kSigQuit);
+ fInterruptSignalHandler = new TerminateSignalHandler(this, kSigInterrupt);
gSystem->AddSignalHandler(fQuitSignalHandler);
gSystem->AddSignalHandler(fInterruptSignalHandler);
fCondition.Signal();
}
+//______________________________________________________________________________________________
+void AliShuttleTrigger::CheckTerminate()
+{
+ //
+ // Checks if the Shuttle got an external terminate request by a created file
+ // This is an alternative to the signal which causes problems with the API libraries
+ //
+
+ if (strlen(fConfig->GetTerminateFilePath()) == 0)
+ return;
+
+ if (gSystem->AccessPathName(fConfig->GetTerminateFilePath()) == kFALSE)
+ {
+ AliInfo("Terminate file exists. Terminating Shuttle...");
+ fTerminate = kTRUE;
+ }
+}
+
//______________________________________________________________________________________________
void AliShuttleTrigger::Run() {
//
fTerminate = kFALSE;
- DATENotifier* notifier = new DATENotifier(this, "/DATE/LOGBOOK/UPDATE");
+ DATENotifier* notifier = new DATENotifier(this, "/LOGBOOK/SUBSCRIBE/ECS_EOR");
+ Int_t nTry=0;
+ Int_t nMaxTry = fConfig->GetMaxRetries()+1;
+ Int_t received=0;
+
+ AliInfo("Listening for ECS trigger");
+
while (1) {
fMutex.Lock();
while (!(fNotified || fTerminate)) {
- fCondition.Wait();
+ for (Int_t iwait = 0; iwait < 10; iwait++){
+ received = fCondition.TimedWaitRelative(1000*fConfig->GetTriggerWait()/10); // to keep the connection to the server alive every minute while waiting for new runs
+ if (received == 1) {
+ if (gGrid) {
+ AliInfo(Form("Keeping the connection to the server alive while waiting for new runs - %d waited from last one", fConfig->GetTriggerWait()/10*iwait));
+ gGrid->Pwd();
+ }
+ else {
+ AliInfo("No gGrid initialized so far, we cannot keep the connection to the server alive while waiting for new runs");
+ }
+ }
+ }
+ //received=fCondition.TimedWaitRelative(1000*fConfig->GetTriggerWait());
+ CheckTerminate();
+ if (received==1) break; // 1 = timeout
}
fNotified = kFALSE;
AliInfo("Terminated.");
break;
}
+
+ if (received == 0)
+ {
+ AliInfo("Trigger from ECS received!");
+ } else if (received == 1) {
+ AliInfo(Form("Timeout (%d s) waiting for trigger. "
+ "Starting collection of new runs!",
+ fConfig->GetTriggerWait()));
+ } else {
+ AliInfo("Error receiving trigger from ECS!");
+ break;
+ }
+
+ nTry++;
+ AliInfo(Form("Received %d triggers so far", nTry));
+
+ if (fConfig->GetRunMode() == AliShuttleConfig::kTest)
+ {
+ if(nTry>=nMaxTry)
+ {
+ AliInfo(Form("Collect() ran more than %d times -> Exiting!",
+ nMaxTry));
+ break;
+ }
+ }
Collect();
+ CheckTerminate();
}
delete notifier;
// then it checks if the shuttle is still running by checking the monitoring functions of the shuttle
//
- return fShuttle->Collect(run);
+ // first checking disk space
+ Long_t id = 0;
+ Long_t bsize = 0;
+ Long_t blocks = 0;
+ Long_t bfree = 0;
+
+ gSystem->GetFsInfo(fConfig->GetShuttleFileSystem(), &id, &bsize, &blocks, &bfree);
+
+ AliInfo(Form("n. of free blocks = %ld, total n. of blocks = %ld",bfree,blocks));
+ Int_t spaceFree = (Int_t)(((Float_t)bfree/(Float_t)blocks)*100);
+
+ if (spaceFree < fConfig->GetFreeDiskWarningThreshold()) {
+ AliWarning(Form("************** Free space left = %d%%, below the Warning Threshold (%d%%)",spaceFree,fConfig->GetFreeDiskWarningThreshold()));
+ if (TMath::Abs(time(0) - fLastMailDiskSpace) >= 86400){ // 86400 = n. of seconds in 1 d
+ SendMailDiskSpace(fConfig->GetFreeDiskWarningThreshold());
+ fLastMailDiskSpace = time(0); // resetting fLastMailDiskSpace to time(0) = now
+ }
+ if (spaceFree < fConfig->GetFreeDiskFatalThreshold()){
+ AliError(Form("*************** Free space left = %d%%, below the Fatal Threshold (%d%%), terminating....",spaceFree,fConfig->GetFreeDiskFatalThreshold()));
+ SendMailDiskSpace(fConfig->GetFreeDiskFatalThreshold());
+ fTerminate = kTRUE; // terminating....
+ }
+ }
+
+ if (fTerminate) {
+ return kFALSE;
+ }
+
+ return fShuttle->Collect(run);
+}
+//______________________________________________________________________________________________
+Bool_t AliShuttleTrigger::SendMailDiskSpace(Short_t percentage)
+{
+ //
+ // sends a mail to the shuttle experts in case of free disk space < theshold
+ //
+
+
+ AliInfo("******************* Sending the Mail!! *********************");
+ if (!fConfig->SendMail())
+ return kTRUE;
+
+ Int_t runMode = (Int_t)fConfig->GetRunMode();
+ TString tmpStr;
+ if (runMode == 0) tmpStr = " Nightly Test:";
+ else tmpStr = " Data Taking:";
+ void* dir = gSystem->OpenDirectory(fShuttle->GetShuttleLogDir());
+ if (dir == NULL)
+ {
+ if (gSystem->mkdir(fShuttle->GetShuttleLogDir(), kTRUE))
+ {
+ AliWarning(Form("SendMail - Can't open directory <%s>", fShuttle->GetShuttleLogDir()));
+ return kFALSE;
+ }
+
+ } else {
+ gSystem->FreeDirectory(dir);
+ }
+
+ // SHUTTLE responsibles in to
+ TString to="";
+ TIter iterAdmins(fConfig->GetAdmins(AliShuttleConfig::kGlobal));
+ TObjString *anAdmin=0;
+ while ((anAdmin = (TObjString*) iterAdmins.Next()))
+ {
+ to += Form("%s,", anAdmin->GetName());
+ }
+ if (to.Length() > 0)
+ to.Remove(to.Length()-1);
+ AliDebug(2, Form("to: %s",to.Data()));
+
+ // mail body
+ TString bodyFileName;
+ bodyFileName.Form("%s/mail.body", fShuttle->GetShuttleLogDir());
+ gSystem->ExpandPathName(bodyFileName);
+
+ ofstream mailBody;
+ mailBody.open(bodyFileName, ofstream::out);
+
+ if (!mailBody.is_open())
+ {
+ AliWarning(Form("Could not open mail body file %s", bodyFileName.Data()));
+ return kFALSE;
+ }
+
+ TString subject;
+ TString body;
+
+ Int_t percentage_used = 100 - percentage;
+ subject = Form("%s CRITICAL Disk Space usage exceeds %d%c!",
+ tmpStr.Data(),percentage_used,'%');
+ AliDebug(2, Form("subject: %s", subject.Data()));
+
+ body = "Dear SHUTTLE experts, \n\n";
+ body += "The usage of the disk space on the shuttle machine has overcome \n";
+ body += Form("the threshold of %d%%. \n \n",percentage_used);
+ body += "Please check! \n \n";
+ body += "Please do not answer this message directly, it is automatically generated.\n\n";
+ body += "Greetings,\n\n \t\t\tthe SHUTTLE\n";
+
+ AliDebug(2, Form("Body : %s", body.Data()));
+
+ mailBody << body.Data();
+ mailBody.close();
+
+ // send mail!
+ TString mailCommand = Form("mail -s \"%s\" %s < %s",
+ subject.Data(),
+ to.Data(),
+ bodyFileName.Data());
+ AliDebug(2, Form("mail command: %s", mailCommand.Data()));
+
+ Bool_t result = gSystem->Exec(mailCommand.Data());
+
+ return result == 0;
}
git://git.uio.no / u / mrichter / AliRoot.git / blobdiff