/*
$Log$
+Revision 1.64 2007/11/13 16:15:47 acolla
+DCS map is stored in a file in the temp folder where the detector is processed.
+If the preprocessor fails, the temp folder is not removed. This will help the debugging of the problem.
+
+Revision 1.63 2007/11/02 10:53:16 acolla
+Protection added to AliShuttle::CopyFileLocally
+
+Revision 1.62 2007/10/31 18:23:13 acolla
+Furter developement on the Shuttle:
+
+- Shuttle now connects to the Grid as alidaq. The OCDB and Reference folders
+are now built from /alice/data, e.g.:
+/alice/data/2007/LHC07a/OCDB
+
+the year and LHC period are taken from the Shuttle.
+Raw metadata files are stored by GRP to:
+/alice/data/2007/LHC07a/<runNb>/Raw/RunMetadata.root
+
+- Shuttle sends a mail to DCS experts each time DP retrieval fails.
+
+Revision 1.61 2007/10/30 20:33:51 acolla
+Improved managing of temporary folders, which weren't correctly handled.
+Resolved bug introduced in StoreReferenceFile, which caused SPD preprocessor fail.
+
+Revision 1.60 2007/10/29 18:06:16 acolla
+
+New function StoreRunMetadataFile added to preprocessor and Shuttle interface
+This function can be used by GRP only. It stores raw data tags merged file to the
+raw data folder (e.g. /alice/data/2008/LHC08a/000099999/Raw).
+
+KNOWN ISSUES:
+
+1. Shuttle cannot write to /alice/data/ because it belongs to alidaq. Tag file is stored in /alice/simulation/... for the time being.
+2. Due to a bug in TAlien::Mkdir, the creation of a folder in recursive mode (-p option) does not work. The problem
+has been corrected in the root package on the Shuttle machine.
+
+Revision 1.59 2007/10/05 12:40:55 acolla
+
+Result error code added to AliDCSClient data members (it was "lost" with the new implementation of TMap* GetAliasValues and GetDPValues).
+
+Revision 1.58 2007/09/28 15:27:40 acolla
+
+AliDCSClient "multiSplit" option added in the DCS configuration
+in AliDCSMessage: variable MAX_BODY_SIZE set to 500000
+
Revision 1.57 2007/09/27 16:53:13 acolla
Detectors can have more than one AMANDA server. SHUTTLE queries the servers sequentially,
merges the dcs aliases/DPs in one TMap and sends it to the preprocessor.
return kFALSE;
}
- Log("SHUTTLE","Storing OCDB data ...");
+ Log("SHUTTLE","StoreOCDB - Storing OCDB data ...");
Bool_t resultCDB = StoreOCDB(fgkMainCDB);
- Log("SHUTTLE","Storing reference data ...");
+ Log("SHUTTLE","StoreOCDB - Storing reference data ...");
Bool_t resultRef = StoreOCDB(fgkMainRefStorage);
- Log("SHUTTLE","Storing reference files ...");
- Bool_t resultRefFiles = StoreRefFilesToGrid();
+ Log("SHUTTLE","StoreOCDB - Storing reference files ...");
+ Bool_t resultRefFiles = CopyFilesToGrid("reference");
- return resultCDB && resultRef && resultRefFiles;
+ Bool_t resultMetadata = kTRUE;
+ if(fCurrentDetector == "GRP")
+ {
+ Log("StoreOCDB - SHUTTLE","Storing Run Metadata file ...");
+ resultMetadata = CopyFilesToGrid("metadata");
+ }
+
+ return resultCDB && resultRef && resultRefFiles && resultMetadata;
}
//______________________________________________________________________________________________
// removing local filename...
TString filename;
localSto->IdToFilename(aLocId, filename);
- AliInfo(Form("Removing local file %s", filename.Data()));
+ Log("SHUTTLE", Form("StoreOCDB - Removing local file %s", filename.Data()));
RemoveFile(filename.Data());
continue;
} else {
TString targetDir = GetRefFilePrefix(localBaseFolder.Data(), detector);
- Log("SHUTTLE", Form("Cleaning %s", targetDir.Data()));
+ Log("SHUTTLE", Form("CleanReferenceStorage - Cleaning %s", targetDir.Data()));
TString begin;
begin.Form("%d_", GetCurrentRun());
if (result)
{
- Log("SHUTTLE", Form("Could not delete file %s!", fileName.Data()));
+ Log("SHUTTLE", Form("CleanReferenceStorage - Could not delete file %s!", fileName.Data()));
success = kFALSE;
} else {
nDel++;
if (result == 0)
{
// delete directory
- result = gSystem->Exec(Form("rm -r %s", targetDir.Data()));
+ result = gSystem->Exec(Form("rm -rf %s", targetDir.Data()));
if (result != 0)
{
- Log("SHUTTLE", Form("StoreReferenceFile - Could not clear directory %s", targetDir.Data()));
+ Log("SHUTTLE", Form("CleanReferenceStorage - Could not clean directory %s", targetDir.Data()));
return kFALSE;
}
}
result = gSystem->mkdir(targetDir, kTRUE);
if (result != 0)
{
- Log("SHUTTLE", Form("StoreReferenceFile - Error creating base directory %s", targetDir.Data()));
+ Log("SHUTTLE", Form("CleanReferenceStorage - Error creating base directory %s", targetDir.Data()));
return kFALSE;
}
TString localBaseFolder = sto->GetBaseFolder();
- TString targetDir = GetRefFilePrefix(localBaseFolder.Data(), detector);
+ TString target = GetRefFilePrefix(localBaseFolder.Data(), detector);
+ target.Append(Form("/%d_%s", GetCurrentRun(), gridFileName));
+
+ return CopyFileLocally(localFile, target);
+}
+
+//______________________________________________________________________________________________
+Bool_t AliShuttle::StoreRunMetadataFile(const char* localFile, const char* gridFileName)
+{
+ //
+ // Stores Run metadata file to the Grid, in the run folder
+ //
+ // Only GRP can call this function.
+
+ if (fTestMode & kErrorStorage)
+ {
+ Log(fCurrentDetector, "StoreRunMetaDataFile - In TESTMODE - Simulating error while storing locally");
+ return kFALSE;
+ }
+
+ AliCDBManager* man = AliCDBManager::Instance();
+ AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
+
+ TString localBaseFolder = sto->GetBaseFolder();
+
+ // Build Run level folder
+ // folder = /alice/data/year/lhcPeriod/runNb/Raw
+
+
+ TString lhcPeriod = GetLHCPeriod();
+ if (lhcPeriod.Length() == 0)
+ {
+ Log("SHUTTLE","StoreRunMetaDataFile - LHCPeriod not found in logbook!");
+ return 0;
+ }
+
+ TString target = Form("%s/GRP/RunMetadata/alice/data/%d/%s/%09d/Raw/%s",
+ localBaseFolder.Data(), GetCurrentYear(),
+ lhcPeriod.Data(), GetCurrentRun(), gridFileName);
+
+ return CopyFileLocally(localFile, target);
+}
+
+//______________________________________________________________________________________________
+Bool_t AliShuttle::CopyFileLocally(const char* localFile, const TString& target)
+{
+ //
+ // Stores file locally. Called by StoreReferenceFile and StoreRunMetadataFile
+ // Files are temporarily stored in the local reference storage. When the preprocessor
+ // finishes, the Shuttle calls CopyFilesToGrid to transfer the files to AliEn
+ // (in reference or run level folders)
+ //
+
+ TString targetDir(target(0, target.Last('/')));
- //try to open folder, if does not exist
+ //try to open base dir folder, if it does not exist
void* dir = gSystem->OpenDirectory(targetDir.Data());
if (dir == NULL) {
if (gSystem->mkdir(targetDir.Data(), kTRUE)) {
- Log("SHUTTLE", Form("Can't open directory <%s>", targetDir.Data()));
+ Log("SHUTTLE", Form("StoreFileLocally - Can't open directory <%s>", targetDir.Data()));
return kFALSE;
}
} else {
gSystem->FreeDirectory(dir);
}
-
- TString target;
- target.Form("%s/%d_%s", targetDir.Data(), GetCurrentRun(), gridFileName);
- Int_t result = gSystem->GetPathInfo(localFile, 0, (Long64_t*) 0, 0, 0);
+ Int_t result = 0;
+
+ result = gSystem->GetPathInfo(localFile, 0, (Long64_t*) 0, 0, 0);
if (result)
{
- Log("SHUTTLE", Form("StoreReferenceFile - %s does not exist", localFile));
+ Log("SHUTTLE", Form("StoreFileLocally - %s does not exist", localFile));
return kFALSE;
}
+ result = gSystem->GetPathInfo(target, 0, (Long64_t*) 0, 0, 0);
+ if (!result)
+ {
+ Log("SHUTTLE", Form("StoreFileLocally - target file %s already exist, removing...", target.Data()));
+ if (gSystem->Unlink(target.Data()))
+ {
+ Log("SHUTTLE", Form("StoreFileLocally - Could not remove existing target file %s!", target.Data()));
+ return kFALSE;
+ }
+ }
+
result = gSystem->CopyFile(localFile, target);
if (result == 0)
{
- Log("SHUTTLE", Form("StoreReferenceFile - File %s stored locally to %s", localFile, target.Data()));
+ Log("SHUTTLE", Form("StoreFileLocally - File %s stored locally to %s", localFile, target.Data()));
return kTRUE;
}
else
{
- Log("SHUTTLE", Form("StoreReferenceFile - Could not store file %s to %s!. Error code = %d",
+ Log("SHUTTLE", Form("StoreFileLocally - Could not store file %s to %s! Error code = %d",
localFile, target.Data(), result));
return kFALSE;
}
+
+
+
}
//______________________________________________________________________________________________
-Bool_t AliShuttle::StoreRefFilesToGrid()
+Bool_t AliShuttle::CopyFilesToGrid(const char* type)
{
//
- // Transfers the reference file to the Grid.
+ // Transfers local files to the Grid. Local files can be reference files
+ // or run metadata file (from GRP only).
//
- // The files are stored under the following location:
- // <base folder of reference storage>/<DET>/<RUN#>_<gridFileName>
+ // According to the type (ref, metadata) the files are stored under the following location:
+ // ref --> <base folder of reference storage>/<DET>/<RUN#>_<gridFileName>
+ // metadata --> <run data folder>/<MetadataFileName>
//
-
+
AliCDBManager* man = AliCDBManager::Instance();
AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
if (!sto)
return kFALSE;
TString localBaseFolder = sto->GetBaseFolder();
-
- TString dir = GetRefFilePrefix(localBaseFolder.Data(), fCurrentDetector.Data());
-
- AliCDBStorage* gridSto = man->GetStorage(fgkMainRefStorage);
- if (!gridSto)
- return kFALSE;
-
- TString gridBaseFolder = gridSto->GetBaseFolder();
-
- TString alienDir = GetRefFilePrefix(gridBaseFolder.Data(), fCurrentDetector.Data());
+ TString dir;
+ TString alienDir;
TString begin;
- begin.Form("%d_", GetCurrentRun());
+ if (strcmp(type, "reference") == 0)
+ {
+ dir = GetRefFilePrefix(localBaseFolder.Data(), fCurrentDetector.Data());
+ AliCDBStorage* gridSto = man->GetStorage(fgkMainRefStorage);
+ if (!gridSto)
+ return kFALSE;
+ TString gridBaseFolder = gridSto->GetBaseFolder();
+ alienDir = GetRefFilePrefix(gridBaseFolder.Data(), fCurrentDetector.Data());
+ begin = Form("%d_", GetCurrentRun());
+ }
+ else if (strcmp(type, "metadata") == 0)
+ {
+
+ TString lhcPeriod = GetLHCPeriod();
+
+ if (lhcPeriod.Length() == 0)
+ {
+ Log("SHUTTLE","CopyFilesToGrid - LHCPeriod not found in logbook!");
+ return 0;
+ }
+
+ dir = Form("%s/GRP/RunMetadata/alice/data/%d/%s/%09d/Raw",
+ localBaseFolder.Data(), GetCurrentYear(),
+ lhcPeriod.Data(), GetCurrentRun());
+ alienDir = dir(dir.Index("/alice/data/"), dir.Length());
+
+ begin = "";
+ }
+ else
+ {
+ Log("SHUTTLE", "CopyFilesToGrid - Unexpected: type label must be reference or metadata!");
+ return kFALSE;
+ }
+
TSystemDirectory* baseDir = new TSystemDirectory("/", dir);
if (!baseDir)
return kTRUE;
if (!gGrid)
{
- Log("SHUTTLE", "Connection to Grid failed: Cannot continue!");
+ Log("SHUTTLE", "CopyFilesToGrid - Connection to Grid failed: Cannot continue!");
delete dirList;
return kFALSE;
}
if (first)
{
first = kFALSE;
- // check that DET folder exists, otherwise create it
+ // check that folder exists, otherwise create it
TGridResult* result = gGrid->Ls(alienDir.Data(), "a");
if (!result)
if (!result->GetFileName(1)) // TODO: It looks like element 0 is always 0!!
{
- if (!gGrid->Mkdir(alienDir.Data(),"",0))
+ // TODO It does not work currently! Bug in TAliEn::Mkdir
+ // TODO Manually fixed in local root v5-16-00
+ if (!gGrid->Mkdir(alienDir.Data(),"-p",0))
{
- Log("SHUTTLE", Form("StoreRefFilesToGrid - Cannot create directory %s",
+ Log("SHUTTLE", Form("CopyFilesToGrid - Cannot create directory %s",
alienDir.Data()));
delete dirList;
return kFALSE;
} else {
- Log("SHUTTLE",Form("Folder %s created", alienDir.Data()));
+ Log("SHUTTLE",Form("CopyFilesToGrid - Folder %s created", alienDir.Data()));
}
} else {
- Log("SHUTTLE",Form("Folder %s found", alienDir.Data()));
+ Log("SHUTTLE",Form("CopyFilesToGrid - Folder %s found", alienDir.Data()));
}
}
if (result)
{
- Log("SHUTTLE", Form("StoreRefFilesToGrid - Copying local file %s to %s succeeded!", fullLocalPath.Data(), fullGridPath.Data()));
+ Log("SHUTTLE", Form("CopyFilesToGrid - Copying local file %s to %s succeeded!",
+ fullLocalPath.Data(), fullGridPath.Data()));
RemoveFile(fullLocalPath);
nTransfer++;
}
else
{
- Log("SHUTTLE", Form("StoreRefFilesToGrid - Copying local file %s to %s FAILED!", fullLocalPath.Data(), fullGridPath.Data()));
+ Log("SHUTTLE", Form("CopyFilesToGrid - Copying local file %s to %s FAILED!",
+ fullLocalPath.Data(), fullGridPath.Data()));
success = kFALSE;
}
}
- Log("SHUTTLE", Form("StoreRefFilesToGrid - %d (over %d) reference files in folder %s copied to Grid.", nTransfer, nDirs, dir.Data()));
+ Log("SHUTTLE", Form("CopyFilesToGrid - %d (over %d) files in folder %s copied to Grid.",
+ nTransfer, nDirs, dir.Data()));
delete dirList;
}
+
//______________________________________________________________________________________________
void AliShuttle::CleanLocalStorage(const TString& uri)
{
TString filename(Form("%s/%s/*/Run*_v%d_s*.root",
localSto->GetBaseFolder().Data(), GetOfflineDetName(fCurrentDetector.Data()), GetCurrentRun()));
- AliInfo(Form("filename = %s", filename.Data()));
+ AliDebug(2, Form("filename = %s", filename.Data()));
- AliInfo(Form("Removing remaining local files from run %d and detector %s ...",
+ Log("SHUTTLE", Form("Removing remaining local files for run %d and detector %s ...",
GetCurrentRun(), fCurrentDetector.Data()));
RemoveFile(filename.Data());
AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
if (!status){
- Log("SHUTTLE", "UNEXPECTED: status could not be read from current CDB entry");
+ Log("SHUTTLE", "UpdateShuttleStatus - UNEXPECTED: status could not be read from current CDB entry");
return;
}
dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
if (!aPreprocessor)
{
- AliInfo(Form("%s: no preprocessor registered", fCurrentDetector.Data()));
+ Log("SHUTTLE", Form("ContinueProcessing - %s: no preprocessor registered", fCurrentDetector.Data()));
return kFALSE;
}
fLogbookEntry->GetDetectorStatus(fCurrentDetector);
if(entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
- AliInfo(Form("ContinueProcessing - %s is %s",
+ Log("SHUTTLE", Form("ContinueProcessing - %s is %s",
fCurrentDetector.Data(),
fLogbookEntry->GetDetectorStatusName(entryStatus)));
return kFALSE;
{
if (fTestMode == kNone)
{
- Log("SHUTTLE", Form("ContinueProcessing - %s requires strict run ordering but this is not the first unprocessed run!"));
+ Log("SHUTTLE", Form("ContinueProcessing - %s requires strict run ordering"
+ " but this is not the first unprocessed run!"));
return kFALSE;
}
else
{
- Log("SHUTTLE", Form("ContinueProcessing - In TESTMODE - Although %s requires strict run ordering and this is not the first unprocessed run, the SHUTTLE continues"));
+ Log("SHUTTLE", Form("ContinueProcessing - In TESTMODE - "
+ "Although %s requires strict run ordering "
+ "and this is not the first unprocessed run, "
+ "the SHUTTLE continues"));
}
}
if (status->GetStatus() == AliShuttleStatus::kStoreError) {
Log("SHUTTLE",
- Form("ContinueProcessing - %s: Grid storage of one or more objects failed. Trying again now",
+ Form("ContinueProcessing - %s: Grid storage of one or more "
+ "objects failed. Trying again now",
fCurrentDetector.Data()));
UpdateShuttleStatus(AliShuttleStatus::kStoreStarted);
if (StoreOCDB()){
- Log("SHUTTLE", Form("ContinueProcessing - %s: all objects successfully stored into main storage",
+ Log("SHUTTLE", Form("ContinueProcessing - %s: all objects "
+ "successfully stored into main storage",
fCurrentDetector.Data()));
UpdateShuttleStatus(AliShuttleStatus::kDone);
UpdateShuttleLogbook(fCurrentDetector.Data(), "DONE");
// UpdateTableFailCase();
// Send mail to detector expert!
- AliInfo(Form("Sending mail to %s expert...", fCurrentDetector.Data()));
+ Log("SHUTTLE", Form("ContinueProcessing - Sending mail to %s expert...",
+ fCurrentDetector.Data()));
if (!SendMail())
Log("SHUTTLE", Form("ContinueProcessing - Could not send mail to %s expert",
fCurrentDetector.Data()));
"Aborted before with %s. Retry number %d.", fCurrentDetector.Data(),
status->GetStatusName(), status->GetCount()));
Bool_t increaseCount = kTRUE;
- if (status->GetStatus() == AliShuttleStatus::kDCSError || status->GetStatus() == AliShuttleStatus::kDCSStarted)
- increaseCount = kFALSE;
+ if (status->GetStatus() == AliShuttleStatus::kDCSError ||
+ status->GetStatus() == AliShuttleStatus::kDCSStarted)
+ increaseCount = kFALSE;
+
UpdateShuttleStatus(AliShuttleStatus::kStarted, increaseCount);
cont = kTRUE;
}
fLogbookEntry = entry;
- AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^* \n",
+ Log("SHUTTLE", Form("\t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^*",
GetCurrentRun()));
// create ML instance that monitors this run
- fMonaLisa = new TMonaLisaWriter(Form("%d", GetCurrentRun()), "SHUTTLE", "aliendb1.cern.ch");
+ fMonaLisa = new TMonaLisaWriter(Form("%d", GetCurrentRun()), fConfig->GetMonitorTable(),
+ fConfig->GetMonitorHost());
// disable monitoring of other parameters that come e.g. from TFile
gMonitoringWriter = 0;
Int_t testMode = tmpStr->String().Atoi();
if (testMode > 0)
{
- Log("SHUTTLE", Form("Enabling test mode %d", testMode));
+ Log("SHUTTLE", Form("Process - Enabling test mode %d", testMode));
SetTestMode((TestMode) testMode);
}
}
}
}
}
-
- Log("SHUTTLE", Form("The test mode flag is %d", (Int_t) fTestMode));
-
+
fLogbookEntry->Print("all");
// Initialization
Bool_t hasError = kFALSE;
+ // Set the CDB and Reference folders according to the year and LHC period
+ TString lhcPeriod(GetLHCPeriod());
+ if (lhcPeriod.Length() == 0)
+ {
+ Log("SHUTTLE","StoreRunMetaDataFile - LHCPeriod not found in logbook!");
+ return 0;
+ }
+
+ if (fgkMainCDB.Length() == 0)
+ fgkMainCDB = Form("alien://folder=/alice/data/%d/%s/OCDB?user=alidaq?cacheFold=/tmp/OCDBCache",
+ GetCurrentYear(), lhcPeriod.Data());
+
+ if (fgkMainRefStorage.Length() == 0)
+ fgkMainRefStorage = Form("alien://folder=/alice/data/%d/%s/Reference?user=alidaq?cacheFold=/tmp/OCDBCache",
+ GetCurrentYear(), lhcPeriod.Data());
+
AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
AliCDBStorage *mainRefSto = AliCDBManager::Instance()->GetStorage(fgkMainRefStorage);
if (ContinueProcessing() == kFALSE) continue;
- AliInfo(Form("\n\n \t\t\t****** run %d - %s: START ******",
+ Log("SHUTTLE", Form("\t\t\t****** run %d - %s: START ******",
GetCurrentRun(), aDetector->GetName()));
for(Int_t iSys=0;iSys<3;iSys++) fFXSCalled[iSys]=kFALSE;
- Log(fCurrentDetector.Data(), "Starting processing");
+ Log(fCurrentDetector.Data(), "Process - Starting processing");
Int_t pid = fork();
if (pid < 0)
{
- Log("SHUTTLE", "ERROR: Forking failed");
+ Log("SHUTTLE", "Process - ERROR: Forking failed");
}
else if (pid > 0)
{
// parent
- AliInfo(Form("In parent process of %d - %s: Starting monitoring",
+ Log("SHUTTLE", Form("Process - In parent process of %d - %s: Starting monitoring",
GetCurrentRun(), aDetector->GetName()));
Long_t begin = time(0);
if (expiredTime > fConfig->GetPPTimeOut())
{
TString tmp;
- tmp.Form("Process of %s time out. Run time: %d seconds. Killing...",
- fCurrentDetector.Data(), expiredTime);
+ tmp.Form("Process - Process of %s time out. "
+ "Run time: %d seconds. Killing...",
+ fCurrentDetector.Data(), expiredTime);
Log("SHUTTLE", tmp);
Log(fCurrentDetector, tmp);
FILE* pipe = gSystem->OpenPipe(checkStr, "r");
if (!pipe)
{
- Log("SHUTTLE", Form("Error: Could not open pipe to %s", checkStr.Data()));
+ Log("SHUTTLE", Form("Process - Error: "
+ "Could not open pipe to %s", checkStr.Data()));
continue;
}
char buffer[100];
if (!fgets(buffer, 100, pipe))
{
- Log("SHUTTLE", "Error: ps did not return anything");
+ Log("SHUTTLE", "Process - Error: ps did not return anything");
gSystem->ClosePipe(pipe);
continue;
}
Int_t mem = 0;
if ((sscanf(buffer, "%d\n", &mem) != 1) || !mem)
{
- Log("SHUTTLE", "Error: Could not parse output of ps");
+ Log("SHUTTLE", "Process - Error: Could not parse output of ps");
continue;
}
if (expiredTime % 60 == 0)
- Log("SHUTTLE", Form("%s: Checking process. Run time: %d seconds - Memory consumption: %d KB",
- fCurrentDetector.Data(), expiredTime, mem));
+ Log("SHUTTLE", Form("Process - %s: Checking process. "
+ "Run time: %d seconds - Memory consumption: %d KB",
+ fCurrentDetector.Data(), expiredTime, mem));
if (mem > fConfig->GetPPMaxMem())
{
TString tmp;
- tmp.Form("Process exceeds maximum allowed memory (%d KB > %d KB). Killing...",
+ tmp.Form("Process - Process exceeds maximum allowed memory "
+ "(%d KB > %d KB). Killing...",
mem, fConfig->GetPPMaxMem());
Log("SHUTTLE", tmp);
Log(fCurrentDetector, tmp);
}
}
- AliInfo(Form("In parent process of %d - %s: Client has terminated.",
+ Log("SHUTTLE", Form("Process - In parent process of %d - %s: Client has terminated.",
GetCurrentRun(), aDetector->GetName()));
if (WIFEXITED(status))
{
Int_t returnCode = WEXITSTATUS(status);
- Log("SHUTTLE", Form("%s: the return code is %d", fCurrentDetector.Data(),
+ Log("SHUTTLE", Form("Process - %s: the return code is %d", fCurrentDetector.Data(),
returnCode));
if (returnCode == 0) hasError = kTRUE;
else if (pid == 0)
{
// client
- AliInfo(Form("In client process of %d - %s", GetCurrentRun(), aDetector->GetName()));
+ Log("SHUTTLE", Form("Process - In client process of %d - %s", GetCurrentRun(),
+ aDetector->GetName()));
- AliInfo("Redirecting output...");
+ Log("SHUTTLE", Form("Process - Redirecting output to %s log",fCurrentDetector.Data()));
if ((freopen(GetLogFileName(fCurrentDetector), "a", stdout)) == 0)
{
- Log("SHUTTLE", "Could not freopen stdout");
+ Log("SHUTTLE", "Process - Could not freopen stdout");
}
else
{
fOutputRedirected = kTRUE;
if ((dup2(fileno(stdout), fileno(stderr))) < 0)
- Log("SHUTTLE", "Could not redirect stderr");
+ Log("SHUTTLE", "Process - Could not redirect stderr");
}
TString wd = gSystem->WorkingDirectory();
- TString tmpDir = Form("%s/%s_process",GetShuttleTempDir(),fCurrentDetector.Data());
+ TString tmpDir = Form("%s/%s_%d_process", GetShuttleTempDir(),
+ fCurrentDetector.Data(), GetCurrentRun());
- gSystem->mkdir(tmpDir.Data());
- gSystem->ChangeDirectory(tmpDir.Data());
+ Int_t result = gSystem->GetPathInfo(tmpDir.Data(), 0, (Long64_t*) 0, 0, 0);
+ if (!result) // temp dir already exists!
+ {
+ Log(fCurrentDetector.Data(),
+ Form("Process - %s dir already exists! Removing...", tmpDir.Data()));
+ gSystem->Exec(Form("rm -rf %s",tmpDir.Data()));
+ }
- Bool_t success = ProcessCurrentDetector();
+ if (gSystem->mkdir(tmpDir.Data(), 1))
+ {
+ Log(fCurrentDetector.Data(), "Process - could not make temp directory!!");
+ gSystem->Exit(1);
+ }
- gSystem->ChangeDirectory(wd.Data());
+ if (!gSystem->ChangeDirectory(tmpDir.Data()))
+ {
+ Log(fCurrentDetector.Data(), "Process - could not change directory!!");
+ gSystem->Exit(1);
+ }
- gSystem->Exec(Form("rm -rf %s",tmpDir.Data()));
+ Bool_t success = ProcessCurrentDetector();
+ gSystem->ChangeDirectory(wd.Data());
+
if (success) // Preprocessor finished successfully!
{
+ // remove temporary folder
+ gSystem->Exec(Form("rm -rf %s",tmpDir.Data()));
+
// Update time_processed field in FXS DB
if (UpdateTable() == kFALSE)
Log("SHUTTLE", Form("Process - %s: Could not update FXS databases!",
UpdateShuttleStatus(AliShuttleStatus::kStoreStarted);
if (StoreOCDB() == kFALSE)
{
- AliInfo(Form("\n \t\t\t****** run %d - %s: STORAGE ERROR ****** \n\n",
+ Log("SHUTTLE",
+ Form("\t\t\t****** run %d - %s: STORAGE ERROR ******",
GetCurrentRun(), aDetector->GetName()));
UpdateShuttleStatus(AliShuttleStatus::kStoreError);
success = kFALSE;
} else {
- AliInfo(Form("\n \t\t\t****** run %d - %s: DONE ****** \n\n",
+ Log("SHUTTLE",
+ Form("\t\t\t****** run %d - %s: DONE ******",
GetCurrentRun(), aDetector->GetName()));
UpdateShuttleStatus(AliShuttleStatus::kDone);
UpdateShuttleLogbook(fCurrentDetector, "DONE");
}
+ } else
+ {
+ Log("SHUTTLE",
+ Form("\t\t\t****** run %d - %s: PP ERROR ******",
+ GetCurrentRun(), aDetector->GetName()));
}
for (UInt_t iSys=0; iSys<3; iSys++)
if (fFXSCalled[iSys]) fFXSlist[iSys].Clear();
}
- AliInfo(Form("Client process of %d - %s is exiting now with %d.",
+ Log("SHUTTLE", Form("Process - Client process of %d - %s is exiting now with %d.",
GetCurrentRun(), aDetector->GetName(), success));
// the client exits here
}
}
- AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^* \n",
+ Log("SHUTTLE", Form("\t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: FINISH ^*^*^*^*^*^*^*^*^*^*^*^*",
GetCurrentRun()));
//check if shuttle is done for this run, if so update logbook
Log("SHUTTLE", Form("ProcessCurrentDetector - Retrieving values for %s, run %d",
fCurrentDetector.Data(), GetCurrentRun()));
+ TString wd = gSystem->WorkingDirectory();
+
if (!CleanReferenceStorage(fCurrentDetector.Data()))
return kFALSE;
-
+
+ gSystem->ChangeDirectory(wd.Data());
+
TMap* dcsMap = new TMap();
// call preprocessor
// Query DCS archive
Int_t nServers = fConfig->GetNServers(fCurrentDetector);
- Log("SHUTTLE", Form("ProcessCurrentDetector -"
- " found %d Amanda servers for %s", nServers, fCurrentDetector.Data()));
for (int iServ=0; iServ<nServers; iServ++)
{
Int_t port = fConfig->GetDCSPort(fCurrentDetector, iServ);
Int_t multiSplit = fConfig->GetMultiSplit(fCurrentDetector, iServ);
+ Log(fCurrentDetector, Form("ProcessCurrentDetector -"
+ " Querying DCS Amanda server %s:%d (%d of %d)",
+ host.Data(), port, iServ+1, nServers));
TMap* aliasMap = 0;
TMap* dpMap = 0;
{
Log(fCurrentDetector,
Form("ProcessCurrentDetector -"
- " Error retrieving DCS aliases from server %s",
- host.Data()));
+ " Error retrieving DCS aliases from server %s."
+ " Sending mail to DCS experts!", host.Data()));
UpdateShuttleStatus(AliShuttleStatus::kDCSError);
+
+ if (!SendMailToDCS())
+ Log("SHUTTLE", Form("ProcessCurrentDetector - Could not send mail to DCS experts!"));
+
delete dcsMap;
return kFALSE;
}
{
Log(fCurrentDetector,
Form("ProcessCurrentDetector -"
- " Error retrieving DCS data points from server %s",
- host.Data()));
+ " Error retrieving DCS data points from server %s."
+ " Sending mail to DCS experts!", host.Data()));
UpdateShuttleStatus(AliShuttleStatus::kDCSError);
+
+ if (!SendMailToDCS())
+ Log("SHUTTLE", Form("ProcessCurrentDetector - Could not send mail to DCS experts!"));
+
if (aliasMap) delete aliasMap;
delete dcsMap;
return kFALSE;
}
}
+ // save map into file, to help debugging in case of preprocessor error
+ TFile* f = TFile::Open("DCSMap.root","recreate");
+ f->cd();
+ dcsMap->Write("DCSMap", TObject::kSingleKey);
+ f->Close();
+ delete f;
+
// DCS Archive DB processing successful. Call Preprocessor!
UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
if (returnValue > 0) // Preprocessor error!
{
- Log(fCurrentDetector, Form("Preprocessor failed. Process returned %d.", returnValue));
+ Log(fCurrentDetector, Form("ProcessCurrentDetector - "
+ "Preprocessor failed. Process returned %d.", returnValue));
UpdateShuttleStatus(AliShuttleStatus::kPPError);
dcsMap->DeleteAll();
delete dcsMap;
AliDebug(2,Form("Query = %s", sqlQuery.Data()));
if(aResult->GetRowCount() == 0) {
- AliInfo("No entries in Shuttle Logbook match request");
+ Log("SHUTTLE", "No entries in Shuttle Logbook match request");
delete aResult;
return kTRUE;
}
// TODO Check field count!
const UInt_t nCols = 23;
if (aResult->GetFieldCount() != (Int_t) nCols) {
- AliError("Invalid SQL result field number!");
+ Log("SHUTTLE", "Invalid SQL result field number!");
delete aResult;
return kFALSE;
}
TSQLResult* aResult = fServer[3]->Query(sqlQuery);
if (!aResult) {
- AliError(Form("Can't execute query <%s>!", sqlQuery.Data()));
+ Log("SHUTTLE", Form("Can't execute query <%s>!", sqlQuery.Data()));
return 0;
}
}
if (aResult->GetRowCount() > 1) {
- AliError(Form("More than one entry in DAQ Logbook for run %d. Skipping", run));
+ Log("SHUTTLE", Form("QueryRunParameters - UNEXPECTED: "
+ "more than one entry in DAQ Logbook for run %d!", run));
delete aResult;
return 0;
}
TSQLRow* aRow = aResult->Next();
if (!aRow)
{
- AliError(Form("Could not retrieve row for run %d. Skipping", run));
+ Log("SHUTTLE", Form("QueryRunParameters - Could not retrieve row for run %d. Skipping", run));
delete aResult;
return 0;
}
if (result == 0)
{
Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get entries! Reason: %s",
- client.GetServerError().Data()));
+ client.GetErrorString(client.GetResultErrorCode())));
+ if (client.GetResultErrorCode() == AliDCSClient::fgkServerError)
+ Log(fCurrentDetector.Data(), Form("GetValueSet - Server error code: %s",
+ client.GetServerError().Data()));
return 0;
}
filePath.Data(), fileSize.Data(), fileChecksum.Data()));
// retrieved file is renamed to make it unique
- TString localFileName = Form("%s_%s_%d_%s_%s.shuttle",
- GetSystemName(system), detector, GetCurrentRun(), id, sourceName.Data());
+ TString localFileName = Form("%s/%s_%d_process/%s_%s_%d_%s_%s.shuttle",
+ GetShuttleTempDir(), detector, GetCurrentRun(),
+ GetSystemName(system), detector, GetCurrentRun(),
+ id, sourceName.Data());
// file retrieval from FXS
if (fileChecksum.Length()>0)
{
// compare md5sum of local file with the one stored in the FXS DB
- Int_t md5Comp = gSystem->Exec(Form("md5sum %s/%s |grep %s 2>&1 > /dev/null",
- GetShuttleTempDir(), localFileName.Data(), fileChecksum.Data()));
+ Int_t md5Comp = gSystem->Exec(Form("md5sum %s |grep %s 2>&1 > /dev/null",
+ localFileName.Data(), fileChecksum.Data()));
if (md5Comp != 0)
{
TObjString *fileParams = new TObjString(Form("%s#!?!#%s", id, sourceName.Data()));
fFXSlist[system].Add(fileParams);
- static TString fullLocalFileName;
- fullLocalFileName.Form("%s/%s", GetShuttleTempDir(), localFileName.Data());
-
- Log(fCurrentDetector, Form("GetFile - Retrieved file with id %s and source %s from %s to %s", id, source, GetSystemName(system), fullLocalFileName.Data()));
-
- return fullLocalFileName.Data();
+ static TString staticLocalFileName;
+ staticLocalFileName.Form("%s", localFileName.Data());
+
+ Log(fCurrentDetector, Form("GetFile - Retrieved file with id %s and "
+ "source %s from %s to %s", id, source,
+ GetSystemName(system), localFileName.Data()));
+
+ return staticLocalFileName.Data();
}
//______________________________________________________________________________________________
//
// check temp directory: trying to cd to temp; if it does not exist, create it
- AliDebug(2, Form("Copy file %s from %s FXS into %s/%s",
- GetSystemName(system), fxsFileName, GetShuttleTempDir(), localFileName));
+ AliDebug(2, Form("Copy file %s from %s FXS into %s",
+ GetSystemName(system), fxsFileName, localFileName));
+
+ TString tmpDir(localFileName);
+
+ tmpDir = tmpDir(0,tmpDir.Last('/'));
- void* dir = gSystem->OpenDirectory(GetShuttleTempDir());
- if (dir == NULL) {
- if (gSystem->mkdir(GetShuttleTempDir(), kTRUE)) {
- AliError(Form("Can't open directory <%s>", GetShuttleTempDir()));
+ Int_t noDir = gSystem->GetPathInfo(tmpDir.Data(), 0, (Long64_t*) 0, 0, 0);
+ if (noDir) // temp dir does not exists!
+ {
+ if (gSystem->mkdir(tmpDir.Data(), 1))
+ {
+ Log(fCurrentDetector.Data(), "RetrieveFile - could not make temp directory!!");
return kFALSE;
}
-
- } else {
- gSystem->FreeDirectory(dir);
}
TString baseFXSFolder;
}
- TString command = Form("scp -oPort=%d -2 %s@%s:%s%s %s/%s",
+ TString command = Form("scp -oPort=%d -2 %s@%s:%s%s %s",
fConfig->GetFXSPort(system),
fConfig->GetFXSUser(system),
fConfig->GetFXSHost(system),
baseFXSFolder.Data(),
fxsFileName,
- GetShuttleTempDir(),
localFileName);
AliDebug(2, Form("%s",command.Data()));
if (system == kDCS)
{
- AliWarning("DCS system has only one source of data!");
+ Log(detector, "GetFileSources - WARNING: DCS system has only one source of data!");
TList *list = new TList();
list->SetOwner(1);
list->Add(new TObjString(" "));
return fLogbookEntry ? fLogbookEntry->GetEndTime() : 0;
}
+//______________________________________________________________________________________________
+UInt_t AliShuttle::GetCurrentYear() const
+{
+ //
+ // Get current year from logbook entry
+ //
+
+ if (!fLogbookEntry) return 0;
+
+ TTimeStamp startTime(GetCurrentStartTime());
+ TString year = Form("%d",startTime.GetDate());
+ year = year(0,4);
+
+ return year.Atoi();
+}
+
+//______________________________________________________________________________________________
+const char* AliShuttle::GetLHCPeriod() const
+{
+ //
+ // Get current LHC period from logbook entry
+ //
+
+ if (!fLogbookEntry) return 0;
+
+ return fLogbookEntry->GetRunParameter("LHCperiod");
+}
+
//______________________________________________________________________________________________
void AliShuttle::Log(const char* detector, const char* message)
{
}
// clean SHUTTLE temp directory
- TString filename = Form("%s/*.shuttle", GetShuttleTempDir());
- RemoveFile(filename.Data());
+ //TString filename = Form("%s/*.shuttle", GetShuttleTempDir());
+ //RemoveFile(filename.Data());
}
return hasError == kFALSE;
{
if (gSystem->mkdir(GetShuttleLogDir(), kTRUE))
{
- AliError(Form("Can't open directory <%s>", GetShuttleLogDir()));
+ Log("SHUTTLE", Form("SendMail - Can't open directory <%s>", GetShuttleLogDir()));
return kFALSE;
}
if (!mailBody.is_open())
{
- AliError(Form("Could not open mail body file %s", bodyFileName.Data()));
+ Log("SHUTTLE", Form("Could not open mail body file %s", bodyFileName.Data()));
return kFALSE;
}
AliDebug(2, Form("to: %s",to.Data()));
if (to.IsNull()) {
- AliInfo("List of detector responsibles not yet set!");
+ Log("SHUTTLE", "List of detector responsibles not yet set!");
return kFALSE;
}
return result == 0;
}
+//______________________________________________________________________________________________
+Bool_t AliShuttle::SendMailToDCS()
+{
+ //
+ // sends a mail to the DCS experts in case of DCS error
+ //
+
+ if (fTestMode != kNone)
+ return kTRUE;
+
+ void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
+ if (dir == NULL)
+ {
+ if (gSystem->mkdir(GetShuttleLogDir(), kTRUE))
+ {
+ Log("SHUTTLE", Form("SendMailToDCS - Can't open directory <%s>", GetShuttleLogDir()));
+ return kFALSE;
+ }
+
+ } else {
+ gSystem->FreeDirectory(dir);
+ }
+
+ TString bodyFileName;
+ bodyFileName.Form("%s/mail.body", GetShuttleLogDir());
+ gSystem->ExpandPathName(bodyFileName);
+
+ ofstream mailBody;
+ mailBody.open(bodyFileName, ofstream::out);
+
+ if (!mailBody.is_open())
+ {
+ Log("SHUTTLE", Form("SendMailToDCS - Could not open mail body file %s", bodyFileName.Data()));
+ return kFALSE;
+ }
+
+ TString to="Vladimir.Fekete@cern.ch, Svetozar.Kapusta@cern.ch";
+ //TString to="alberto.colla@cern.ch";
+ AliDebug(2, Form("to: %s",to.Data()));
+
+ if (to.IsNull()) {
+ Log("SHUTTLE", "List of detector responsibles not yet set!");
+ return kFALSE;
+ }
+
+ TString cc="alberto.colla@cern.ch";
+
+ TString subject = Form("Retrieval of data points for %s FAILED in run %d !",
+ fCurrentDetector.Data(), GetCurrentRun());
+ AliDebug(2, Form("subject: %s", subject.Data()));
+
+ TString body = Form("Dear DCS experts, \n\n");
+ body += Form("SHUTTLE couldn\'t retrieve the data points for detector %s "
+ "in run %d!!\n\n", fCurrentDetector.Data(), GetCurrentRun());
+ body += Form("Please check %s status on the SHUTTLE monitoring page: \n\n", fCurrentDetector.Data());
+ body += Form("\thttp://pcalimonitor.cern.ch:8889/shuttle.jsp?time=168 \n\n");
+ body += Form("Find the %s log for the current run on \n\n"
+ "\thttp://pcalishuttle01.cern.ch:8880/logs/%s_%d.log \n\n",
+ fCurrentDetector.Data(), fCurrentDetector.Data(), GetCurrentRun());
+ body += Form("The last 10 lines of %s log file are following:\n\n");
+
+ AliDebug(2, Form("Body begin: %s", body.Data()));
+
+ mailBody << body.Data();
+ mailBody.close();
+ mailBody.open(bodyFileName, ofstream::out | ofstream::app);
+
+ TString logFileName = Form("%s/%s_%d.log", GetShuttleLogDir(), fCurrentDetector.Data(), GetCurrentRun());
+ TString tailCommand = Form("tail -n 10 %s >> %s", logFileName.Data(), bodyFileName.Data());
+ if (gSystem->Exec(tailCommand.Data()))
+ {
+ mailBody << Form("%s log file not found ...\n\n", fCurrentDetector.Data());
+ }
+
+ TString endBody = Form("------------------------------------------------------\n\n");
+ endBody += Form("In case of problems please contact the SHUTTLE core team.\n\n");
+ endBody += "Please do not answer this message directly, it is automatically generated.\n\n";
+ endBody += "Greetings,\n\n \t\t\tthe SHUTTLE\n";
+
+ AliDebug(2, Form("Body end: %s", endBody.Data()));
+
+ mailBody << endBody.Data();
+
+ mailBody.close();
+
+ // send mail!
+ TString mailCommand = Form("mail -s \"%s\" -c %s %s < %s",
+ subject.Data(),
+ cc.Data(),
+ to.Data(),
+ bodyFileName.Data());
+ AliDebug(2, Form("mail command: %s", mailCommand.Data()));
+
+ Bool_t result = gSystem->Exec(mailCommand.Data());
+
+ return result == 0;
+}
+
//______________________________________________________________________________________________
const char* AliShuttle::GetRunType()
{
git://git.uio.no / u / mrichter / AliRoot.git / blobdiff