/*
$Log$
+Revision 1.55 2007/08/06 12:26:40 acolla
+Function Bool_t GetHLTStatus added to preprocessor. It returns the status of HLT
+read from the run logbook.
+
+Revision 1.54 2007/07/12 09:51:25 jgrosseo
+removed duplicated log message in GetFile
+
+Revision 1.53 2007/07/12 09:26:28 jgrosseo
+updating hlt fxs base path
+
+Revision 1.52 2007/07/12 08:06:45 jgrosseo
+adding log messages in getfile... functions
+adding not implemented copy constructor in alishuttleconfigholder
+
+Revision 1.51 2007/07/03 17:24:52 acolla
+root moved to v5-16-00. TFileMerger->Cp moved to TFile::Cp.
+
+Revision 1.50 2007/07/02 17:19:32 acolla
+preprocessor is run in a temp directory that is removed when process is finished.
+
+Revision 1.49 2007/06/29 10:45:06 acolla
+Number of columns in MySql Shuttle logbook increased by one (HLT added)
+
+Revision 1.48 2007/06/21 13:06:19 acolla
+GetFileSources returns dummy list with 1 source if system=DCS (better than
+returning error as it was)
+
+Revision 1.47 2007/06/19 17:28:56 acolla
+HLT updated; missing map bug removed.
+
+Revision 1.46 2007/06/09 13:01:09 jgrosseo
+Switching to retrieval of several DCS DPs at a time (multiDPrequest)
+
+Revision 1.45 2007/05/30 06:35:20 jgrosseo
+Adding functionality to the Shuttle/TestShuttle:
+o) Function to retrieve list of sources from a given system (GetFileSources with id=0)
+o) Function to retrieve list of IDs for a given source (GetFileIDs)
+These functions are needed for dealing with the tag files that are saved for the GRP preprocessor
+Example code has been added to the TestProcessor in TestShuttle
+
+Revision 1.44 2007/05/11 16:09:32 acolla
+Reference files for ITS, MUON and PHOS are now stored in OfflineDetName/OnlineDetName/run_...
+example: ITS/SPD/100_filename.root
+
Revision 1.43 2007/05/10 09:59:51 acolla
Various bug fixes in StoreRefFilesToGrid; Cleaning of reference storage before processing detector (CleanReferenceStorage)
#include <TMutex.h>
#include <TSystemDirectory.h>
#include <TSystemFile.h>
+#include <TFile.h>
#include <TFileMerger.h>
#include <TGrid.h>
#include <TGridResult.h>
fullGridPath.Form("alien://%s/%s", alienDir.Data(), fileName.Data());
TFileMerger fileMerger;
- Bool_t result = fileMerger.Cp(fullLocalPath, fullGridPath);
+ Bool_t result = TFile::Cp(fullLocalPath, fullGridPath);
if (result)
{
}
+ TString wd = gSystem->WorkingDirectory();
+ TString tmpDir = Form("%s/%s_process",GetShuttleTempDir(),fCurrentDetector.Data());
+
+ gSystem->mkdir(tmpDir.Data());
+ gSystem->ChangeDirectory(tmpDir.Data());
+
Bool_t success = ProcessCurrentDetector();
+
+ gSystem->ChangeDirectory(wd.Data());
+
+ gSystem->Exec(Form("rm -rf %s",tmpDir.Data()));
+
if (success) // Preprocessor finished successfully!
{
// Update time_processed field in FXS DB
if (UpdateTable() == kFALSE)
- Log("SHUTTLE", Form("Process - %s: Could not update FXS databases!"));
+ Log("SHUTTLE", Form("Process - %s: Could not update FXS databases!",
+ fCurrentDetector.Data()));
// Transfer the data from local storage to main storage (Grid)
UpdateShuttleStatus(AliShuttleStatus::kStoreStarted);
if (!CleanReferenceStorage(fCurrentDetector.Data()))
return kFALSE;
- TMap dcsMap;
- dcsMap.SetOwner(1);
-
- Bool_t aDCSError = kFALSE;
+ TMap* dcsMap = 0;
// call preprocessor
AliPreprocessor* aPreprocessor =
TString host(fConfig->GetDCSHost(fCurrentDetector));
Int_t port = fConfig->GetDCSPort(fCurrentDetector);
- // Retrieval of Aliases
- TObjString* anAlias = 0;
- Int_t iAlias = 0;
- Int_t nTotAliases= ((TMap*)fConfig->GetDCSAliases(fCurrentDetector))->GetEntries();
- TIter iterAliases(fConfig->GetDCSAliases(fCurrentDetector));
- while ((anAlias = (TObjString*) iterAliases.Next()))
+ if (fConfig->GetDCSAliases(fCurrentDetector)->GetEntries() > 0)
{
- TObjArray *valueSet = new TObjArray();
- valueSet->SetOwner(1);
-
- iAlias++;
- aDCSError = (GetValueSet(host, port, anAlias->String(), valueSet, kAlias) == 0);
-
- if(!aDCSError)
+ dcsMap = GetValueSet(host, port, fConfig->GetDCSAliases(fCurrentDetector), kAlias);
+ if (!dcsMap)
{
- if (((iAlias-1) % 500) == 0 || iAlias == nTotAliases)
- AliInfo(Form("Alias %s (%d of %d) - %d values collected",
- anAlias->GetName(), iAlias, nTotAliases, valueSet->GetEntriesFast()));
- dcsMap.Add(anAlias->Clone(), valueSet);
- } else {
- Log(fCurrentDetector,
- Form("ProcessCurrentDetector - Error while retrieving alias %s",
- anAlias->GetName()));
+ Log(fCurrentDetector, "ProcessCurrentDetector - Error while retrieving DCS aliases");
UpdateShuttleStatus(AliShuttleStatus::kDCSError);
- dcsMap.DeleteAll();
return kFALSE;
}
}
-
- // Retrieval of Data Points
- TObjString* aDP = 0;
- Int_t iDP = 0;
- Int_t nTotDPs= ((TMap*)fConfig->GetDCSDataPoints(fCurrentDetector))->GetEntries();
- TIter iterDP(fConfig->GetDCSDataPoints(fCurrentDetector));
- while ((aDP = (TObjString*) iterDP.Next()))
+
+ if (fConfig->GetDCSDataPoints(fCurrentDetector)->GetEntries() > 0)
{
- TObjArray *valueSet = new TObjArray();
- valueSet->SetOwner(1);
- if (((iDP-1) % 500) == 0 || iDP == nTotDPs)
- AliInfo(Form("Querying DCS archive: DP %s (%d of %d)",
- aDP->GetName(), iDP++, nTotDPs));
- aDCSError = (GetValueSet(host, port, aDP->String(), valueSet, kDP) == 0);
-
- if(!aDCSError)
+ TMap* dcsMap2 = GetValueSet(host, port, fConfig->GetDCSDataPoints(fCurrentDetector), kDP);
+ if (!dcsMap2)
{
- dcsMap.Add(aDP->Clone(), valueSet);
- } else {
- Log(fCurrentDetector,
- Form("ProcessCurrentDetector - Error while retrieving data point %s",
- aDP->GetName()));
+ Log(fCurrentDetector, "ProcessCurrentDetector - Error while retrieving DCS data points");
UpdateShuttleStatus(AliShuttleStatus::kDCSError);
- dcsMap.DeleteAll();
+ if (dcsMap)
+ delete dcsMap;
return kFALSE;
}
+
+ if (!dcsMap)
+ {
+ dcsMap = dcsMap2;
+ }
+ else // merge
+ {
+ TIter iter(dcsMap2);
+ TObjString* key = 0;
+ while ((key = (TObjString*) iter.Next()))
+ dcsMap->Add(key, dcsMap2->GetValue(key->String()));
+
+ dcsMap2->SetOwner(kFALSE);
+ delete dcsMap2;
+ }
}
+
}
+ // still no map?
+ if (!dcsMap)
+ dcsMap = new TMap;
+
// DCS Archive DB processing successful. Call Preprocessor!
UpdateShuttleStatus(AliShuttleStatus::kPPStarted);
- UInt_t returnValue = aPreprocessor->Process(&dcsMap);
+ UInt_t returnValue = aPreprocessor->Process(dcsMap);
if (returnValue > 0) // Preprocessor error!
{
Log(fCurrentDetector, Form("Preprocessor failed. Process returned %d.", returnValue));
UpdateShuttleStatus(AliShuttleStatus::kPPError);
- dcsMap.DeleteAll();
+ dcsMap->DeleteAll();
+ delete dcsMap;
return kFALSE;
}
Log(fCurrentDetector, Form("ProcessCurrentDetector - %s preprocessor returned success",
fCurrentDetector.Data()));
- dcsMap.DeleteAll();
+ dcsMap->DeleteAll();
+ delete dcsMap;
return kTRUE;
}
}
// TODO Check field count!
- const UInt_t nCols = 22;
+ const UInt_t nCols = 23;
if (aResult->GetFieldCount() != (Int_t) nCols) {
AliError("Invalid SQL result field number!");
delete aResult;
return kTRUE;
}
+//______________________________________________________________________________________________
+TMap* AliShuttle::GetValueSet(const char* host, Int_t port, const TSeqCollection* entries,
+ DCSType type)
+{
+ // Retrieve all "entry" data points from the DCS server
+ // host, port: TSocket connection parameters
+ // entries: list of name of the alias or data point
+ // type: kAlias or kDP
+ // returns TMap of values, 0 when failure
+
+ AliDCSClient client(host, port, fTimeout, fRetries);
+
+ TMap* result = 0;
+ if (type == kAlias)
+ {
+ result = client.GetAliasValues(entries, GetCurrentStartTime(),
+ GetCurrentEndTime());
+ }
+ else if (type == kDP)
+ {
+ result = client.GetDPValues(entries, GetCurrentStartTime(),
+ GetCurrentEndTime());
+ }
+
+ if (result == 0)
+ {
+ Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get entries! Reason: %s",
+ client.GetServerError().Data()));
+
+ return 0;
+ }
+
+ return result;
+}
+
//______________________________________________________________________________________________
const char* AliShuttle::GetFile(Int_t system, const char* detector,
const char* id, const char* source)
Log(detector, Form("GetFileName - Copy of file %s from %s FXS failed",
filePath.Data(), GetSystemName(system)));
continue;
- } else {
- AliInfo(Form("File %s copied from %s FXS into %s/%s",
- filePath.Data(), GetSystemName(system),
- GetShuttleTempDir(), localFileName.Data()));
- }
+ }
if (fileChecksum.Length()>0)
{
fFXSlist[system].Add(fileParams);
static TString fullLocalFileName;
- fullLocalFileName = TString::Format("%s/%s", GetShuttleTempDir(), localFileName.Data());
+ fullLocalFileName.Form("%s/%s", GetShuttleTempDir(), localFileName.Data());
- AliInfo(Form("fullLocalFileName = %s", fullLocalFileName.Data()));
+ Log(fCurrentDetector, Form("GetFile - Retrieved file with id %s and source %s from %s to %s", id, source, GetSystemName(system), fullLocalFileName.Data()));
return fullLocalFileName.Data();
-
}
//______________________________________________________________________________________________
}
else if (system == kHLT)
{
- baseFXSFolder = "~/";
+ baseFXSFolder = "/opt/FXS/";
}
{
//
// Get sources producing the condition file Id from file exchange servers
+ // if id is NULL all sources are returned (distinct)
//
+
+ Log(detector, Form("GetFileSources - Retrieving sources with id %s from %s", id, GetSystemName(system)));
// check if test mode should simulate a FXS error
if (fTestMode & kErrorFXSSources)
return 0;
}
-
if (system == kDCS)
{
- AliError("DCS system has only one source of data!");
- return NULL;
+ AliWarning("DCS system has only one source of data!");
+ TList *list = new TList();
+ list->SetOwner(1);
+ list->Add(new TObjString(" "));
+ return list;
}
// check connection, in case connect
if (!Connect(system))
{
- Log(detector, Form("GetFile - Couldn't connect to %s FXS database", GetSystemName(system)));
+ Log(detector, Form("GetFileSources - Couldn't connect to %s FXS database", GetSystemName(system)));
return NULL;
}
sourceName = "DDLnumbers";
}
- TString sqlQueryStart = Form("select %s from %s where", sourceName.Data(), fConfig->GetFXSdbTable(system));
- TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
- GetCurrentRun(), detector, id);
+ TString sqlQueryStart = Form("select distinct %s from %s where", sourceName.Data(), fConfig->GetFXSdbTable(system));
+ TString whereClause = Form("run=%d and detector=\"%s\"",
+ GetCurrentRun(), detector);
+ if (id)
+ whereClause += Form(" and fileId=\"%s\"", id);
TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
return list;
}
- TSQLRow* aRow;
+ Log(detector, Form("GetFileSources - Found %d sources", aResult->GetRowCount()));
+ TSQLRow* aRow;
while ((aRow = aResult->Next()))
{
return list;
}
+//______________________________________________________________________________________________
+TList* AliShuttle::GetFileIDs(Int_t system, const char* detector, const char* source)
+{
+ //
+ // Get all ids of condition files produced by a given source from file exchange servers
+ //
+
+ Log(detector, Form("GetFileIDs - Retrieving ids with source %s with %s", source, GetSystemName(system)));
+
+ // check if test mode should simulate a FXS error
+ if (fTestMode & kErrorFXSSources)
+ {
+ Log(detector, Form("GetFileIDs - In TESTMODE - Simulating error while connecting to %s FXS", GetSystemName(system)));
+ return 0;
+ }
+
+ // check connection, in case connect
+ if (!Connect(system))
+ {
+ Log(detector, Form("GetFileIDs - Couldn't connect to %s FXS database", GetSystemName(system)));
+ return NULL;
+ }
+
+ TString sourceName = 0;
+ if (system == kDAQ)
+ {
+ sourceName = "DAQsource";
+ } else if (system == kHLT)
+ {
+ sourceName = "DDLnumbers";
+ }
+
+ TString sqlQueryStart = Form("select fileId from %s where", fConfig->GetFXSdbTable(system));
+ TString whereClause = Form("run=%d and detector=\"%s\"",
+ GetCurrentRun(), detector);
+ if (sourceName.Length() > 0 && source)
+ whereClause += Form(" and %s=\"%s\"", sourceName.Data(), source);
+ TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
+
+ AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
+
+ // Query execution
+ TSQLResult* aResult;
+ aResult = fServer[system]->Query(sqlQuery);
+ if (!aResult) {
+ Log(detector, Form("GetFileIDs - Can't execute SQL query to %s database for source: %s",
+ GetSystemName(system), source));
+ return 0;
+ }
+
+ TList *list = new TList();
+ list->SetOwner(1);
+
+ if (aResult->GetRowCount() == 0)
+ {
+ Log(detector,
+ Form("GetFileIDs - No entry in %s FXS table for source: %s", GetSystemName(system), source));
+ delete aResult;
+ return list;
+ }
+
+ Log(detector, Form("GetFileIDs - Found %d ids", aResult->GetRowCount()));
+
+ TSQLRow* aRow;
+
+ while ((aRow = aResult->Next()))
+ {
+
+ TString id(aRow->GetField(0), aRow->GetFieldLength(0));
+ AliDebug(2, Form("fileId = %s", id.Data()));
+ list->Add(new TObjString(id));
+ delete aRow;
+ }
+
+ delete aResult;
+
+ return list;
+}
+
//______________________________________________________________________________________________
Bool_t AliShuttle::Connect(Int_t system)
{
return fLogbookEntry->GetRunType();
}
+//______________________________________________________________________________________________
+Bool_t AliShuttle::GetHLTStatus()
+{
+ // Return HLT status (ON=1 OFF=0)
+ // Converts the HLT status from the status string read in the run logbook (not just a bool)
+
+ if(!fLogbookEntry) {
+ AliError("No logbook entry!");
+ return 0;
+ }
+
+ // TODO implement when HLTStatus is inserted in run logbook
+ //TString hltStatus = fLogbookEntry->GetRunParameter("HLTStatus");
+ //if(hltStatus == "OFF") {return kFALSE};
+
+ return kTRUE;
+}
+
//______________________________________________________________________________________________
void AliShuttle::SetShuttleTempDir(const char* tmpDir)
{
git://git.uio.no / u / mrichter / AliRoot.git / blobdiff