+ fPreprocessorMap.Add(new TObjString(detName), preprocessor);
+}
+//______________________________________________________________________________________________
+Bool_t AliShuttle::Store(const AliCDBPath& path, TObject* object,
+ AliCDBMetaData* metaData, Int_t validityStart, Bool_t validityInfinite)
+{
+ // Stores a CDB object in the storage for offline reconstruction. Objects that are not needed for
+ // offline reconstruction, but should be stored anyway (e.g. for debugging) should NOT be stored
+ // using this function. Use StoreReferenceData instead!
+ // It calls StoreLocally function which temporarily stores the data locally; when the preprocessor
+ // finishes the data are transferred to the main storage (Grid).
+
+ return StoreLocally(fgkLocalCDB, path, object, metaData, validityStart, validityInfinite);
+}
+
+//______________________________________________________________________________________________
+Bool_t AliShuttle::StoreReferenceData(const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData)
+{
+ // Stores a CDB object in the storage for reference data. This objects will not be available during
+ // offline reconstrunction. Use this function for reference data only!
+ // It calls StoreLocally function which temporarily stores the data locally; when the preprocessor
+ // finishes the data are transferred to the main storage (Grid).
+
+ return StoreLocally(fgkLocalRefStorage, path, object, metaData);
+}
+
+//______________________________________________________________________________________________
+Bool_t AliShuttle::StoreLocally(const TString& localUri,
+ const AliCDBPath& path, TObject* object, AliCDBMetaData* metaData,
+ Int_t validityStart, Bool_t validityInfinite)
+{
+ // Store object temporarily in local storage. Parameters are passed by Store and StoreReferenceData functions.
+ // when the preprocessor finishes the data are transferred to the main storage (Grid).
+ // The parameters are:
+ // 1) Uri of the backup storage (Local)
+ // 2) the object's path.
+ // 3) the object to be stored
+ // 4) the metaData to be associated with the object
+ // 5) the validity start run number w.r.t. the current run,
+ // if the data is valid only for this run leave the default 0
+ // 6) specifies if the calibration data is valid for infinity (this means until updated),
+ // typical for calibration runs, the default is kFALSE
+ //
+ // returns 0 if fail, 1 otherwise
+
+ if (fTestMode & kErrorStorage)
+ {
+ Log(fCurrentDetector, "StoreLocally - In TESTMODE - Simulating error while storing locally");
+ return kFALSE;
+ }
+
+ const char* cdbType = (localUri == fgkLocalCDB) ? "CDB" : "Reference";
+
+ Int_t firstRun = GetCurrentRun() - validityStart;
+ if(firstRun < 0) {
+ AliWarning("First valid run happens to be less than 0! Setting it to 0.");
+ firstRun=0;
+ }
+
+ Int_t lastRun = -1;
+ if(validityInfinite) {
+ lastRun = AliCDBRunRange::Infinity();
+ } else {
+ lastRun = GetCurrentRun();
+ }
+
+ // Version is set to current run, it will be used later to transfer data to Grid
+ AliCDBId id(path, firstRun, lastRun, GetCurrentRun(), -1);
+
+ if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
+ TObjString runUsed = Form("%d", GetCurrentRun());
+ metaData->SetProperty("RunUsed(TObjString)", runUsed.Clone());
+ }
+
+ Bool_t result = kFALSE;
+
+ if (!(AliCDBManager::Instance()->GetStorage(localUri))) {
+ Log("SHUTTLE", Form("StoreLocally - Cannot activate local %s storage", cdbType));
+ } else {
+ result = AliCDBManager::Instance()->GetStorage(localUri)
+ ->Put(object, id, metaData);
+ }
+
+ if(!result) {
+
+ Log(fCurrentDetector, Form("StoreLocally - Can't store object <%s>!", id.ToString().Data()));
+ }
+
+ return result;
+}
+
+//______________________________________________________________________________________________
+Bool_t AliShuttle::StoreOCDB()
+{
+ //
+ // Called when preprocessor ends successfully or when previous storage attempt failed (kStoreError status)
+ // Calls underlying StoreOCDB(const char*) function twice, for OCDB and Reference storage.
+ // Then calls StoreRefFilesToGrid to store reference files.
+ //
+
+ if (fTestMode & kErrorGrid)
+ {
+ Log("SHUTTLE", "StoreOCDB - In TESTMODE - Simulating error while storing in the Grid");
+ Log(fCurrentDetector, "StoreOCDB - In TESTMODE - Simulating error while storing in the Grid");
+ return kFALSE;
+ }
+
+ Log("SHUTTLE","StoreOCDB - Storing OCDB data ...");
+ Bool_t resultCDB = StoreOCDB(fgkMainCDB);
+
+ Log("SHUTTLE","StoreOCDB - Storing reference data ...");
+ Bool_t resultRef = StoreOCDB(fgkMainRefStorage);
+
+ Log("SHUTTLE","StoreOCDB - Storing reference files ...");
+ Bool_t resultRefFiles = CopyFilesToGrid("reference");
+
+ Bool_t resultMetadata = kTRUE;
+ if(fCurrentDetector == "GRP")
+ {
+ Log("StoreOCDB - SHUTTLE","Storing Run Metadata file ...");
+ resultMetadata = CopyFilesToGrid("metadata");
+ }
+
+ return resultCDB && resultRef && resultRefFiles && resultMetadata;
+}
+
+//______________________________________________________________________________________________
+Bool_t AliShuttle::StoreOCDB(const TString& gridURI)
+{
+ //
+ // Called by StoreOCDB(), performs actual storage to the main OCDB and reference storages (Grid)
+ //
+
+ TObjArray* gridIds=0;
+
+ Bool_t result = kTRUE;
+
+ const char* type = 0;
+ TString localURI;
+ if(gridURI == fgkMainCDB) {
+ type = "OCDB";
+ localURI = fgkLocalCDB;
+ } else if(gridURI == fgkMainRefStorage) {
+ type = "reference";
+ localURI = fgkLocalRefStorage;
+ } else {
+ AliError(Form("Invalid storage URI: %s", gridURI.Data()));
+ return kFALSE;
+ }
+
+ AliCDBManager* man = AliCDBManager::Instance();
+
+ AliCDBStorage *gridSto = man->GetStorage(gridURI);
+ if(!gridSto) {
+ Log("SHUTTLE",
+ Form("StoreOCDB - cannot activate main %s storage", type));
+ return kFALSE;
+ }
+
+ gridIds = gridSto->GetQueryCDBList();
+
+ // get objects previously stored in local CDB
+ AliCDBStorage *localSto = man->GetStorage(localURI);
+ if(!localSto) {
+ Log("SHUTTLE",
+ Form("StoreOCDB - cannot activate local %s storage", type));
+ return kFALSE;
+ }
+ AliCDBPath aPath(GetOfflineDetName(fCurrentDetector.Data()),"*","*");
+ // Local objects were stored with current run as Grid version!
+ TList* localEntries = localSto->GetAll(aPath.GetPath(), GetCurrentRun(), GetCurrentRun());
+ localEntries->SetOwner(1);
+
+ // loop on local stored objects
+ TIter localIter(localEntries);
+ AliCDBEntry *aLocEntry = 0;
+ while((aLocEntry = dynamic_cast<AliCDBEntry*> (localIter.Next()))){
+ aLocEntry->SetOwner(1);
+ AliCDBId aLocId = aLocEntry->GetId();
+ aLocEntry->SetVersion(-1);
+ aLocEntry->SetSubVersion(-1);
+
+ // If local object is valid up to infinity we store it only if it is
+ // the first unprocessed run!
+ if (aLocId.GetLastRun() == AliCDBRunRange::Infinity() &&
+ !fFirstUnprocessed[GetDetPos(fCurrentDetector)])
+ {
+ Log("SHUTTLE", Form("StoreOCDB - %s: object %s has validity infinite but "
+ "there are previous unprocessed runs!",
+ fCurrentDetector.Data(), aLocId.GetPath().Data()));
+ result = kFALSE;
+ continue;
+ }
+
+ // loop on Grid valid Id's
+ Bool_t store = kTRUE;
+ TIter gridIter(gridIds);
+ AliCDBId* aGridId = 0;
+ while((aGridId = dynamic_cast<AliCDBId*> (gridIter.Next()))){
+ if(aGridId->GetPath() != aLocId.GetPath()) continue;
+ // skip all objects valid up to infinity
+ if(aGridId->GetLastRun() == AliCDBRunRange::Infinity()) continue;
+ // if we get here, it means there's already some more recent object stored on Grid!
+ store = kFALSE;
+ break;
+ }
+
+ // If we get here, the file can be stored!
+ Bool_t storeOk = gridSto->Put(aLocEntry);
+ if(!store || storeOk){
+
+ if (!store)
+ {
+ Log(fCurrentDetector.Data(),
+ Form("StoreOCDB - A more recent object already exists in %s storage: <%s>",
+ type, aGridId->ToString().Data()));
+ } else {
+ Log("SHUTTLE",
+ Form("StoreOCDB - Object <%s> successfully put into %s storage",
+ aLocId.ToString().Data(), type));
+ Log(fCurrentDetector.Data(),
+ Form("StoreOCDB - Object <%s> successfully put into %s storage",
+ aLocId.ToString().Data(), type));
+ }
+
+ // removing local filename...
+ TString filename;
+ localSto->IdToFilename(aLocId, filename);
+ Log("SHUTTLE", Form("StoreOCDB - Removing local file %s", filename.Data()));
+ RemoveFile(filename.Data());
+ continue;
+ } else {
+ Log("SHUTTLE",
+ Form("StoreOCDB - Grid %s storage of object <%s> failed",
+ type, aLocId.ToString().Data()));
+ Log(fCurrentDetector.Data(),
+ Form("StoreOCDB - Grid %s storage of object <%s> failed",
+ type, aLocId.ToString().Data()));
+ result = kFALSE;
+ }
+ }
+ localEntries->Clear();
+
+ return result;
+}
+
+//______________________________________________________________________________________________
+Bool_t AliShuttle::CleanReferenceStorage(const char* detector)
+{
+ // clears the directory used to store reference files of a given subdetector
+
+ AliCDBManager* man = AliCDBManager::Instance();
+ AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
+ TString localBaseFolder = sto->GetBaseFolder();
+
+ TString targetDir = GetRefFilePrefix(localBaseFolder.Data(), detector);
+
+ Log("SHUTTLE", Form("CleanReferenceStorage - Cleaning %s", targetDir.Data()));
+
+ TString begin;
+ begin.Form("%d_", GetCurrentRun());
+
+ TSystemDirectory* baseDir = new TSystemDirectory("/", targetDir);
+ if (!baseDir)
+ return kTRUE;
+
+ TList* dirList = baseDir->GetListOfFiles();
+ delete baseDir;
+
+ if (!dirList) return kTRUE;
+
+ if (dirList->GetEntries() < 3)
+ {
+ delete dirList;
+ return kTRUE;
+ }
+
+ Int_t nDirs = 0, nDel = 0;
+ TIter dirIter(dirList);
+ TSystemFile* entry = 0;
+
+ Bool_t success = kTRUE;
+
+ while ((entry = dynamic_cast<TSystemFile*> (dirIter.Next())))
+ {
+ if (entry->IsDirectory())
+ continue;
+
+ TString fileName(entry->GetName());
+ if (!fileName.BeginsWith(begin))
+ continue;
+
+ nDirs++;
+
+ // delete file
+ Int_t result = gSystem->Unlink(fileName.Data());
+
+ if (result)
+ {
+ Log("SHUTTLE", Form("CleanReferenceStorage - Could not delete file %s!", fileName.Data()));
+ success = kFALSE;
+ } else {
+ nDel++;
+ }
+ }
+
+ if(nDirs > 0)
+ Log("SHUTTLE", Form("CleanReferenceStorage - %d (over %d) reference files in folder %s were deleted.",
+ nDel, nDirs, targetDir.Data()));
+
+
+ delete dirList;
+ return success;
+
+
+
+
+
+
+ Int_t result = gSystem->GetPathInfo(targetDir, 0, (Long64_t*) 0, 0, 0);
+ if (result == 0)
+ {
+ // delete directory
+ result = gSystem->Exec(Form("rm -rf %s", targetDir.Data()));
+ if (result != 0)
+ {
+ Log("SHUTTLE", Form("CleanReferenceStorage - Could not clean directory %s", targetDir.Data()));
+ return kFALSE;
+ }
+ }
+
+ result = gSystem->mkdir(targetDir, kTRUE);
+ if (result != 0)
+ {
+ Log("SHUTTLE", Form("CleanReferenceStorage - Error creating base directory %s", targetDir.Data()));
+ return kFALSE;
+ }
+
+ return kTRUE;
+}
+
+//______________________________________________________________________________________________
+Bool_t AliShuttle::StoreReferenceFile(const char* detector, const char* localFile, const char* gridFileName)
+{
+ //
+ // Stores reference file directly (without opening it). This function stores the file locally.
+ //
+ // The file is stored under the following location:
+ // <base folder of local reference storage>/<DET>/<RUN#>_<gridFileName>
+ // where <gridFileName> is the second parameter given to the function
+ //
+
+ if (fTestMode & kErrorStorage)
+ {
+ Log(fCurrentDetector, "StoreReferenceFile - In TESTMODE - Simulating error while storing locally");
+ return kFALSE;
+ }
+
+ AliCDBManager* man = AliCDBManager::Instance();
+ AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
+
+ TString localBaseFolder = sto->GetBaseFolder();
+
+ TString target = GetRefFilePrefix(localBaseFolder.Data(), detector);
+ target.Append(Form("/%d_%s", GetCurrentRun(), gridFileName));
+
+ return CopyFileLocally(localFile, target);
+}
+
+//______________________________________________________________________________________________
+Bool_t AliShuttle::StoreRunMetadataFile(const char* localFile, const char* gridFileName)
+{
+ //
+ // Stores Run metadata file to the Grid, in the run folder
+ //
+ // Only GRP can call this function.
+
+ if (fTestMode & kErrorStorage)
+ {
+ Log(fCurrentDetector, "StoreRunMetaDataFile - In TESTMODE - Simulating error while storing locally");
+ return kFALSE;
+ }
+
+ AliCDBManager* man = AliCDBManager::Instance();
+ AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
+
+ TString localBaseFolder = sto->GetBaseFolder();
+
+ // Build Run level folder
+ // folder = /alice/data/year/lhcPeriod/runNb/raw
+
+
+ TString lhcPeriod = GetLHCPeriod();
+ if (lhcPeriod.Length() == 0)
+ {
+ Log("SHUTTLE","StoreRunMetaDataFile - LHCPeriod not found in logbook!");
+ return 0;
+ }
+
+ TString target = Form("%s/GRP/RunMetadata/alice/data/%d/%s/%09d/raw/%s",
+ localBaseFolder.Data(), GetCurrentYear(),
+ lhcPeriod.Data(), GetCurrentRun(), gridFileName);
+
+ return CopyFileLocally(localFile, target);
+}
+
+//______________________________________________________________________________________________
+Bool_t AliShuttle::CopyFileLocally(const char* localFile, const TString& target)
+{
+ //
+ // Stores file locally. Called by StoreReferenceFile and StoreRunMetadataFile
+ // Files are temporarily stored in the local reference storage. When the preprocessor
+ // finishes, the Shuttle calls CopyFilesToGrid to transfer the files to AliEn
+ // (in reference or run level folders)
+ //
+
+ TString targetDir(target(0, target.Last('/')));
+
+ //try to open base dir folder, if it does not exist
+ void* dir = gSystem->OpenDirectory(targetDir.Data());
+ if (dir == NULL) {
+ if (gSystem->mkdir(targetDir.Data(), kTRUE)) {
+ Log("SHUTTLE", Form("StoreFileLocally - Can't open directory <%s>", targetDir.Data()));
+ return kFALSE;
+ }
+
+ } else {
+ gSystem->FreeDirectory(dir);
+ }
+
+ Int_t result = 0;
+
+ result = gSystem->GetPathInfo(localFile, 0, (Long64_t*) 0, 0, 0);
+ if (result)
+ {
+ Log("SHUTTLE", Form("StoreFileLocally - %s does not exist", localFile));
+ return kFALSE;
+ }
+
+ result = gSystem->GetPathInfo(target, 0, (Long64_t*) 0, 0, 0);
+ if (!result)
+ {
+ Log("SHUTTLE", Form("StoreFileLocally - target file %s already exist, removing...", target.Data()));
+ if (gSystem->Unlink(target.Data()))
+ {
+ Log("SHUTTLE", Form("StoreFileLocally - Could not remove existing target file %s!", target.Data()));
+ return kFALSE;
+ }
+ }
+
+ result = gSystem->CopyFile(localFile, target);
+
+ if (result == 0)
+ {
+ Log("SHUTTLE", Form("StoreFileLocally - File %s stored locally to %s", localFile, target.Data()));
+ return kTRUE;
+ }
+ else
+ {
+ Log("SHUTTLE", Form("StoreFileLocally - Could not store file %s to %s! Error code = %d",
+ localFile, target.Data(), result));
+ return kFALSE;
+ }
+
+
+
+}
+
+//______________________________________________________________________________________________
+Bool_t AliShuttle::CopyFilesToGrid(const char* type)
+{
+ //
+ // Transfers local files to the Grid. Local files can be reference files
+ // or run metadata file (from GRP only).
+ //
+ // According to the type (ref, metadata) the files are stored under the following location:
+ // ref --> <base folder of reference storage>/<DET>/<RUN#>_<gridFileName>
+ // metadata --> <run data folder>/<MetadataFileName>
+ //
+
+ AliCDBManager* man = AliCDBManager::Instance();
+ AliCDBStorage* sto = man->GetStorage(fgkLocalRefStorage);
+ if (!sto)
+ return kFALSE;
+ TString localBaseFolder = sto->GetBaseFolder();
+
+ TString dir;
+ TString alienDir;
+ TString begin;
+
+ if (strcmp(type, "reference") == 0)
+ {
+ dir = GetRefFilePrefix(localBaseFolder.Data(), fCurrentDetector.Data());
+ AliCDBStorage* gridSto = man->GetStorage(fgkMainRefStorage);
+ if (!gridSto)
+ return kFALSE;
+ TString gridBaseFolder = gridSto->GetBaseFolder();
+ alienDir = GetRefFilePrefix(gridBaseFolder.Data(), fCurrentDetector.Data());
+ begin = Form("%d_", GetCurrentRun());
+ }
+ else if (strcmp(type, "metadata") == 0)
+ {
+
+ TString lhcPeriod = GetLHCPeriod();
+
+ if (lhcPeriod.Length() == 0)
+ {
+ Log("SHUTTLE","CopyFilesToGrid - LHCPeriod not found in logbook!");
+ return 0;
+ }
+
+ dir = Form("%s/GRP/RunMetadata/alice/data/%d/%s/%09d/raw",
+ localBaseFolder.Data(), GetCurrentYear(),
+ lhcPeriod.Data(), GetCurrentRun());
+ alienDir = dir(dir.Index("/alice/data/"), dir.Length());
+
+ begin = "";
+ }
+ else
+ {
+ Log("SHUTTLE", "CopyFilesToGrid - Unexpected: type label must be reference or metadata!");
+ return kFALSE;
+ }
+
+ TSystemDirectory* baseDir = new TSystemDirectory("/", dir);
+ if (!baseDir)
+ return kTRUE;
+
+ TList* dirList = baseDir->GetListOfFiles();
+ delete baseDir;
+
+ if (!dirList) return kTRUE;
+
+ if (dirList->GetEntries() < 3)
+ {
+ delete dirList;
+ return kTRUE;
+ }
+
+ if (!gGrid)
+ {
+ Log("SHUTTLE", "CopyFilesToGrid - Connection to Grid failed: Cannot continue!");
+ delete dirList;
+ return kFALSE;
+ }
+
+ Int_t nDirs = 0, nTransfer = 0;
+ TIter dirIter(dirList);
+ TSystemFile* entry = 0;
+
+ Bool_t success = kTRUE;
+ Bool_t first = kTRUE;
+
+ while ((entry = dynamic_cast<TSystemFile*> (dirIter.Next())))
+ {
+ if (entry->IsDirectory())
+ continue;
+
+ TString fileName(entry->GetName());
+ if (!fileName.BeginsWith(begin))
+ continue;
+
+ nDirs++;
+
+ if (first)
+ {
+ first = kFALSE;
+ // check that folder exists, otherwise create it
+ TGridResult* result = gGrid->Ls(alienDir.Data(), "a");
+
+ if (!result)
+ {
+ delete dirList;
+ return kFALSE;
+ }
+
+ if (!result->GetFileName(1)) // TODO: It looks like element 0 is always 0!!
+ {
+ // TODO It does not work currently! Bug in TAliEn::Mkdir
+ // TODO Manually fixed in local root v5-16-00
+ if (!gGrid->Mkdir(alienDir.Data(),"-p",0))
+ {
+ Log("SHUTTLE", Form("CopyFilesToGrid - Cannot create directory %s",
+ alienDir.Data()));
+ delete dirList;
+ return kFALSE;
+ } else {
+ Log("SHUTTLE",Form("CopyFilesToGrid - Folder %s created", alienDir.Data()));
+ }
+
+ } else {
+ Log("SHUTTLE",Form("CopyFilesToGrid - Folder %s found", alienDir.Data()));
+ }
+ }
+
+ TString fullLocalPath;
+ fullLocalPath.Form("%s/%s", dir.Data(), fileName.Data());
+
+ TString fullGridPath;
+ fullGridPath.Form("alien://%s/%s", alienDir.Data(), fileName.Data());
+
+ Bool_t result = TFile::Cp(fullLocalPath, fullGridPath);
+
+ if (result)
+ {
+ Log("SHUTTLE", Form("CopyFilesToGrid - Copying local file %s to %s succeeded!",
+ fullLocalPath.Data(), fullGridPath.Data()));
+ RemoveFile(fullLocalPath);
+ nTransfer++;
+ }
+ else
+ {
+ Log("SHUTTLE", Form("CopyFilesToGrid - Copying local file %s to %s FAILED!",
+ fullLocalPath.Data(), fullGridPath.Data()));
+ success = kFALSE;
+ }
+ }
+
+ Log("SHUTTLE", Form("CopyFilesToGrid - %d (over %d) files in folder %s copied to Grid.",
+ nTransfer, nDirs, dir.Data()));