X-Git-Url: http://git.uio.no/git/?a=blobdiff_plain;f=STEER%2FAliCDBGrid.cxx;h=44b2f4ce4e4c5b1505670e9d1e90a86e1a1f9a82;hb=406a77e00b0b6ffdf5114ac52eb95b055bd8f555;hp=8aea78782431432fbd5561da660035641ac0ad21;hpb=c3a7b59ad78b88072feaab8b40e071ae84bad21d;p=u%2Fmrichter%2FAliRoot.git diff --git a/STEER/AliCDBGrid.cxx b/STEER/AliCDBGrid.cxx index 8aea7878243..44b2f4ce4e4 100644 --- a/STEER/AliCDBGrid.cxx +++ b/STEER/AliCDBGrid.cxx @@ -20,13 +20,13 @@ // // ///////////////////////////////////////////////////////////////////////////////////////////////// - +#include #include #include #include #include #include -#include +#include #include #include #include @@ -34,27 +34,34 @@ #include "AliLog.h" #include "AliCDBEntry.h" #include "AliCDBGrid.h" +#include "AliCDBManager.h" ClassImp(AliCDBGrid) //_____________________________________________________________________________ -AliCDBGrid::AliCDBGrid(const char *gridUrl, const char *user, const char *dbFolder, const char *se) : +AliCDBGrid::AliCDBGrid(const char *gridUrl, const char *user, const char *dbFolder, + const char *se, const char* cacheFolder, Bool_t operateDisconnected, + Long64_t cacheSize, Long_t cleanupInterval) : AliCDBStorage(), fGridUrl(gridUrl), fUser(user), fDBFolder(dbFolder), -fSE(se) +fSE(se), +fCacheFolder(cacheFolder), +fOperateDisconnected(operateDisconnected), +fCacheSize(cacheSize), +fCleanupInterval(cleanupInterval) { // constructor // // if the same Grid is alreay active, skip connection - if (!gGrid || fGridUrl != gGrid->GridUrl() + if (!gGrid || fGridUrl != gGrid->GridUrl() || (( fUser != "" ) && ( fUser != gGrid->GetUser() )) ) { // connection to the Grid AliInfo("Connection to the Grid..."); if(gGrid){ - AliInfo(Form("gGrid = %x; fGridUrl = %s; gGrid->GridUrl() = %s",gGrid,fGridUrl.Data(), gGrid->GridUrl())); + AliInfo(Form("gGrid = %p; fGridUrl = %s; gGrid->GridUrl() = %s",gGrid,fGridUrl.Data(), gGrid->GridUrl())); AliInfo(Form("fUser = %s; gGrid->GetUser() = %s",fUser.Data(), gGrid->GetUser())); } TGrid::Connect(fGridUrl.Data(),fUser.Data()); @@ -73,8 +80,11 @@ fSE(se) // check DBFolder: trying to cd to DBFolder; if it does not exist, create it if(!gGrid->Cd(fDBFolder.Data(),0)){ AliDebug(2,Form("Creating new folder <%s> ...",fDBFolder.Data())); - if(!gGrid->Mkdir(fDBFolder.Data(),"",0)){ - AliError(Form("Cannot create folder <%s> !",fDBFolder.Data())); + TGridResult* res = gGrid->Command(Form("mkdir -p %s",fDBFolder.Data())); + TString result = res->GetKey(0,"__result__"); + if(result == "0"){ + AliFatal(Form("Cannot create folder <%s> !",fDBFolder.Data())); + return; } } else { AliDebug(2,Form("Folder <%s> found",fDBFolder.Data())); @@ -87,6 +97,36 @@ fSE(se) fType="alien"; fBaseFolder = fDBFolder; + // Setting the cache + + // Check if local cache folder is already defined + TString origCache(TFile::GetCacheFileDir()); + if(fCacheFolder.Length() > 0) { + if(origCache.Length() == 0) { + AliInfo(Form("Setting local cache to: %s", fCacheFolder.Data())); + } else if(fCacheFolder != origCache) { + AliWarning(Form("Local cache folder was already defined, changing it to: %s", + fCacheFolder.Data())); + } + + // default settings are: operateDisconnected=kTRUE, forceCacheread = kFALSE + if(!TFile::SetCacheFileDir(fCacheFolder.Data(), fOperateDisconnected)) { + AliError(Form("Could not set cache folder %s !", fCacheFolder.Data())); + fCacheFolder = ""; + } else { + // reset fCacheFolder because the function may have + // slightly changed the folder name (e.g. '/' added) + fCacheFolder = TFile::GetCacheFileDir(); + } + + // default settings are: cacheSize=1GB, cleanupInterval = 0 + if(!TFile::ShrinkCacheFileDir(fCacheSize, fCleanupInterval)) { + AliError(Form("Could not set following values " + "to ShrinkCacheFileDir: cacheSize = %lld, cleanupInterval = %ld !", + fCacheSize, fCleanupInterval)); + } + } + // return to the initial directory gGrid->Cd(initDir.Data(),0); } @@ -101,7 +141,7 @@ AliCDBGrid::~AliCDBGrid() //_____________________________________________________________________________ Bool_t AliCDBGrid::FilenameToId(TString& filename, AliCDBId& id) { -// build AliCDBId from full path filename (fDBFolder/path/Run#x_#y_v#z.root) +// build AliCDBId from full path filename (fDBFolder/path/Run#x_#y_v#z_s0.root) if(filename.Contains(fDBFolder)){ filename = filename(fDBFolder.Length(),filename.Length()-fDBFolder.Length()); @@ -114,13 +154,26 @@ Bool_t AliCDBGrid::FilenameToId(TString& filename, AliCDBId& id) { filename=filename(idPath.Length()+1,filename.Length()-idPath.Length()); Ssiz_t mSize; - // valid filename: Run#firstRun_#lastRun_v#version.root - TRegexp keyPattern("^Run[0-9]+_[0-9]+_v[0-9]+.root$"); + // valid filename: Run#firstRun_#lastRun_v#version_s0.root + TRegexp keyPattern("^Run[0-9]+_[0-9]+_v[0-9]+_s0.root$"); keyPattern.Index(filename, &mSize); if (!mSize) { - AliDebug(2,Form("Bad filename <%s>.", filename.Data())); - return kFALSE; - } + + // TODO backward compatibility ... maybe remove later! + Ssiz_t oldmSize; + TRegexp oldKeyPattern("^Run[0-9]+_[0-9]+_v[0-9]+.root$"); + oldKeyPattern.Index(filename, &oldmSize); + if(!oldmSize) { + AliDebug(2,Form("Bad filename <%s>.", filename.Data())); + return kFALSE; + } else { + AliDebug(2,Form("Old filename format <%s>.", filename.Data())); + id.SetSubVersion(-11); // TODO trick to ensure backward compatibility + } + + } else { + id.SetSubVersion(-1); // TODO trick to ensure backward compatibility + } filename.Resize(filename.Length() - sizeof(".root") + 1); @@ -153,11 +206,14 @@ Bool_t AliCDBGrid::IdToFilename(const AliCDBId& id, TString& filename) const { return kFALSE; } - filename = Form("Run%d_%d_v%d.root", + filename = Form("Run%d_%d_v%d", id.GetFirstRun(), id.GetLastRun(), id.GetVersion()); + if (id.GetSubVersion() != -11) filename += "_s0"; // TODO to ensure backward compatibility + filename += ".root"; + filename.Prepend(fDBFolder + id.GetPath() + '/'); return kTRUE; @@ -168,17 +224,20 @@ Bool_t AliCDBGrid::PrepareId(AliCDBId& id) { // prepare id (version) of the object that will be stored (called by PutEntry) TString initDir(gGrid->Pwd(0)); - TString pathName= id.GetPath(); TString dirName(fDBFolder); Bool_t dirExist=kFALSE; - + + + // go to the path; if directory does not exist, create it - TObjArray *arrName=pathName.Tokenize("/"); - for(int i=0;iGetEntries();i++){ - TString buffer((arrName->At(i))->GetName()); - dirName+=buffer; dirName+="/"; + for(int i=0;i<3;i++){ + //TString cmd("find -d "); + //cmd += Form("%s ",dirName); + //cmd += + //gGrid->Command(cmd.Data()); + dirName+=Form("%s/",id.GetPathLevel(i).Data()); dirExist=gGrid->Cd(dirName,0); if (!dirExist) { AliDebug(2,Form("Creating new folder <%s> ...",dirName.Data())); @@ -187,9 +246,62 @@ Bool_t AliCDBGrid::PrepareId(AliCDBId& id) { gGrid->Cd(initDir.Data()); return kFALSE; } + + // if folders are new add tags to them + if(i == 1) { + // TODO Currently disabled + // add short lived tag! + // AliInfo("Tagging level 1 folder with \"ShortLived\" tag"); + // if(!AddTag(dirName,"ShortLived_try")){ + // AliError(Form("Could not tag folder %s !", dirName.Data())); + // if(!gGrid->Rmdir(dirName.Data())){ + // AliError(Form("Unexpected: could not remove %s directory!", dirName.Data())); + // } + // return 0; + //} + + } else if(i == 2) { + AliDebug(2,"Tagging level 2 folder with \"CDB\" and \"CDB_MD\" tag"); + if(!AddTag(dirName,"CDB")){ + AliError(Form("Could not tag folder %s !", dirName.Data())); + if(!gGrid->Rmdir(dirName.Data())){ + AliError(Form("Unexpected: could not remove %s directory!", dirName.Data())); + } + return 0; + } + if(!AddTag(dirName,"CDB_MD")){ + AliError(Form("Could not tag folder %s !", dirName.Data())); + if(!gGrid->Rmdir(dirName.Data())){ + AliError(Form("Unexpected: could not remove %s directory!", dirName.Data())); + } + return 0; + } + + // TODO Currently disabled + // add short lived tag! + // TString path=id.GetPath(); + // if(AliCDBManager::Instance()->IsShortLived(path.Data())) { + // AliInfo(Form("Tagging %s as short lived", dirName.Data())); + // if(!TagShortLived(dirName, kTRUE)){ + // AliError(Form("Could not tag folder %s !", dirName.Data())); + // if(!gGrid->Rmdir(dirName.Data())){ + // AliError(Form("Unexpected: could not remove %s directory!", dirName.Data())); + // } + // return 0; + // } + // } else { + // AliInfo(Form("Tagging %s as long lived", dirName.Data())); + // if(!TagShortLived(dirName, kFALSE)){ + // AliError(Form("Could not tag folder %s !", dirName.Data())); + // if(!gGrid->Rmdir(dirName.Data())){ + // AliError(Form("Unexpected: could not remove %s directory!", dirName.Data())); + // } + // return 0; + // } + // } + } } } - delete arrName; gGrid->Cd(initDir,0); TString filename; @@ -209,9 +321,17 @@ Bool_t AliCDBGrid::PrepareId(AliCDBId& id) { } } - delete res; - + delete res; + + // GRP entries with explicitly set version escape default incremental versioning + if(id.GetPath().Contains("GRP") && id.HasVersion() && lastVersion!=0) + { + AliDebug(5,Form("Entry %s won't be put in the destination OCDB", id.ToString().Data())); + return kFALSE; + } + id.SetVersion(lastVersion + 1); + id.SetSubVersion(0); TString lastStorage = id.GetLastStorage(); if(lastStorage.Contains(TString("new"), TString::kIgnoreCase) && id.GetVersion() > 1 ){ @@ -221,22 +341,19 @@ Bool_t AliCDBGrid::PrepareId(AliCDBId& id) { id.GetVersion()-1)); } - if(!lastRunRange.IsAnyRange() && !(lastRunRange.IsEqual(&id.GetAliCDBRunRange()))) + if(!lastRunRange.IsAnyRange() && !(lastRunRange.IsEqual(&id.GetAliCDBRunRange()))) AliWarning(Form("Run range modified w.r.t. previous version (Run%d_%d_v%d)", lastRunRange.GetFirstRun(), lastRunRange.GetLastRun(), id.GetVersion())); - + return kTRUE; } //_____________________________________________________________________________ -AliCDBId* AliCDBGrid::GetId(const TList& validFileIds, const AliCDBId& query) { +AliCDBId* AliCDBGrid::GetId(const TObjArray& validFileIds, const AliCDBId& query) { // look for the Id that matches query's requests (highest or exact version) - if(validFileIds.GetEntries() < 1) { + if(validFileIds.GetEntriesFast() < 1) return NULL; - } else if (validFileIds.GetEntries() == 1) { - return dynamic_cast (validFileIds.At(0)); - } TIter iter(&validFileIds); @@ -251,7 +368,7 @@ AliCDBId* AliCDBGrid::GetId(const TList& validFileIds, const AliCDBId& query) { if (!query.HasVersion()){ // look for highest version if(result && result->GetVersion() > anIdPtr->GetVersion()) continue; if(result && result->GetVersion() == anIdPtr->GetVersion()) { - AliDebug(2,Form("More than one object valid for run %d, version %d!", + AliError(Form("More than one object valid for run %d, version %d!", query.GetFirstRun(), anIdPtr->GetVersion())); return NULL; } @@ -259,7 +376,7 @@ AliCDBId* AliCDBGrid::GetId(const TList& validFileIds, const AliCDBId& query) { } else { // look for specified version if(query.GetVersion() != anIdPtr->GetVersion()) continue; if(result && result->GetVersion() == anIdPtr->GetVersion()){ - AliDebug(2,Form("More than one object valid for run %d, version %d!", + AliError(Form("More than one object valid for run %d, version %d!", query.GetFirstRun(), anIdPtr->GetVersion())); return NULL; } @@ -267,14 +384,16 @@ AliCDBId* AliCDBGrid::GetId(const TList& validFileIds, const AliCDBId& query) { } } + + if (!result) return NULL; - - return result; + return dynamic_cast (result->Clone()); } //_____________________________________________________________________________ -AliCDBEntry* AliCDBGrid::GetEntry(const AliCDBId& queryId) { -// get AliCDBEntry from the database +AliCDBId* AliCDBGrid::GetEntryId(const AliCDBId& queryId) { +// get AliCDBId from the database +// User must delete returned object AliCDBId* dataId=0; @@ -284,59 +403,109 @@ AliCDBEntry* AliCDBGrid::GetEntry(const AliCDBId& queryId) { GetSelection(&selectedId); } - TList validFileIds; + TObjArray validFileIds; validFileIds.SetOwner(1); // look for file matching query requests (path, runRange, version) if(selectedId.GetFirstRun() == fRun && fPathFilter.Comprises(selectedId.GetAliCDBPath()) && fVersion < 0 && !fMetaDataFilter){ // look into list of valid files previously loaded with AliCDBStorage::FillValidFileIds() - AliDebug(2, Form("List of files valid for run %d and for path %s was loaded. Looking there!", + AliDebug(2, Form("List of files valid for run %d was loaded. Looking there for fileids valid for path %s!", selectedId.GetFirstRun(), selectedId.GetPath().Data())); dataId = GetId(fValidFileIds, selectedId); } else { // List of files valid for reqested run was not loaded. Looking directly into CDB - AliDebug(2, Form("List of files valid for run %d and for path %s was not loaded. Looking directly into CDB!", + AliDebug(2, Form("List of files valid for run %d was not loaded. Looking directly into CDB for fileids valid for path %s!", selectedId.GetFirstRun(), selectedId.GetPath().Data())); TString filter; - MakeQueryFilter(selectedId.GetFirstRun(), selectedId.GetLastRun(), - selectedId.GetAliCDBPath(), selectedId.GetVersion(), 0, filter); + MakeQueryFilter(selectedId.GetFirstRun(), selectedId.GetLastRun(), 0, filter); - TGridResult *res = gGrid->Query(fDBFolder, "Run*.root", filter, ""); - AliCDBId validFileId; - for(int i=0; iGetEntries(); i++){ - TString filename = res->GetKey(i, "lfn"); - if(filename == "") continue; - if(FilenameToId(filename, validFileId)) - validFileIds.AddLast(validFileId.Clone()); + TString pattern = Form("%s/Run*", selectedId.GetPath().Data()); + TString optionQuery = "-y"; + if(selectedId.GetVersion() >= 0) { + pattern += Form("_v%d*",selectedId.GetVersion()); + optionQuery = ""; } - delete res; + + pattern += ".root"; + TString det = pattern(0,4); + TString patternCopy(pattern); + patternCopy.Remove(0,4); + TString folderCopy(fDBFolder); + folderCopy += det; + + if (optionQuery == "-y"){ + AliInfo("Only latest version will be returned"); + } + + AliDebug(2,Form("** fDBFolder = %s, pattern = %s, filter = %s",folderCopy.Data(), patternCopy.Data(), filter.Data())); + TGridResult *res = gGrid->Query(folderCopy, patternCopy, filter, optionQuery.Data()); + if (res) { + AliCDBId validFileId; + for(int i=0; iGetEntries(); i++){ + TString filename = res->GetKey(i, "lfn"); + if(filename == "") continue; + if(FilenameToId(filename, validFileId)) + validFileIds.AddLast(validFileId.Clone()); + } + delete res; + } dataId = GetId(validFileIds, selectedId); } + return dataId; +} + +//_____________________________________________________________________________ +AliCDBEntry* AliCDBGrid::GetEntry(const AliCDBId& queryId) { +// get AliCDBEntry from the database + + AliCDBId* dataId = GetEntryId(queryId); + if (!dataId) return NULL; TString filename; if (!IdToFilename(*dataId, filename)) { AliDebug(2,Form("Bad data ID encountered! Subnormal error!")); + delete dataId; return NULL; } AliCDBEntry* anEntry = GetEntryFromFile(filename, dataId); + delete dataId; return anEntry; } //_____________________________________________________________________________ -AliCDBEntry* AliCDBGrid::GetEntryFromFile(TString& filename, const AliCDBId* dataId){ +AliCDBEntry* AliCDBGrid::GetEntryFromFile(TString& filename, AliCDBId* dataId){ // Get AliCBEntry object from file "filename" AliDebug(2,Form("Opening file: %s",filename.Data())); filename.Prepend("/alien"); - TFile *file = TFile::Open(filename); + + // if option="CACHEREAD" TFile will use the local caching facility! + TString option="READ"; + if(fCacheFolder != ""){ + + // Check if local cache folder was changed in the meanwhile + TString origCache(TFile::GetCacheFileDir()); + if(fCacheFolder != origCache) { + AliWarning(Form("Local cache folder has been overwritten!! fCacheFolder = %s origCache = %s", + fCacheFolder.Data(), origCache.Data())); + TFile::SetCacheFileDir(fCacheFolder.Data(), fOperateDisconnected); + TFile::ShrinkCacheFileDir(fCacheSize, fCleanupInterval); + } + + option.Prepend("CACHE"); + } + + AliDebug(2, Form("Option: %s", option.Data())); + + TFile *file = TFile::Open(filename, option); if (!file) { AliDebug(2,Form("Can't open file <%s>!", filename.Data())); return NULL; @@ -359,16 +528,21 @@ AliCDBEntry* AliCDBGrid::GetEntryFromFile(TString& filename, const AliCDBId* dat if(anEntry){ AliCDBId entryId = anEntry->GetId(); - if(!((entryId.GetAliCDBRunRange()).IsEqual(&(dataId->GetAliCDBRunRange()))) || - entryId.GetVersion() != dataId->GetVersion()){ - AliWarning(Form("Either RunRange or gridVersion in the object's metadata")); - AliWarning(Form("do noth match with fileName numbers:")); - AliWarning(Form("someone renamed file by hand!")); + Int_t tmpSubVersion = dataId->GetSubVersion(); + dataId->SetSubVersion(entryId.GetSubVersion()); // otherwise filename and id may mismatch + if(!entryId.IsEqual(dataId)){ + AliWarning(Form("Mismatch between file name and object's Id!")); + AliWarning(Form("File name: %s", dataId->ToString().Data())); + AliWarning(Form("Object's Id: %s", entryId.ToString().Data())); } + dataId->SetSubVersion(tmpSubVersion); } anEntry->SetLastStorage("grid"); + // Check whether entry contains a TTree. In case load the tree in memory! + LoadTreeFromFile(anEntry); + // close file, return retieved entry file->Close(); delete file; file=0; @@ -382,7 +556,7 @@ TList* AliCDBGrid::GetEntries(const AliCDBId& queryId) { TList* result = new TList(); result->SetOwner(); - TList validFileIds; + TObjArray validFileIds; validFileIds.SetOwner(1); Bool_t alreadyLoaded = kFALSE; @@ -402,10 +576,20 @@ TList* AliCDBGrid::GetEntries(const AliCDBId& queryId) { queryId.GetFirstRun(), queryId.GetPath().Data())); TString filter; - MakeQueryFilter(queryId.GetFirstRun(), queryId.GetLastRun(), - queryId.GetAliCDBPath(), queryId.GetVersion(), 0, filter); + MakeQueryFilter(queryId.GetFirstRun(), queryId.GetLastRun(), 0, filter); + + TString pattern = Form("%s/Run*.root", queryId.GetPath().Data()); + AliDebug(2,Form("pattern: %s", pattern.Data())); + + TString optionQuery("-y"); + if(queryId.GetVersion() >= 0) optionQuery = ""; + + if (optionQuery == "-y"){ + AliInfo("Only latest version will be returned"); + } + + TGridResult *res = gGrid->Query(fDBFolder, pattern, filter, optionQuery.Data()); - TGridResult *res = gGrid->Query(fDBFolder, "Run*.root", filter, ""); AliCDBId validFileId; for(int i=0; iGetEntries(); i++){ TString filename = res->GetKey(i, "lfn"); @@ -423,7 +607,7 @@ TList* AliCDBGrid::GetEntries(const AliCDBId& queryId) { iter = new TIter(&validFileIds); } - TList selectedIds; + TObjArray selectedIds; selectedIds.SetOwner(1); // loop on list of valid Ids to select the right version to get. @@ -447,7 +631,7 @@ TList* AliCDBGrid::GetEntries(const AliCDBId& queryId) { } else { dataId = GetId(validFileIds, thisId); } - if(dataId) selectedIds.Add(dataId->Clone()); + if(dataId) selectedIds.Add(dataId); } delete iter; iter=0; @@ -492,30 +676,28 @@ Bool_t AliCDBGrid::PutEntry(AliCDBEntry* entry) { fDBFolder.Data(), id.GetPath().Data()); - // add CDB and CDB_MD tag to folder - // TODO how to check that folder has already tags? - AddTag(folderToTag,"CDB"); - AddTag(folderToTag,"CDB_MD"); - TDirectory* saveDir = gDirectory; + TString fullFilename = Form("/alien%s", filename.Data()); // specify SE to filename - TString fullFilename = Form("/alien%s?se=%s", filename.Data(), fSE.Data()); + if (fSE != "default") fullFilename += Form("?se=%s",fSE.Data()); // open file - TFile *file = TFile::Open(fullFilename,"CREATE"); - if(!file || !file->IsWritable()){ - AliError(Form("Can't open file <%s>!", filename.Data())); - if(file && !file->IsWritable()) file->Close(); delete file; file=0; - return kFALSE; - } + TFile *file = TFile::Open(fullFilename,"CREATE"); + if(!file || !file->IsWritable()){ + AliError(Form("Can't open file <%s>!", filename.Data())); + if(file && !file->IsWritable()) file->Close(); delete file; file=0; + return kFALSE; + } file->cd(); + //SetTreeToFile(entry, file); + entry->SetVersion(id.GetVersion()); // write object (key name: "AliCDBEntry") - Bool_t result = (entry->Write("AliCDBEntry") != 0); + Bool_t result = (file->WriteTObject(entry, "AliCDBEntry") != 0); if (!result) AliError(Form("Can't write entry to file <%s>!", filename.Data())); @@ -523,9 +705,7 @@ Bool_t AliCDBGrid::PutEntry(AliCDBEntry* entry) { file->Close(); delete file; file=0; if(result) { - AliInfo(Form("CDB object stored into file %s", filename.Data())); - AliInfo(Form("using S.E. %s", fSE.Data())); - + if(!TagFileId(filename, &id)){ AliInfo(Form("CDB tagging failed. Deleting file %s!",filename.Data())); if(!gGrid->Rm(filename.Data())) @@ -536,6 +716,8 @@ Bool_t AliCDBGrid::PutEntry(AliCDBEntry* entry) { TagFileMetaData(filename, entry->GetMetaData()); } + AliInfo(Form("CDB object stored into file %s", filename.Data())); + AliInfo(Form("Storage Element: %s", fSE.Data())); return result; } //_____________________________________________________________________________ @@ -560,19 +742,26 @@ Bool_t AliCDBGrid::AddTag(TString& folderToTag, const char* tagname){ Bool_t AliCDBGrid::TagFileId(TString& filename, const AliCDBId* id){ // tag stored object in CDB table using object Id's parameters - TString addTagValue_1 = Form("addTagValue %s CDB ", filename.Data()); - TString addTagValue_2 = Form("first_run=%d last_run=%d version=%d ", + + TString dirname(filename); + Int_t dirNumber = gGrid->Mkdir(dirname.Remove(dirname.Last('/')),"-d"); + + TString addTagValue1 = Form("addTagValue %s CDB ", filename.Data()); + TString addTagValue2 = Form("first_run=%d last_run=%d version=%d ", id->GetFirstRun(), id->GetLastRun(), id->GetVersion()); - TString addTagValue_3 = Form("path_level_0=\"%s\" path_level_1=\"%s\" path_level_2=\"%s\"", - id->GetLevel0().Data(), - id->GetLevel1().Data(), - id->GetLevel2().Data()); - TString addTagValue = Form("%s%s%s", - addTagValue_1.Data(), - addTagValue_2.Data(), - addTagValue_3.Data()); + TString addTagValue3 = Form("path_level_0=\"%s\" path_level_1=\"%s\" path_level_2=\"%s\" ", + id->GetPathLevel(0).Data(), + id->GetPathLevel(1).Data(), + id->GetPathLevel(2).Data()); + //TString addTagValue4 = Form("version_path=\"%s\" dir_number=%d",Form("%d_%s",id->GetVersion(),filename.Data()),dirNumber); + TString addTagValue4 = Form("version_path=\"%09d%s\" dir_number=%d",id->GetVersion(),filename.Data(),dirNumber); + TString addTagValue = Form("%s%s%s%s", + addTagValue1.Data(), + addTagValue2.Data(), + addTagValue3.Data(), + addTagValue4.Data()); Bool_t result = kFALSE; AliDebug(2, Form("Tagging file. Tag command: %s", addTagValue.Data())); @@ -583,7 +772,29 @@ Bool_t AliCDBGrid::TagFileId(TString& filename, const AliCDBId* id){ filename.Data())); result = kFALSE; } else { - AliInfo("Object successfully tagged."); + AliDebug(2, "Object successfully tagged."); + result = kTRUE; + } + delete res; + return result; + +} + +//_____________________________________________________________________________ +Bool_t AliCDBGrid::TagShortLived(TString& filename, Bool_t value){ +// tag folder with ShortLived tag + + TString addTagValue = Form("addTagValue %s ShortLived_try value=%d", filename.Data(), value); + + Bool_t result = kFALSE; + AliDebug(2, Form("Tagging file. Tag command: %s", addTagValue.Data())); + TGridResult* res = gGrid->Command(addTagValue.Data()); + const char* resCode = res->GetKey(0,"__result__"); // '1' if success + if(resCode[0] != '1') { + AliError(Form("Couldn't add ShortLived tag value to file %s !", filename.Data())); + result = kFALSE; + } else { + AliDebug(2,"Object successfully tagged."); result = kTRUE; } delete res; @@ -595,18 +806,18 @@ Bool_t AliCDBGrid::TagFileId(TString& filename, const AliCDBId* id){ Bool_t AliCDBGrid::TagFileMetaData(TString& filename, const AliCDBMetaData* md){ // tag stored object in CDB table using object Id's parameters - TString addTagValue_1 = Form("addTagValue %s CDB_MD ", filename.Data()); - TString addTagValue_2 = Form("object_classname=\"%s\" responsible=\"%s\" beam_period=%d ", + TString addTagValue1 = Form("addTagValue %s CDB_MD ", filename.Data()); + TString addTagValue2 = Form("object_classname=\"%s\" responsible=\"%s\" beam_period=%d ", md->GetObjectClassName(), md->GetResponsible(), md->GetBeamPeriod()); - TString addTagValue_3 = Form("aliroot_version=\"%s\" comment=\"%s\"", + TString addTagValue3 = Form("aliroot_version=\"%s\" comment=\"%s\"", md->GetAliRootVersion(), md->GetComment()); TString addTagValue = Form("%s%s%s", - addTagValue_1.Data(), - addTagValue_2.Data(), - addTagValue_3.Data()); + addTagValue1.Data(), + addTagValue2.Data(), + addTagValue3.Data()); Bool_t result = kFALSE; AliDebug(2, Form("Tagging file. Tag command: %s", addTagValue.Data())); @@ -617,7 +828,7 @@ Bool_t AliCDBGrid::TagFileMetaData(TString& filename, const AliCDBMetaData* md){ filename.Data())); result = kFALSE; } else { - AliInfo("Object successfully tagged."); + AliDebug(2,"Object successfully tagged."); result = kTRUE; } return result; @@ -677,18 +888,47 @@ void AliCDBGrid::QueryValidFiles() // fills list fValidFileIds with AliCDBId objects created from file name TString filter; - MakeQueryFilter(fRun, fRun, fPathFilter, fVersion, fMetaDataFilter, filter); + MakeQueryFilter(fRun, fRun, fMetaDataFilter, filter); - TGridResult *res = gGrid->Query(fDBFolder, "Run*.root", filter, ""); - AliCDBId validFileId; - for(int i=0; iGetEntries(); i++){ - TString filename = res->GetKey(i, "lfn"); - if(filename == "") continue; - AliDebug(2,Form("Found valid file: %s", filename.Data())); - Bool_t result = FilenameToId(filename, validFileId); - if(result) { - fValidFileIds.AddLast(validFileId.Clone()); - } + TString pattern = Form("%s/Run*", fPathFilter.GetPath().Data()); + TString optionQuery = "-y"; + if(fVersion >= 0) { + pattern += Form("_v%d*", fVersion); + optionQuery = ""; + } + pattern += ".root"; + AliDebug(2,Form("pattern: %s", pattern.Data())); + + AliInfo(Form("fDBFolder = %s, pattern = %s, filter = %s",fDBFolder.Data(), pattern.Data(), filter.Data())); + + if (optionQuery == "-y"){ + AliInfo("Only latest version will be returned"); + } + + TGridResult *res = gGrid->Query(fDBFolder, pattern, filter, optionQuery.Data()); + + if (!res) { + AliError("Grid query failed"); + return; + } + + TIter next(res); + TMap *map; + while ((map = (TMap*)next())) { + TObjString *entry; + if ((entry = (TObjString *) ((TMap *)map)->GetValue("lfn"))) { + TString& filename = entry->String(); + if(filename.IsNull()) continue; + AliDebug(2,Form("Found valid file: %s", filename.Data())); + AliCDBId *validFileId = new AliCDBId; + Bool_t result = FilenameToId(filename, *validFileId); + if(result) { + fValidFileIds.AddLast(validFileId); + } + else { + delete validFileId; + } + } } delete res; @@ -696,25 +936,24 @@ void AliCDBGrid::QueryValidFiles() //_____________________________________________________________________________ void AliCDBGrid::MakeQueryFilter(Int_t firstRun, Int_t lastRun, - const AliCDBPath& pathFilter, Int_t version, const AliCDBMetaData* md, TString& result) const { // create filter for file query result = Form("CDB:first_run<=%d and CDB:last_run>=%d", firstRun, lastRun); - if(version >= 0) { - result += Form(" and CDB:version=%d", version); - } - if(pathFilter.GetLevel0() != "*") { - result += Form(" and CDB:path_level_0=\"%s\"", pathFilter.GetLevel0().Data()); - } - if(pathFilter.GetLevel1() != "*") { - result += Form(" and CDB:path_level_1=\"%s\"", pathFilter.GetLevel1().Data()); - } - if(pathFilter.GetLevel2() != "*") { - result += Form(" and CDB:path_level_2=\"%s\"", pathFilter.GetLevel2().Data()); - } +// if(version >= 0) { +// result += Form(" and CDB:version=%d", version); +// } +// if(pathFilter.GetLevel0() != "*") { +// result += Form(" and CDB:path_level_0=\"%s\"", pathFilter.GetLevel0().Data()); +// } +// if(pathFilter.GetLevel1() != "*") { +// result += Form(" and CDB:path_level_1=\"%s\"", pathFilter.GetLevel1().Data()); +// } +// if(pathFilter.GetLevel2() != "*") { +// result += Form(" and CDB:path_level_2=\"%s\"", pathFilter.GetLevel2().Data()); +// } if(md){ if(md->GetObjectClassName()[0] != '\0') { @@ -741,49 +980,75 @@ void AliCDBGrid::MakeQueryFilter(Int_t firstRun, Int_t lastRun, Int_t AliCDBGrid::GetLatestVersion(const char* path, Int_t run){ // get last version found in the database valid for run and path - TList validFileIds; - validFileIds.SetOwner(1); - AliCDBPath aCDBPath(path); if(!aCDBPath.IsValid() || aCDBPath.IsWildcard()) { AliError(Form("Invalid path in request: %s", path)); return -1; } AliCDBId query(path, run, run, -1, -1); - AliCDBId* dataId = 0; + AliCDBId* dataId = 0; // look for file matching query requests (path, runRange, version) - if(run == fRun && - fPathFilter.Comprises(aCDBPath) && fVersion < 0){ + if(run == fRun && fPathFilter.Comprises(aCDBPath) && fVersion < 0){ // look into list of valid files previously loaded with AliCDBStorage::FillValidFileIds() AliDebug(2, Form("List of files valid for run %d and for path %s was loaded. Looking there!", run, path)); dataId = GetId(fValidFileIds, query); if (!dataId) return -1; - return dataId->GetVersion(); + Int_t version = dataId->GetVersion(); + delete dataId; + return version; } // List of files valid for reqested run was not loaded. Looking directly into CDB AliDebug(2, Form("List of files valid for run %d and for path %s was not loaded. Looking directly into CDB!", run, path)); + TObjArray validFileIds; + validFileIds.SetOwner(1); + TString filter; - MakeQueryFilter(run, run, aCDBPath, -1, 0, filter); + MakeQueryFilter(run, run, 0, filter); - TGridResult *res = gGrid->Query(fDBFolder, "Run*.root", filter, ""); + TString pattern = Form("%s/Run*.root", path); + AliDebug(2,Form("pattern: %s", pattern.Data())); + + TGridResult *res = gGrid->Query(fDBFolder, pattern, filter, "-y"); AliCDBId validFileId; - for(int i=0; iGetEntries(); i++){ - TString filename = res->GetKey(i, "lfn"); - if(filename == "") continue; - if(FilenameToId(filename, validFileId)) + if (res->GetEntries()>1){ + AliWarning("Number of found entries >1, even if option -y was used"); + for(int i=0; iGetEntries(); i++){ + TString filename = res->GetKey(i, "lfn"); + if(filename == "") continue; + if(FilenameToId(filename, validFileId)) validFileIds.AddLast(validFileId.Clone()); + } + dataId = GetId(validFileIds, query); + if (!dataId) return -1; + + Int_t version = dataId->GetVersion(); + delete dataId; + return version; + } + else if (res->GetEntries()==1){ + TString filename = res->GetKey(0, "lfn"); + if(filename == "") { + AliError("The only entry found has filename empty"); + return -1; + } + if(FilenameToId(filename, validFileId)) return validFileId.GetVersion(); + else{ + AliError("Impossible to get FileId from filename"); + return -1; + } + } + else { + AliError("No entries found"); + return -1; } - delete res; - dataId = GetId(validFileIds, query); - if (!dataId) return -1; + delete res; - return dataId->GetVersion(); } @@ -807,10 +1072,6 @@ ClassImp(AliCDBGridFactory) Bool_t AliCDBGridFactory::Validate(const char* gridString) { // check if the string is valid Grid URI - // pattern: alien://hostName:Port;user;dbPath;SE - // example of a valid pattern: - // "alien://aliendb4.cern.ch:9000;colla;DBTest;ALICE::CERN::Server" -// TRegexp gridPattern("^alien://.+:[0-9]+;[a-zA-Z0-9_-.]+;.+;.+$"); TRegexp gridPattern("^alien://.+$"); return TString(gridString).Contains(gridPattern); @@ -823,18 +1084,22 @@ AliCDBParam* AliCDBGridFactory::CreateParameter(const char* gridString) { if (!Validate(gridString)) { return NULL; } - //TString buffer(gridString + sizeof("alien://") - 1); + TString buffer(gridString); TString gridUrl = "alien://"; TString user = ""; - TString dbFolder = "DBGrid"; - TString se = "ALICE::CERN::se01"; + TString dbFolder = ""; + TString se = "default"; + TString cacheFolder = ""; + Bool_t operateDisconnected = kTRUE; + Long64_t cacheSize = (UInt_t) 1024*1024*1024; // 1GB + Long_t cleanupInterval = 0; TObjArray *arr = buffer.Tokenize('?'); TIter iter(arr); TObjString *str = 0; - + while((str = (TObjString*) iter.Next())){ TString entry(str->String()); Int_t indeq = entry.Index('='); @@ -853,28 +1118,73 @@ AliCDBParam* AliCDBGridFactory::CreateParameter(const char* gridString) { if(key.Contains("grid",TString::kIgnoreCase)) { gridUrl += value; - } + } else if (key.Contains("user",TString::kIgnoreCase)){ user = value; } + else if (key.Contains("se",TString::kIgnoreCase)){ + se = value; + } + else if (key.Contains("cacheF",TString::kIgnoreCase)){ + cacheFolder = value; + if (!cacheFolder.IsNull() && !cacheFolder.EndsWith("/")) + cacheFolder += "/"; + } else if (key.Contains("folder",TString::kIgnoreCase)){ dbFolder = value; } - else if (key.Contains("se",TString::kIgnoreCase)){ - se = value; + else if (key.Contains("operateDisc",TString::kIgnoreCase)){ + if(value == "kTRUE") { + operateDisconnected = kTRUE; + } else if (value == "kFALSE") { + operateDisconnected = kFALSE; + } else if (value == "0" || value == "1") { + operateDisconnected = (Bool_t) value.Atoi(); + } else { + AliError(Form("Invalid entry! %s",entry.Data())); + return NULL; + } + } + else if (key.Contains("cacheS",TString::kIgnoreCase)){ + if(value.IsDigit()) { + cacheSize = value.Atoi(); + } else { + AliError(Form("Invalid entry! %s",entry.Data())); + return NULL; + } + } + else if (key.Contains("cleanupInt",TString::kIgnoreCase)){ + if(value.IsDigit()) { + cleanupInterval = value.Atoi(); + } else { + AliError(Form("Invalid entry! %s",entry.Data())); + return NULL; + } } else{ AliError(Form("Invalid entry! %s",entry.Data())); + return NULL; } } delete arr; arr=0; - - AliDebug(2, Form("gridUrl: %s",gridUrl.Data())); - AliDebug(2, Form("user: %s",user.Data())); - AliDebug(2, Form("dbFolder: %s",dbFolder.Data())); - AliDebug(2, Form("s.e.: %s",se.Data())); - return new AliCDBGridParam(gridUrl.Data(), user.Data(), dbFolder.Data(), se.Data()); + AliDebug(2, Form("gridUrl: %s", gridUrl.Data())); + AliDebug(2, Form("user: %s", user.Data())); + AliDebug(2, Form("dbFolder: %s", dbFolder.Data())); + AliDebug(2, Form("s.e.: %s", se.Data())); + AliDebug(2, Form("local cache folder: %s", cacheFolder.Data())); + AliDebug(2, Form("local cache operate disconnected: %d", operateDisconnected)); + AliDebug(2, Form("local cache size: %lld", cacheSize)); + AliDebug(2, Form("local cache cleanup interval: %ld", cleanupInterval)); + + if(dbFolder == ""){ + AliError("Base folder must be specified!"); + return NULL; + } + + return new AliCDBGridParam(gridUrl.Data(), user.Data(), + dbFolder.Data(), se.Data(), cacheFolder.Data(), + operateDisconnected, cacheSize, cleanupInterval); } //_____________________________________________________________________________ @@ -888,7 +1198,11 @@ AliCDBStorage* AliCDBGridFactory::Create(const AliCDBParam* param) { grid = new AliCDBGrid(gridParam->GridUrl().Data(), gridParam->GetUser().Data(), gridParam->GetDBFolder().Data(), - gridParam->GetSE().Data()); + gridParam->GetSE().Data(), + gridParam->GetCacheFolder().Data(), + gridParam->GetOperateDisconnected(), + gridParam->GetCacheSize(), + gridParam->GetCleanupInterval()); } @@ -901,7 +1215,7 @@ AliCDBStorage* AliCDBGridFactory::Create(const AliCDBParam* param) { ///////////////////////////////////////////////////////////////////////////////////////////////// // // -// AliCDBGrid Parameter class // // +// AliCDBGrid Parameter class // // // // ///////////////////////////////////////////////////////////////////////////////////////////////// @@ -913,30 +1227,40 @@ AliCDBGridParam::AliCDBGridParam(): fGridUrl(), fUser(), fDBFolder(), - fSE() + fSE(), + fCacheFolder(), + fOperateDisconnected(), + fCacheSize(), + fCleanupInterval() + { // default constructor } //_____________________________________________________________________________ -AliCDBGridParam::AliCDBGridParam(const char* gridUrl, - const char* user, - const char* dbFolder, - const char* se): +AliCDBGridParam::AliCDBGridParam(const char* gridUrl, const char* user, const char* dbFolder, + const char* se, const char* cacheFolder, Bool_t operateDisconnected, + Long64_t cacheSize, Long_t cleanupInterval): AliCDBParam(), fGridUrl(gridUrl), fUser(user), fDBFolder(dbFolder), - fSE(se) + fSE(se), + fCacheFolder(cacheFolder), + fOperateDisconnected(operateDisconnected), + fCacheSize(cacheSize), + fCleanupInterval(cleanupInterval) { // constructor - + SetType("alien"); - TString uri = Form("%s?User=%s?DBFolder=%s?SE=%s", + TString uri = Form("%s?User=%s?DBFolder=%s?SE=%s?CacheFolder=%s" + "?OperateDisconnected=%d?CacheSize=%lld?CleanupInterval=%ld", fGridUrl.Data(), fUser.Data(), - fDBFolder.Data(), fSE.Data()); + fDBFolder.Data(), fSE.Data(), fCacheFolder.Data(), + fOperateDisconnected, fCacheSize, fCleanupInterval); SetURI(uri.Data()); } @@ -952,14 +1276,15 @@ AliCDBParam* AliCDBGridParam::CloneParam() const { // clone parameter return new AliCDBGridParam(fGridUrl.Data(), fUser.Data(), - fDBFolder.Data(), fSE.Data()); + fDBFolder.Data(), fSE.Data(), fCacheFolder.Data(), + fOperateDisconnected, fCacheSize, fCleanupInterval); } //_____________________________________________________________________________ ULong_t AliCDBGridParam::Hash() const { // return Hash function - return fGridUrl.Hash()+fUser.Hash()+fDBFolder.Hash()+fSE.Hash(); + return fGridUrl.Hash()+fUser.Hash()+fDBFolder.Hash()+fSE.Hash()+fCacheFolder.Hash(); } //_____________________________________________________________________________ @@ -980,6 +1305,10 @@ Bool_t AliCDBGridParam::IsEqual(const TObject* obj) const { if(fUser != other->fUser) return kFALSE; if(fDBFolder != other->fDBFolder) return kFALSE; if(fSE != other->fSE) return kFALSE; + if(fCacheFolder != other->fCacheFolder) return kFALSE; + if(fOperateDisconnected != other->fOperateDisconnected) return kFALSE; + if(fCacheSize != other->fCacheSize) return kFALSE; + if(fCleanupInterval != other->fCleanupInterval) return kFALSE; return kTRUE; }