1 /**************************************************************************
2 * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
16 /////////////////////////////////////////////////////////////////////////////////////////////////
19 // access class to a DataBase in an AliEn storage //
21 /////////////////////////////////////////////////////////////////////////////////////////////////
25 #include <TGridResult.h>
30 #include <TObjArray.h>
31 #include <TObjString.h>
36 #include "AliCDBEntry.h"
37 #include "AliCDBGrid.h"
38 #include "AliCDBManager.h"
43 //_____________________________________________________________________________
44 AliCDBGrid::AliCDBGrid(const char *gridUrl, const char *user, const char *dbFolder,
45 const char *se, const char* cacheFolder, Bool_t operateDisconnected,
46 Long64_t cacheSize, Long_t cleanupInterval) :
53 fCacheFolder(cacheFolder),
54 fOperateDisconnected(operateDisconnected),
55 fCacheSize(cacheSize),
56 fCleanupInterval(cleanupInterval)
60 // if the same Grid is alreay active, skip connection
61 if (!gGrid || fGridUrl != gGrid->GridUrl()
62 || (( fUser != "" ) && ( fUser != gGrid->GetUser() )) ) {
63 // connection to the Grid
64 AliInfo("Connection to the Grid...");
66 AliInfo(Form("gGrid = %p; fGridUrl = %s; gGrid->GridUrl() = %s",gGrid,fGridUrl.Data(), gGrid->GridUrl()));
67 AliInfo(Form("fUser = %s; gGrid->GetUser() = %s",fUser.Data(), gGrid->GetUser()));
69 TGrid::Connect(fGridUrl.Data(),fUser.Data());
73 AliError("Connection failed!");
77 TString initDir(gGrid->Pwd(0));
78 if (fDBFolder[0] != '/') {
79 fDBFolder.Prepend(initDir);
82 // check DBFolder: trying to cd to DBFolder; if it does not exist, create it
83 if(!gGrid->Cd(fDBFolder.Data(),0)){
84 AliDebug(2,Form("Creating new folder <%s> ...",fDBFolder.Data()));
85 TGridResult* res = gGrid->Command(Form("mkdir -p %s",fDBFolder.Data()));
86 TString result = res->GetKey(0,"__result__");
88 AliFatal(Form("Cannot create folder <%s> !",fDBFolder.Data()));
92 AliDebug(2,Form("Folder <%s> found",fDBFolder.Data()));
95 // removes any '/' at the end of path, then append one '/'
96 while(fDBFolder.EndsWith("/")) fDBFolder.Remove(fDBFolder.Last('/'));
100 fBaseFolder = fDBFolder;
104 // Check if local cache folder is already defined
105 TString origCache(TFile::GetCacheFileDir());
106 if(fCacheFolder.Length() > 0) {
107 if(origCache.Length() == 0) {
108 AliInfo(Form("Setting local cache to: %s", fCacheFolder.Data()));
109 } else if(fCacheFolder != origCache) {
110 AliWarning(Form("Local cache folder was already defined, changing it to: %s",
111 fCacheFolder.Data()));
114 // default settings are: operateDisconnected=kTRUE, forceCacheread = kFALSE
115 if(!TFile::SetCacheFileDir(fCacheFolder.Data(), fOperateDisconnected)) {
116 AliError(Form("Could not set cache folder %s !", fCacheFolder.Data()));
119 // reset fCacheFolder because the function may have
120 // slightly changed the folder name (e.g. '/' added)
121 fCacheFolder = TFile::GetCacheFileDir();
124 // default settings are: cacheSize=1GB, cleanupInterval = 0
125 if(!TFile::ShrinkCacheFileDir(fCacheSize, fCleanupInterval)) {
126 AliError(Form("Could not set following values "
127 "to ShrinkCacheFileDir: cacheSize = %lld, cleanupInterval = %ld !",
128 fCacheSize, fCleanupInterval));
132 // return to the initial directory
133 gGrid->Cd(initDir.Data(),0);
135 fNretry = 3; // default
136 fInitRetrySeconds = 5; // default
139 //_____________________________________________________________________________
140 AliCDBGrid::~AliCDBGrid() {
142 delete gGrid; gGrid=0;
146 //_____________________________________________________________________________
147 Bool_t AliCDBGrid::FilenameToId(TString& filename, AliCDBId& id) {
148 // build AliCDBId from full path filename (fDBFolder/path/Run#x_#y_v#z_s0.root)
150 if(filename.Contains(fDBFolder)){
151 filename = filename(fDBFolder.Length(),filename.Length()-fDBFolder.Length());
154 TString idPath = filename(0,filename.Last('/'));
156 if(!id.IsValid()) return kFALSE;
158 filename=filename(idPath.Length()+1,filename.Length()-idPath.Length());
161 // valid filename: Run#firstRun_#lastRun_v#version_s0.root
162 TRegexp keyPattern("^Run[0-9]+_[0-9]+_v[0-9]+_s0.root$");
163 keyPattern.Index(filename, &mSize);
166 // TODO backward compatibility ... maybe remove later!
168 TRegexp oldKeyPattern("^Run[0-9]+_[0-9]+_v[0-9]+.root$");
169 oldKeyPattern.Index(filename, &oldmSize);
171 AliDebug(2,Form("Bad filename <%s>.", filename.Data()));
174 AliDebug(2,Form("Old filename format <%s>.", filename.Data()));
175 id.SetSubVersion(-11); // TODO trick to ensure backward compatibility
179 id.SetSubVersion(-1); // TODO trick to ensure backward compatibility
182 filename.Resize(filename.Length() - sizeof(".root") + 1);
184 TObjArray* strArray = (TObjArray*) filename.Tokenize("_");
186 TString firstRunString(((TObjString*) strArray->At(0))->GetString());
187 id.SetFirstRun(atoi(firstRunString.Data() + 3));
188 id.SetLastRun(atoi(((TObjString*) strArray->At(1))->GetString()));
190 TString verString(((TObjString*) strArray->At(2))->GetString());
191 id.SetVersion(atoi(verString.Data() + 1));
198 //_____________________________________________________________________________
199 Bool_t AliCDBGrid::IdToFilename(const AliCDBId& id, TString& filename) const {
200 // build file name from AliCDBId (path, run range, version) and fDBFolder
202 if (!id.GetAliCDBRunRange().IsValid()) {
203 AliDebug(2,Form("Invalid run range <%d, %d>.",
204 id.GetFirstRun(), id.GetLastRun()));
208 if (id.GetVersion() < 0) {
209 AliDebug(2,Form("Invalid version <%d>.", id.GetVersion()));
213 filename = Form("Run%d_%d_v%d",
218 if (id.GetSubVersion() != -11) filename += "_s0"; // TODO to ensure backward compatibility
221 filename.Prepend(fDBFolder + id.GetPath() + '/');
226 //_____________________________________________________________________________
227 void AliCDBGrid::SetRetry(Int_t nretry, Int_t initsec) {
229 // Function to set the exponential retry for putting entries in the OCDB
231 AliWarning("WARNING!!! You are changing the exponential retry times and delay: this function should be used by experts!");
233 fInitRetrySeconds = initsec;
234 AliDebug(2,Form("fNretry = %d, fInitRetrySeconds = %d", fNretry, fInitRetrySeconds));
238 //_____________________________________________________________________________
239 Bool_t AliCDBGrid::PrepareId(AliCDBId& id) {
240 // prepare id (version) of the object that will be stored (called by PutEntry)
242 TString initDir(gGrid->Pwd(0));
244 TString dirName(fDBFolder);
246 Bool_t dirExist=kFALSE;
250 // go to the path; if directory does not exist, create it
251 for(int i=0;i<3;i++){
252 //TString cmd("find -d ");
253 //cmd += Form("%s ",dirName);
255 //gGrid->Command(cmd.Data());
256 dirName+=Form("%s/",id.GetPathLevel(i).Data());
257 dirExist=gGrid->Cd(dirName,0);
259 AliDebug(2,Form("Creating new folder <%s> ...",dirName.Data()));
260 if(!gGrid->Mkdir(dirName,"",0)){
261 AliError(Form("Cannot create directory <%s> !",dirName.Data()));
262 gGrid->Cd(initDir.Data());
266 // if folders are new add tags to them
268 // TODO Currently disabled
269 // add short lived tag!
270 // AliInfo("Tagging level 1 folder with \"ShortLived\" tag");
271 // if(!AddTag(dirName,"ShortLived_try")){
272 // AliError(Form("Could not tag folder %s !", dirName.Data()));
273 // if(!gGrid->Rmdir(dirName.Data())){
274 // AliError(Form("Unexpected: could not remove %s directory!", dirName.Data()));
280 AliDebug(2,"Tagging level 2 folder with \"CDB\" and \"CDB_MD\" tag");
281 if(!AddTag(dirName,"CDB")){
282 AliError(Form("Could not tag folder %s !", dirName.Data()));
283 if(!gGrid->Rmdir(dirName.Data())){
284 AliError(Form("Unexpected: could not remove %s directory!", dirName.Data()));
288 if(!AddTag(dirName,"CDB_MD")){
289 AliError(Form("Could not tag folder %s !", dirName.Data()));
290 if(!gGrid->Rmdir(dirName.Data())){
291 AliError(Form("Unexpected: could not remove %s directory!", dirName.Data()));
296 // TODO Currently disabled
297 // add short lived tag!
298 // TString path=id.GetPath();
299 // if(AliCDBManager::Instance()->IsShortLived(path.Data())) {
300 // AliInfo(Form("Tagging %s as short lived", dirName.Data()));
301 // if(!TagShortLived(dirName, kTRUE)){
302 // AliError(Form("Could not tag folder %s !", dirName.Data()));
303 // if(!gGrid->Rmdir(dirName.Data())){
304 // AliError(Form("Unexpected: could not remove %s directory!", dirName.Data()));
309 // AliInfo(Form("Tagging %s as long lived", dirName.Data()));
310 // if(!TagShortLived(dirName, kFALSE)){
311 // AliError(Form("Could not tag folder %s !", dirName.Data()));
312 // if(!gGrid->Rmdir(dirName.Data())){
313 // AliError(Form("Unexpected: could not remove %s directory!", dirName.Data()));
321 gGrid->Cd(initDir,0);
324 AliCDBId anId; // the id got from filename
325 AliCDBRunRange lastRunRange(-1,-1); // highest runRange found
326 Int_t lastVersion=0; // highest version found
328 TGridResult *res = gGrid->Ls(dirName);
330 //loop on the files in the directory, look for highest version
331 for(int i=0; i < res->GetEntries(); i++){
332 filename=res->GetFileNamePath(i);
333 if (!FilenameToId(filename, anId)) continue;
334 if (anId.GetAliCDBRunRange().Overlaps(id.GetAliCDBRunRange()) && anId.GetVersion() > lastVersion) {
335 lastVersion = anId.GetVersion();
336 lastRunRange = anId.GetAliCDBRunRange();
342 // GRP entries with explicitly set version escape default incremental versioning
343 if(id.GetPath().Contains("GRP") && id.HasVersion() && lastVersion!=0)
345 AliDebug(5,Form("Entry %s won't be put in the destination OCDB", id.ToString().Data()));
349 id.SetVersion(lastVersion + 1);
352 TString lastStorage = id.GetLastStorage();
353 if(lastStorage.Contains(TString("new"), TString::kIgnoreCase) && id.GetVersion() > 1 ){
354 AliDebug(2, Form("A NEW object is being stored with version %d",
356 AliDebug(2, Form("and it will hide previously stored object with version %d!",
360 if(!lastRunRange.IsAnyRange() && !(lastRunRange.IsEqual(&id.GetAliCDBRunRange())))
361 AliWarning(Form("Run range modified w.r.t. previous version (Run%d_%d_v%d)",
362 lastRunRange.GetFirstRun(), lastRunRange.GetLastRun(), id.GetVersion()));
367 //_____________________________________________________________________________
368 AliCDBId* AliCDBGrid::GetId(const TObjArray& validFileIds, const AliCDBId& query) {
369 // look for the Id that matches query's requests (highest or exact version)
371 if(validFileIds.GetEntriesFast() < 1)
374 TIter iter(&validFileIds);
379 while((anIdPtr = dynamic_cast<AliCDBId*> (iter.Next()))){
380 if(anIdPtr->GetPath() != query.GetPath()) continue;
382 //if(!CheckVersion(query, anIdPtr, result)) return NULL;
384 if (!query.HasVersion()){ // look for highest version
385 if(result && result->GetVersion() > anIdPtr->GetVersion()) continue;
386 if(result && result->GetVersion() == anIdPtr->GetVersion()) {
387 AliError(Form("More than one object valid for run %d, version %d!",
388 query.GetFirstRun(), anIdPtr->GetVersion()));
391 result = new AliCDBId(*anIdPtr);
392 } else { // look for specified version
393 if(query.GetVersion() != anIdPtr->GetVersion()) continue;
394 if(result && result->GetVersion() == anIdPtr->GetVersion()){
395 AliError(Form("More than one object valid for run %d, version %d!",
396 query.GetFirstRun(), anIdPtr->GetVersion()));
399 result = new AliCDBId(*anIdPtr);
407 //_____________________________________________________________________________
408 AliCDBId* AliCDBGrid::GetEntryId(const AliCDBId& queryId) {
409 // get AliCDBId from the database
410 // User must delete returned object
414 AliCDBId selectedId(queryId);
415 if (!selectedId.HasVersion()) {
416 // if version is not specified, first check the selection criteria list
417 GetSelection(&selectedId);
420 TObjArray validFileIds;
421 validFileIds.SetOwner(1);
423 // look for file matching query requests (path, runRange, version)
424 if(selectedId.GetFirstRun() == fRun && fPathFilter.Comprises(selectedId.GetAliCDBPath()) &&
425 fVersion == selectedId.GetVersion() && !fMetaDataFilter){
426 // look into list of valid files previously loaded with AliCDBStorage::FillValidFileIds()
427 AliDebug(2, Form("List of files valid for run %d was loaded. Looking there for fileids valid for path %s!",
428 selectedId.GetFirstRun(), selectedId.GetPath().Data()));
429 dataId = GetId(fValidFileIds, selectedId);
432 // List of files valid for reqested run was not loaded. Looking directly into CDB
433 AliDebug(2, Form("List of files valid for run %d and version %d was not loaded. Looking directly into CDB for fileids valid for path %s!",
434 selectedId.GetFirstRun(), selectedId.GetVersion(), selectedId.GetPath().Data()));
437 MakeQueryFilter(selectedId.GetFirstRun(), selectedId.GetLastRun(), 0, filter);
439 TString pattern = ".root";
440 TString optionQuery = "-y -m";
441 if(selectedId.GetVersion() >= 0) {
442 pattern.Prepend(Form("_v%d_s0",selectedId.GetVersion()));
446 TString folderCopy(Form("%s%s/Run",fDBFolder.Data(),selectedId.GetPath().Data()));
448 if (optionQuery.Contains("-y")){
449 AliInfo("Only latest version will be returned");
452 AliDebug(2,Form("** fDBFolder = %s, pattern = %s, filter = %s",folderCopy.Data(), pattern.Data(), filter.Data()));
453 TGridResult *res = gGrid->Query(folderCopy, pattern, filter, optionQuery.Data());
455 for(int i=0; i<res->GetEntries(); i++){
456 AliCDBId *validFileId = new AliCDBId();
457 TString filename = res->GetKey(i, "lfn");
458 if(filename == "") continue;
459 if(FilenameToId(filename, *validFileId))
460 validFileIds.AddLast(validFileId);
464 return 0; // this should be only in case of file catalogue glitch
467 dataId = GetId(validFileIds, selectedId);
473 //_____________________________________________________________________________
474 AliCDBEntry* AliCDBGrid::GetEntry(const AliCDBId& queryId) {
475 // get AliCDBEntry from the database
477 AliCDBId* dataId = GetEntryId(queryId);
480 AliFatal(TString::Format("No valid CDB object found! request was: %s", queryId.ToString().Data()));
485 if (!IdToFilename(*dataId, filename)) {
486 AliDebug(2,Form("Bad data ID encountered! Subnormal error!"));
488 AliFatal(TString::Format("No valid CDB object found! request was: %s", queryId.ToString().Data()));
491 AliCDBEntry* anEntry = GetEntryFromFile(filename, dataId);
495 AliFatal(TString::Format("No valid CDB object found! request was: %s", queryId.ToString().Data()));
500 //_____________________________________________________________________________
501 AliCDBEntry* AliCDBGrid::GetEntryFromFile(TString& filename, AliCDBId* dataId){
502 // Get AliCBEntry object from file "filename"
504 AliDebug(2,Form("Opening file: %s",filename.Data()));
506 filename.Prepend("/alien");
508 // if option="CACHEREAD" TFile will use the local caching facility!
509 TString option="READ";
510 if(fCacheFolder != ""){
512 // Check if local cache folder was changed in the meanwhile
513 TString origCache(TFile::GetCacheFileDir());
514 if(fCacheFolder != origCache) {
515 AliWarning(Form("Local cache folder has been overwritten!! fCacheFolder = %s origCache = %s",
516 fCacheFolder.Data(), origCache.Data()));
517 TFile::SetCacheFileDir(fCacheFolder.Data(), fOperateDisconnected);
518 TFile::ShrinkCacheFileDir(fCacheSize, fCleanupInterval);
521 option.Prepend("CACHE");
524 AliDebug(2, Form("Option: %s", option.Data()));
526 TFile *file = TFile::Open(filename, option);
528 AliDebug(2,Form("Can't open file <%s>!", filename.Data()));
532 // get the only AliCDBEntry object from the file
533 // the object in the file is an AliCDBEntry entry named "AliCDBEntry"
535 AliCDBEntry* anEntry = dynamic_cast<AliCDBEntry*> (file->Get("AliCDBEntry"));
538 AliDebug(2,Form("Bad storage data: file does not contain an AliCDBEntry object!"));
543 // The object's Id is not reset during storage
544 // If object's Id runRange or version do not match with filename,
545 // it means that someone renamed file by hand. In this case a warning msg is issued.
548 AliCDBId entryId = anEntry->GetId();
549 Int_t tmpSubVersion = dataId->GetSubVersion();
550 dataId->SetSubVersion(entryId.GetSubVersion()); // otherwise filename and id may mismatch
551 if(!entryId.IsEqual(dataId)){
552 AliWarning(Form("Mismatch between file name and object's Id!"));
553 AliWarning(Form("File name: %s", dataId->ToString().Data()));
554 AliWarning(Form("Object's Id: %s", entryId.ToString().Data()));
556 dataId->SetSubVersion(tmpSubVersion);
559 anEntry->SetLastStorage("grid");
561 // Check whether entry contains a TTree. In case load the tree in memory!
562 LoadTreeFromFile(anEntry);
564 // close file, return retieved entry
565 file->Close(); delete file; file=0;
570 //_____________________________________________________________________________
571 TList* AliCDBGrid::GetEntries(const AliCDBId& queryId) {
572 // multiple request (AliCDBStorage::GetAll)
574 TList* result = new TList();
577 TObjArray validFileIds;
578 validFileIds.SetOwner(1);
580 Bool_t alreadyLoaded = kFALSE;
582 // look for file matching query requests (path, runRange)
583 if(queryId.GetFirstRun() == fRun &&
584 fPathFilter.Comprises(queryId.GetAliCDBPath()) && fVersion < 0 && !fMetaDataFilter){
585 // look into list of valid files previously loaded with AliCDBStorage::FillValidFileIds()
586 AliDebug(2,Form("List of files valid for run %d and for path %s was loaded. Looking there!",
587 queryId.GetFirstRun(), queryId.GetPath().Data()));
589 alreadyLoaded = kTRUE;
592 // List of files valid for reqested run was not loaded. Looking directly into CDB
593 AliDebug(2,Form("List of files valid for run %d and for path %s was not loaded. Looking directly into CDB!",
594 queryId.GetFirstRun(), queryId.GetPath().Data()));
597 MakeQueryFilter(queryId.GetFirstRun(), queryId.GetLastRun(), 0, filter);
599 TString path = queryId.GetPath();
601 TString pattern = "Run*.root";
602 TString optionQuery = "-y";
604 TString addFolder = "";
605 if (!path.Contains("*")){
606 if (!path.BeginsWith("/")) addFolder += "/";
610 if (path.BeginsWith("/")) path.Remove(0,1);
611 if (path.EndsWith("/")) path.Remove(path.Length()-1,1);
612 TObjArray* tokenArr = path.Tokenize("/");
613 if (tokenArr->GetEntries() != 3) {
614 AliError("Not a 3 level path! Keeping old query...");
615 pattern.Prepend(path+"/");
618 TString str0 = ((TObjString*)tokenArr->At(0))->String();
619 TString str1 = ((TObjString*)tokenArr->At(1))->String();
620 TString str2 = ((TObjString*)tokenArr->At(2))->String();
621 if (str0 != "*" && str1 != "*" && str2 == "*"){
622 // e.g. "ITS/Calib/*"
623 addFolder = "/"+str0+"/"+str1;
625 else if (str0 != "*" && str1 == "*" && str2 == "*"){
627 addFolder = "/"+str0;
629 else if (str0 == "*" && str1 == "*" && str2 == "*"){
631 // do nothing: addFolder is already an empty string;
634 // e.g. "ITS/*/RecoParam"
635 pattern.Prepend(path+"/");
638 delete tokenArr; tokenArr=0;
641 TString folderCopy(Form("%s%s",fDBFolder.Data(),addFolder.Data()));
643 AliDebug(2,Form("fDBFolder = %s, pattern = %s, filter = %s",folderCopy.Data(), pattern.Data(), filter.Data()));
645 TGridResult *res = gGrid->Query(folderCopy, pattern, filter, optionQuery.Data());
648 AliError("Grid query failed");
652 for(int i=0; i<res->GetEntries(); i++){
653 AliCDBId *validFileId = new AliCDBId();
654 TString filename = res->GetKey(i, "lfn");
655 if(filename == "") continue;
656 if(FilenameToId(filename, *validFileId))
657 validFileIds.AddLast(validFileId);
664 iter = new TIter(&fValidFileIds);
666 iter = new TIter(&validFileIds);
669 TObjArray selectedIds;
670 selectedIds.SetOwner(1);
672 // loop on list of valid Ids to select the right version to get.
673 // According to query and to the selection criteria list, version can be the highest or exact
677 AliCDBPath queryPath = queryId.GetAliCDBPath();
678 while((anIdPtr = dynamic_cast<AliCDBId*> (iter->Next()))){
679 AliCDBPath thisCDBPath = anIdPtr->GetAliCDBPath();
680 if(!(queryPath.Comprises(thisCDBPath)) || pathCopy.GetPath() == thisCDBPath.GetPath()) continue;
681 pathCopy = thisCDBPath;
683 // check the selection criteria list for this query
684 AliCDBId thisId(*anIdPtr);
685 thisId.SetVersion(queryId.GetVersion());
686 if(!thisId.HasVersion()) GetSelection(&thisId);
689 dataId = GetId(fValidFileIds, thisId);
691 dataId = GetId(validFileIds, thisId);
693 if(dataId) selectedIds.Add(dataId);
698 // selectedIds contains the Ids of the files matching all requests of query!
699 // All the objects are now ready to be retrieved
700 iter = new TIter(&selectedIds);
701 while((anIdPtr = dynamic_cast<AliCDBId*> (iter->Next()))){
703 if (!IdToFilename(*anIdPtr, filename)) {
704 AliDebug(2,Form("Bad data ID encountered! Subnormal error!"));
708 AliCDBEntry* anEntry = GetEntryFromFile(filename, anIdPtr);
710 if(anEntry) result->Add(anEntry);
718 //_____________________________________________________________________________
719 Bool_t AliCDBGrid::PutEntry(AliCDBEntry* entry, const char* mirrors) {
720 // put an AliCDBEntry object into the database
722 AliCDBId& id = entry->GetId();
724 // set version for the entry to be stored
725 if (!PrepareId(id)) return kFALSE;
727 // build filename from entry's id
729 if (!IdToFilename(id, filename)) {
730 AliError("Bad ID encountered! Subnormal error!");
734 TString folderToTag = Form("%s%s",
736 id.GetPath().Data());
738 TDirectory* saveDir = gDirectory;
740 TString fullFilename = Form("/alien%s", filename.Data());
741 TString seMirrors(mirrors);
742 if(seMirrors.IsNull() || seMirrors.IsWhitespace()) seMirrors=GetMirrorSEs();
743 // specify SE to filename
744 // if a list of SEs was passed to this method or set via SetMirrorSEs, set the first as SE for opening the file.
745 // The other SEs will be used in cascade in case of failure in opening the file.
746 // The remaining SEs will be used to create replicas.
747 TObjArray *arraySEs = seMirrors.Tokenize(',');
748 Int_t nSEs = arraySEs->GetEntries();
749 Int_t remainingSEs = 1;
751 if (fSE != "default") fullFilename += Form("?se=%s",fSE.Data());
758 AliDebug(2, Form("fNretry = %d, fInitRetrySeconds = %d",fNretry,fInitRetrySeconds));
759 TString targetSE("");
761 Bool_t result = kFALSE;
762 Bool_t reOpenResult = kFALSE;
763 Int_t reOpenAttempts=0;
764 while( !reOpenResult && reOpenAttempts<2){ //loop to check the file after closing it, to catch the unlikely but possible case when the file
765 // is cleaned up by alien just before closing as a consequence of a network disconnection while writing
767 while( !file && remainingSEs>0){
769 TObjString *target = (TObjString*) arraySEs->At(nSEs-remainingSEs);
770 targetSE=target->String();
771 if ( !(targetSE.BeginsWith("ALICE::") && targetSE.CountChar(':')==4) ) {
772 AliError(Form("\"%s\" is an invalid storage element identifier.",targetSE.Data()));
775 if(fullFilename.Contains('?')) fullFilename.Remove(fullFilename.Last('?'));
776 fullFilename += Form("?se=%s",targetSE.Data());
778 Int_t remainingAttempts=fNretry;
779 Int_t nsleep = fInitRetrySeconds; // number of seconds between attempts. We let it increase exponentially
780 AliDebug(2, Form("Uploading file into SE #%d: %s",nSEs-remainingSEs+1,targetSE.Data()));
781 while(remainingAttempts > 0) {
782 AliDebug(2, Form("Uploading file into OCDB at %s - Attempt #%d",targetSE.Data(),fNretry-remainingAttempts+1));
784 file = TFile::Open(fullFilename,"CREATE");
785 if(!file || !file->IsWritable()){
786 if(file) file->Close(); delete file; file=0; // file is not writable
787 TString message(TString::Format("Attempt %d failed.",fNretry-remainingAttempts));
788 if(remainingAttempts>0) {
789 message += " Sleeping for "; message += nsleep; message += " seconds";
791 if(remainingSEs>0) message += " Trying to upload at next SE";
793 AliDebug(2, message.Data());
794 if(remainingAttempts>0) sleep(nsleep);
798 nsleep*=fInitRetrySeconds;
803 AliError(Form("All %d attempts have failed on all %d SEs. Returning...",fNretry,nSEs));
809 //SetTreeToFile(entry, file);
810 entry->SetVersion(id.GetVersion());
812 // write object (key name: "AliCDBEntry")
813 result = (file->WriteTObject(entry, "AliCDBEntry") != 0);
814 if (!result) AliError(Form("Can't write entry to file <%s>!", filename.Data()));
819 AliDebug(2, Form("Reopening file %s for checking its correctness",fullFilename.Data()));
820 TFile* ffile = TFile::Open(fullFilename.Data(),"READ");
822 reOpenResult = kFALSE;
823 AliInfo(Form("The file %s was closed successfully but cannot be reopened. Trying now to regenerate it (regeneration attempt number %d)",
824 fullFilename.Data(),++reOpenAttempts));
826 AliDebug(2, Form("Removing file %s", filename.Data()));
827 if(!gGrid->Rm(filename.Data()))
828 AliError("Can't delete file!");
831 reOpenResult = kTRUE;
834 delete ffile; ffile=0;
838 if (saveDir) saveDir->cd(); else gROOT->cd();
841 if(result && reOpenResult) {
843 if(!TagFileId(filename, &id)){
844 AliInfo(Form("CDB tagging failed. Deleting file %s!",filename.Data()));
845 if(!gGrid->Rm(filename.Data()))
846 AliError("Can't delete file!");
850 TagFileMetaData(filename, entry->GetMetaData());
852 AliError("The file could not be opend or the object could not be written");
853 if(!gGrid->Rm(filename.Data()))
854 AliError("Can't delete file!");
858 AliInfo(Form("CDB object stored into file %s", filename.Data()));
860 AliInfo(Form("Storage Element: %s", fSE.Data()));
862 AliInfo(Form("Storage Element: %s", targetSE.Data()));
864 //In case of other SEs specified by the user, mirror the file to the remaining SEs
865 for(Int_t i=0; i<nSEs; i++){
866 if(i==nSEs-remainingSEs-1) continue; // skip mirroring to the SE where the file was saved
867 TString mirrorCmd("mirror ");
868 mirrorCmd += filename;
870 TObjString *target = (TObjString*) arraySEs->At(i);
871 TString mirrorSE(target->String());
872 mirrorCmd += mirrorSE;
873 AliDebug(5,Form("mirror command: \"%s\"",mirrorCmd.Data()));
874 AliInfo(Form("Mirroring to storage element: %s", mirrorSE.Data()));
875 gGrid->Command(mirrorCmd.Data());
877 arraySEs->Delete(); arraySEs=0;
882 //_____________________________________________________________________________
883 Bool_t AliCDBGrid::AddTag(TString& folderToTag, const char* tagname){
884 // add "tagname" tag (CDB or CDB_MD) to folder where object will be stored
886 Bool_t result = kTRUE;
887 AliDebug(2, Form("adding %s tag to folder %s", tagname, folderToTag.Data()));
888 TString addTag = Form("addTag %s %s", folderToTag.Data(), tagname);
889 TGridResult *gridres = gGrid->Command(addTag.Data());
890 const char* resCode = gridres->GetKey(0,"__result__"); // '1' if success
891 if(resCode[0] != '1') {
892 AliError(Form("Couldn't add %s tags to folder %s !",
893 tagname, folderToTag.Data()));
900 //_____________________________________________________________________________
901 Bool_t AliCDBGrid::TagFileId(TString& filename, const AliCDBId* id){
902 // tag stored object in CDB table using object Id's parameters
905 TString dirname(filename);
906 Int_t dirNumber = gGrid->Mkdir(dirname.Remove(dirname.Last('/')),"-d");
908 TString addTagValue1 = Form("addTagValue %s CDB ", filename.Data());
909 TString addTagValue2 = Form("first_run=%d last_run=%d version=%d ",
913 TString addTagValue3 = Form("path_level_0=\"%s\" path_level_1=\"%s\" path_level_2=\"%s\" ",
914 id->GetPathLevel(0).Data(),
915 id->GetPathLevel(1).Data(),
916 id->GetPathLevel(2).Data());
917 //TString addTagValue4 = Form("version_path=\"%s\" dir_number=%d",Form("%d_%s",id->GetVersion(),filename.Data()),dirNumber);
918 TString addTagValue4 = Form("version_path=\"%09d%s\" dir_number=%d",id->GetVersion(),filename.Data(),dirNumber);
919 TString addTagValue = Form("%s%s%s%s",
923 addTagValue4.Data());
925 Bool_t result = kFALSE;
926 AliDebug(2, Form("Tagging file. Tag command: %s", addTagValue.Data()));
927 TGridResult* res = gGrid->Command(addTagValue.Data());
928 const char* resCode = res->GetKey(0,"__result__"); // '1' if success
929 if(resCode[0] != '1') {
930 AliError(Form("Couldn't add CDB tag value to file %s !",
934 AliDebug(2, "Object successfully tagged.");
942 //_____________________________________________________________________________
943 Bool_t AliCDBGrid::TagShortLived(TString& filename, Bool_t value){
944 // tag folder with ShortLived tag
946 TString addTagValue = Form("addTagValue %s ShortLived_try value=%d", filename.Data(), value);
948 Bool_t result = kFALSE;
949 AliDebug(2, Form("Tagging file. Tag command: %s", addTagValue.Data()));
950 TGridResult* res = gGrid->Command(addTagValue.Data());
951 const char* resCode = res->GetKey(0,"__result__"); // '1' if success
952 if(resCode[0] != '1') {
953 AliError(Form("Couldn't add ShortLived tag value to file %s !", filename.Data()));
956 AliDebug(2,"Object successfully tagged.");
964 //_____________________________________________________________________________
965 Bool_t AliCDBGrid::TagFileMetaData(TString& filename, const AliCDBMetaData* md){
966 // tag stored object in CDB table using object Id's parameters
968 TString addTagValue1 = Form("addTagValue %s CDB_MD ", filename.Data());
969 TString addTagValue2 = Form("object_classname=\"%s\" responsible=\"%s\" beam_period=%d ",
970 md->GetObjectClassName(),
971 md->GetResponsible(),
972 md->GetBeamPeriod());
973 TString addTagValue3 = Form("aliroot_version=\"%s\" comment=\"%s\"",
974 md->GetAliRootVersion(),
976 TString addTagValue = Form("%s%s%s",
979 addTagValue3.Data());
981 Bool_t result = kFALSE;
982 AliDebug(2, Form("Tagging file. Tag command: %s", addTagValue.Data()));
983 TGridResult* res = gGrid->Command(addTagValue.Data());
984 const char* resCode = res->GetKey(0,"__result__"); // '1' if success
985 if(resCode[0] != '1') {
986 AliWarning(Form("Couldn't add CDB_MD tag value to file %s !",
990 AliDebug(2,"Object successfully tagged.");
996 //_____________________________________________________________________________
997 TList* AliCDBGrid::GetIdListFromFile(const char* fileName){
999 TString turl(fileName);
1000 turl.Prepend("/alien" + fDBFolder);
1001 turl += "?se="; turl += fSE.Data();
1002 TFile *file = TFile::Open(turl);
1004 AliError(Form("Can't open selection file <%s>!", turl.Data()));
1008 TList *list = new TList();
1016 keycycle = "AliCDBId;";
1019 id = (AliCDBId*) file->Get(keycycle);
1023 file->Close(); delete file; file=0;
1030 //_____________________________________________________________________________
1031 Bool_t AliCDBGrid::Contains(const char* path) const{
1032 // check for path in storage's DBFolder
1034 TString initDir(gGrid->Pwd(0));
1035 TString dirName(fDBFolder);
1036 dirName += path; // dirName = fDBFolder/path
1037 Bool_t result=kFALSE;
1038 if (gGrid->Cd(dirName,0)) result=kTRUE;
1039 gGrid->Cd(initDir.Data(),0);
1043 //_____________________________________________________________________________
1044 void AliCDBGrid::QueryValidFiles()
1046 // Query the CDB for files valid for AliCDBStorage::fRun
1047 // Fills list fValidFileIds with AliCDBId objects extracted from CDB files
1048 // selected from AliEn metadata.
1049 // If fVersion was not set, fValidFileIds is filled with highest versions.
1052 MakeQueryFilter(fRun, fRun, fMetaDataFilter, filter);
1054 TString path = fPathFilter.GetPath();
1056 TString pattern = "Run*";
1057 TString optionQuery = "-y";
1059 pattern += Form("_v%d_s0", fVersion);
1063 AliDebug(2,Form("pattern: %s", pattern.Data()));
1065 TString addFolder = "";
1066 if (!path.Contains("*")){
1067 if (!path.BeginsWith("/")) addFolder += "/";
1071 if (path.BeginsWith("/")) path.Remove(0,1);
1072 if (path.EndsWith("/")) path.Remove(path.Length()-1,1);
1073 TObjArray* tokenArr = path.Tokenize("/");
1074 if (tokenArr->GetEntries() != 3) {
1075 AliError("Not a 3 level path! Keeping old query...");
1076 pattern.Prepend(path+"/");
1079 TString str0 = ((TObjString*)tokenArr->At(0))->String();
1080 TString str1 = ((TObjString*)tokenArr->At(1))->String();
1081 TString str2 = ((TObjString*)tokenArr->At(2))->String();
1082 if (str0 != "*" && str1 != "*" && str2 == "*"){
1083 // e.g. "ITS/Calib/*"
1084 addFolder = "/"+str0+"/"+str1;
1086 else if (str0 != "*" && str1 == "*" && str2 == "*"){
1088 addFolder = "/"+str0;
1090 else if (str0 == "*" && str1 == "*" && str2 == "*"){
1092 // do nothing: addFolder is already an empty string;
1095 // e.g. "ITS/*/RecoParam"
1096 pattern.Prepend(path+"/");
1099 delete tokenArr; tokenArr=0;
1102 TString folderCopy(Form("%s%s",fDBFolder.Data(),addFolder.Data()));
1104 AliDebug(2,Form("fDBFolder = %s, pattern = %s, filter = %s",folderCopy.Data(), pattern.Data(), filter.Data()));
1106 if (optionQuery == "-y"){
1107 AliInfo("Only latest version will be returned");
1110 TGridResult *res = gGrid->Query(folderCopy, pattern, filter, optionQuery.Data());
1113 AliError("Grid query failed");
1119 while ((map = (TMap*)next())) {
1121 if ((entry = (TObjString *) ((TMap *)map)->GetValue("lfn"))) {
1122 TString& filename = entry->String();
1123 if(filename.IsNull()) continue;
1124 AliDebug(2,Form("Found valid file: %s", filename.Data()));
1125 AliCDBId *validFileId = new AliCDBId();
1126 Bool_t result = FilenameToId(filename, *validFileId);
1128 fValidFileIds.AddLast(validFileId);
1139 //_____________________________________________________________________________
1140 void AliCDBGrid::MakeQueryFilter(Int_t firstRun, Int_t lastRun,
1141 const AliCDBMetaData* md, TString& result) const
1143 // create filter for file query
1145 result = Form("CDB:first_run<=%d and CDB:last_run>=%d", firstRun, lastRun);
1147 // if(version >= 0) {
1148 // result += Form(" and CDB:version=%d", version);
1150 // if(pathFilter.GetLevel0() != "*") {
1151 // result += Form(" and CDB:path_level_0=\"%s\"", pathFilter.GetLevel0().Data());
1153 // if(pathFilter.GetLevel1() != "*") {
1154 // result += Form(" and CDB:path_level_1=\"%s\"", pathFilter.GetLevel1().Data());
1156 // if(pathFilter.GetLevel2() != "*") {
1157 // result += Form(" and CDB:path_level_2=\"%s\"", pathFilter.GetLevel2().Data());
1161 if(md->GetObjectClassName()[0] != '\0') {
1162 result += Form(" and CDB_MD:object_classname=\"%s\"", md->GetObjectClassName());
1164 if(md->GetResponsible()[0] != '\0') {
1165 result += Form(" and CDB_MD:responsible=\"%s\"", md->GetResponsible());
1167 if(md->GetBeamPeriod() != 0) {
1168 result += Form(" and CDB_MD:beam_period=%d", md->GetBeamPeriod());
1170 if(md->GetAliRootVersion()[0] != '\0') {
1171 result += Form(" and CDB_MD:aliroot_version=\"%s\"", md->GetAliRootVersion());
1173 if(md->GetComment()[0] != '\0') {
1174 result += Form(" and CDB_MD:comment=\"%s\"", md->GetComment());
1177 AliDebug(2, Form("filter: %s",result.Data()));
1182 /////////////////////////////////////////////////////////////////////////////////////////////////
1184 // AliCDBGrid factory //
1186 /////////////////////////////////////////////////////////////////////////////////////////////////
1188 ClassImp(AliCDBGridFactory)
1190 //_____________________________________________________________________________
1191 Bool_t AliCDBGridFactory::Validate(const char* gridString) {
1192 // check if the string is valid Grid URI
1194 TRegexp gridPattern("^alien://.+$");
1196 return TString(gridString).Contains(gridPattern);
1199 //_____________________________________________________________________________
1200 AliCDBParam* AliCDBGridFactory::CreateParameter(const char* gridString) {
1201 // create AliCDBGridParam class from the URI string
1203 if (!Validate(gridString)) {
1207 TString buffer(gridString);
1209 TString gridUrl = "alien://";
1211 TString dbFolder = "";
1212 TString se = "default";
1213 TString cacheFolder = "";
1214 Bool_t operateDisconnected = kTRUE;
1215 Long64_t cacheSize = (UInt_t) 1024*1024*1024; // 1GB
1216 Long_t cleanupInterval = 0;
1218 TObjArray *arr = buffer.Tokenize('?');
1220 TObjString *str = 0;
1222 while((str = (TObjString*) iter.Next())){
1223 TString entry(str->String());
1224 Int_t indeq = entry.Index('=');
1226 if(entry.BeginsWith("alien://")) { // maybe it's a gridUrl!
1230 AliError(Form("Invalid entry! %s",entry.Data()));
1235 TString key = entry(0,indeq);
1236 TString value = entry(indeq+1,entry.Length()-indeq);
1238 if(key.Contains("grid",TString::kIgnoreCase)) {
1241 else if (key.Contains("user",TString::kIgnoreCase)){
1244 else if (key.Contains("se",TString::kIgnoreCase)){
1247 else if (key.Contains("cacheF",TString::kIgnoreCase)){
1248 cacheFolder = value;
1249 if (!cacheFolder.IsNull() && !cacheFolder.EndsWith("/"))
1252 else if (key.Contains("folder",TString::kIgnoreCase)){
1255 else if (key.Contains("operateDisc",TString::kIgnoreCase)){
1256 if(value == "kTRUE") {
1257 operateDisconnected = kTRUE;
1258 } else if (value == "kFALSE") {
1259 operateDisconnected = kFALSE;
1260 } else if (value == "0" || value == "1") {
1261 operateDisconnected = (Bool_t) value.Atoi();
1263 AliError(Form("Invalid entry! %s",entry.Data()));
1267 else if (key.Contains("cacheS",TString::kIgnoreCase)){
1268 if(value.IsDigit()) {
1269 cacheSize = value.Atoi();
1271 AliError(Form("Invalid entry! %s",entry.Data()));
1275 else if (key.Contains("cleanupInt",TString::kIgnoreCase)){
1276 if(value.IsDigit()) {
1277 cleanupInterval = value.Atoi();
1279 AliError(Form("Invalid entry! %s",entry.Data()));
1284 AliError(Form("Invalid entry! %s",entry.Data()));
1290 AliDebug(2, Form("gridUrl: %s", gridUrl.Data()));
1291 AliDebug(2, Form("user: %s", user.Data()));
1292 AliDebug(2, Form("dbFolder: %s", dbFolder.Data()));
1293 AliDebug(2, Form("s.e.: %s", se.Data()));
1294 AliDebug(2, Form("local cache folder: %s", cacheFolder.Data()));
1295 AliDebug(2, Form("local cache operate disconnected: %d", operateDisconnected));
1296 AliDebug(2, Form("local cache size: %lld", cacheSize));
1297 AliDebug(2, Form("local cache cleanup interval: %ld", cleanupInterval));
1300 AliError("Base folder must be specified!");
1304 return new AliCDBGridParam(gridUrl.Data(), user.Data(),
1305 dbFolder.Data(), se.Data(), cacheFolder.Data(),
1306 operateDisconnected, cacheSize, cleanupInterval);
1309 //_____________________________________________________________________________
1310 AliCDBStorage* AliCDBGridFactory::Create(const AliCDBParam* param) {
1311 // create AliCDBGrid storage instance from parameters
1313 AliCDBGrid *grid = 0;
1314 if (AliCDBGridParam::Class() == param->IsA()) {
1316 const AliCDBGridParam* gridParam = (const AliCDBGridParam*) param;
1317 grid = new AliCDBGrid(gridParam->GridUrl().Data(),
1318 gridParam->GetUser().Data(),
1319 gridParam->GetDBFolder().Data(),
1320 gridParam->GetSE().Data(),
1321 gridParam->GetCacheFolder().Data(),
1322 gridParam->GetOperateDisconnected(),
1323 gridParam->GetCacheSize(),
1324 gridParam->GetCleanupInterval());
1328 if(!gGrid && grid) {
1329 delete grid; grid=0;
1335 /////////////////////////////////////////////////////////////////////////////////////////////////
1337 // AliCDBGrid Parameter class // //
1339 /////////////////////////////////////////////////////////////////////////////////////////////////
1341 ClassImp(AliCDBGridParam)
1343 //_____________________________________________________________________________
1344 AliCDBGridParam::AliCDBGridParam():
1351 fOperateDisconnected(),
1356 // default constructor
1360 //_____________________________________________________________________________
1361 AliCDBGridParam::AliCDBGridParam(const char* gridUrl, const char* user, const char* dbFolder,
1362 const char* se, const char* cacheFolder, Bool_t operateDisconnected,
1363 Long64_t cacheSize, Long_t cleanupInterval):
1367 fDBFolder(dbFolder),
1369 fCacheFolder(cacheFolder),
1370 fOperateDisconnected(operateDisconnected),
1371 fCacheSize(cacheSize),
1372 fCleanupInterval(cleanupInterval)
1378 TString uri = Form("%s?User=%s?DBFolder=%s?SE=%s?CacheFolder=%s"
1379 "?OperateDisconnected=%d?CacheSize=%lld?CleanupInterval=%ld",
1380 fGridUrl.Data(), fUser.Data(),
1381 fDBFolder.Data(), fSE.Data(), fCacheFolder.Data(),
1382 fOperateDisconnected, fCacheSize, fCleanupInterval);
1387 //_____________________________________________________________________________
1388 AliCDBGridParam::~AliCDBGridParam() {
1393 //_____________________________________________________________________________
1394 AliCDBParam* AliCDBGridParam::CloneParam() const {
1397 return new AliCDBGridParam(fGridUrl.Data(), fUser.Data(),
1398 fDBFolder.Data(), fSE.Data(), fCacheFolder.Data(),
1399 fOperateDisconnected, fCacheSize, fCleanupInterval);
1402 //_____________________________________________________________________________
1403 ULong_t AliCDBGridParam::Hash() const {
1404 // return Hash function
1406 return fGridUrl.Hash()+fUser.Hash()+fDBFolder.Hash()+fSE.Hash()+fCacheFolder.Hash();
1409 //_____________________________________________________________________________
1410 Bool_t AliCDBGridParam::IsEqual(const TObject* obj) const {
1411 // check if this object is equal to AliCDBParam obj
1417 if (AliCDBGridParam::Class() != obj->IsA()) {
1421 AliCDBGridParam* other = (AliCDBGridParam*) obj;
1423 if(fGridUrl != other->fGridUrl) return kFALSE;
1424 if(fUser != other->fUser) return kFALSE;
1425 if(fDBFolder != other->fDBFolder) return kFALSE;
1426 if(fSE != other->fSE) return kFALSE;
1427 if(fCacheFolder != other->fCacheFolder) return kFALSE;
1428 if(fOperateDisconnected != other->fOperateDisconnected) return kFALSE;
1429 if(fCacheSize != other->fCacheSize) return kFALSE;
1430 if(fCleanupInterval != other->fCleanupInterval) return kFALSE;