X-Git-Url: http://git.uio.no/git/?a=blobdiff_plain;f=ANALYSIS%2FAliAnalysisAlien.cxx;h=a88525a42005e2cac4af96b36ebfce6954f1da82;hb=a6889bf1ecfd1414eaf6212c5c4e3ed54e04efa8;hp=c562fec866efadeae11ec0749e74ad2f91425756;hpb=a8e31fb08ae2c581d24ad64d7700b99b339be536;p=u%2Fmrichter%2FAliRoot.git diff --git a/ANALYSIS/AliAnalysisAlien.cxx b/ANALYSIS/AliAnalysisAlien.cxx index c562fec866e..a88525a4200 100644 --- a/ANALYSIS/AliAnalysisAlien.cxx +++ b/ANALYSIS/AliAnalysisAlien.cxx @@ -29,6 +29,7 @@ #include "TError.h" #include "TROOT.h" #include "TSystem.h" +#include "TInterpreter.h" #include "TFile.h" #include "TFileCollection.h" #include "TChain.h" @@ -48,6 +49,10 @@ #include "AliAnalysisDataContainer.h" #include "AliMultiInputEventHandler.h" +using std::ofstream; +using std::ifstream; +using std::ios; +using std::endl; ClassImp(AliAnalysisAlien) #if 0 ; @@ -90,6 +95,8 @@ AliAnalysisAlien::AliAnalysisAlien() fNproofWorkers(0), fNproofWorkersPerSlave(0), fProofReset(0), + fNMCevents(0), + fNMCjobs(0), fRunNumbers(), fExecutable(), fExecutableCommand(), @@ -100,6 +107,7 @@ AliAnalysisAlien::AliAnalysisAlien() fValidationScript(), fAdditionalRootLibs(), fAdditionalLibs(), + fGeneratorLibs(), fSplitMode(), fAPIVersion(), fROOTVersion(), @@ -127,13 +135,19 @@ AliAnalysisAlien::AliAnalysisAlien() fProofCluster(), fProofDataSet(), fFileForTestMode(), - fRootVersionForProof(), fAliRootMode(), + fProofProcessOpt(), fMergeDirName(), fInputFiles(0), fPackages(0), fModules(0), - fProofParam() + fProofParam(), + fDropToShell(true), + fMCLoop(false), + fGridJobIDs(""), + fGridStages(""), + fFriendLibs(""), + fTreeName() { // Dummy ctor. SetDefaults(); @@ -163,6 +177,8 @@ AliAnalysisAlien::AliAnalysisAlien(const char *name) fNproofWorkers(0), fNproofWorkersPerSlave(0), fProofReset(0), + fNMCevents(0), + fNMCjobs(0), fRunNumbers(), fExecutable(), fExecutableCommand(), @@ -173,6 +189,7 @@ AliAnalysisAlien::AliAnalysisAlien(const char *name) fValidationScript(), fAdditionalRootLibs(), fAdditionalLibs(), + fGeneratorLibs(), fSplitMode(), fAPIVersion(), fROOTVersion(), @@ -200,13 +217,19 @@ AliAnalysisAlien::AliAnalysisAlien(const char *name) fProofCluster(), fProofDataSet(), fFileForTestMode(), - fRootVersionForProof(), fAliRootMode(), + fProofProcessOpt(), fMergeDirName(), fInputFiles(0), fPackages(0), fModules(0), - fProofParam() + fProofParam(), + fDropToShell(true), + fMCLoop(false), + fGridJobIDs(""), + fGridStages(""), + fFriendLibs(""), + fTreeName() { // Default ctor. SetDefaults(); @@ -236,6 +259,8 @@ AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other) fNproofWorkers(other.fNproofWorkers), fNproofWorkersPerSlave(other.fNproofWorkersPerSlave), fProofReset(other.fProofReset), + fNMCevents(other.fNMCevents), + fNMCjobs(other.fNMCjobs), fRunNumbers(other.fRunNumbers), fExecutable(other.fExecutable), fExecutableCommand(other.fExecutableCommand), @@ -246,6 +271,7 @@ AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other) fValidationScript(other.fValidationScript), fAdditionalRootLibs(other.fAdditionalRootLibs), fAdditionalLibs(other.fAdditionalLibs), + fGeneratorLibs(other.fGeneratorLibs), fSplitMode(other.fSplitMode), fAPIVersion(other.fAPIVersion), fROOTVersion(other.fROOTVersion), @@ -273,13 +299,19 @@ AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other) fProofCluster(other.fProofCluster), fProofDataSet(other.fProofDataSet), fFileForTestMode(other.fFileForTestMode), - fRootVersionForProof(other.fRootVersionForProof), fAliRootMode(other.fAliRootMode), + fProofProcessOpt(other.fProofProcessOpt), fMergeDirName(other.fMergeDirName), fInputFiles(0), fPackages(0), fModules(0), - fProofParam() + fProofParam(), + fDropToShell(other.fDropToShell), + fMCLoop(other.fMCLoop), + fGridJobIDs(other.fGridJobIDs), + fGridStages(other.fGridStages), + fFriendLibs(other.fFriendLibs), + fTreeName(other.fTreeName) { // Copy ctor. fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()"); @@ -351,6 +383,8 @@ AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other) fNproofWorkers = other.fNproofWorkers; fNproofWorkersPerSlave = other.fNproofWorkersPerSlave; fProofReset = other.fProofReset; + fNMCevents = other.fNMCevents; + fNMCjobs = other.fNMCjobs; fRunNumbers = other.fRunNumbers; fExecutable = other.fExecutable; fExecutableCommand = other.fExecutableCommand; @@ -361,6 +395,7 @@ AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other) fValidationScript = other.fValidationScript; fAdditionalRootLibs = other.fAdditionalRootLibs; fAdditionalLibs = other.fAdditionalLibs; + fGeneratorLibs = other.fGeneratorLibs; fSplitMode = other.fSplitMode; fAPIVersion = other.fAPIVersion; fROOTVersion = other.fROOTVersion; @@ -388,9 +423,15 @@ AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other) fProofCluster = other.fProofCluster; fProofDataSet = other.fProofDataSet; fFileForTestMode = other.fFileForTestMode; - fRootVersionForProof = other.fRootVersionForProof; fAliRootMode = other.fAliRootMode; + fProofProcessOpt = other.fProofProcessOpt; fMergeDirName = other.fMergeDirName; + fDropToShell = other.fDropToShell; + fMCLoop = other.fMCLoop; + fGridJobIDs = other.fGridJobIDs; + fGridStages = other.fGridStages; + fFriendLibs = other.fFriendLibs; + fTreeName = other.fTreeName; if (other.fInputFiles) { fInputFiles = new TObjArray(); TIter next(other.fInputFiles); @@ -512,9 +553,13 @@ AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, co { // Create the analysis manager and optionally execute the macro in filename. AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager(); - if (mgr) return mgr; + if (mgr) { + mgr->SetMCLoop(fMCLoop); + return mgr; + } mgr = new AliAnalysisManager(name); mgr->SetGridHandler((AliAnalysisGrid*)this); + mgr->SetMCLoop(fMCLoop); if (strlen(filename)) { TString line = gSystem->ExpandPathName(filename); line.Prepend(".x "); @@ -596,8 +641,8 @@ Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod) TString lib; for (Int_t i=0; iGetLibrary(i); - if (fAdditionalLibs.Contains(lib)) continue; lib = Form("lib%s.so", lib.Data()); + if (fAdditionalLibs.Contains(lib)) continue; if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " "; fAdditionalLibs += lib; } @@ -646,6 +691,7 @@ Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname) return kFALSE; } if (!LoadModule(mod)) return kFALSE; + if (!LoadFriendLibs()) return kFALSE; } else if (!LoadModules()) return kFALSE; AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager(); if (!mgr->InitAnalysis()) return kFALSE; @@ -660,12 +706,23 @@ Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname) TString execCommand = fExecutableCommand; SetAnalysisMacro(Form("%s.C", name)); SetExecutable(Form("%s.sh", name)); + fOutputFiles = GetListOfFiles("outaod"); + // Add extra files registered to the analysis manager + TString extra = GetListOfFiles("ext"); + if (!extra.IsNull()) { + extra.ReplaceAll(".root", "*.root"); + if (!fOutputFiles.IsNull()) fOutputFiles += ","; + fOutputFiles += extra; + } // SetExecutableCommand("aliroot -b -q "); SetValidationScript(Form("%s_validation.sh", name)); WriteAnalysisFile(); WriteAnalysisMacro(); WriteExecutable(); WriteValidationScript(); + WriteMergingMacro(); + WriteMergeExecutable(); + WriteValidationScript(kTRUE); SetLocalTest(kFALSE); SetProductionMode(productionMode); fAnalysisMacro = macro; @@ -697,9 +754,38 @@ Bool_t AliAnalysisAlien::LoadModules() mod = (AliAnalysisTaskCfg*)fModules->At(imod); if (!LoadModule(mod)) return kFALSE; } - return kTRUE; + // Load additional friend libraries + return LoadFriendLibs(); } +//______________________________________________________________________________ +Bool_t AliAnalysisAlien::LoadFriendLibs() const +{ +// Load libraries required for reading friends. + if (fFriendLibs.Length()) { + TObjArray *list = 0; + TString lib; + if (fFriendLibs.Contains(",")) list = fFriendLibs.Tokenize(","); + else list = fFriendLibs.Tokenize(" "); + for (Int_t ilib=0; ilibGetEntriesFast(); ilib++) { + lib = list->At(ilib)->GetName(); + lib.ReplaceAll(".so",""); + lib.ReplaceAll(".dylib",""); + lib.ReplaceAll(" ",""); + if (lib.BeginsWith("lib")) lib.Remove(0, 3); + lib.Prepend("lib"); + Int_t loaded = strlen(gSystem->GetLibraries(lib,"",kFALSE)); + if (!loaded) loaded = gSystem->Load(lib); + if (loaded < 0) { + Error("LoadModules", "Cannot load library for friends %s", lib.Data()); + return kFALSE; + } + } + delete list; + } + return kTRUE; +} + //______________________________________________________________________________ void AliAnalysisAlien::SetRunPrefix(const char *prefix) { @@ -1040,18 +1126,18 @@ Bool_t AliAnalysisAlien::CheckInputData() } //______________________________________________________________________________ -Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir) +Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir) { // Copy data from the given grid directory according a pattern and make a local // dataset. // archivefile (optional) results in that the archive containing the file is copied. archivefile can contain a list of files (semicolon-separated) which are all copied if (!Connect()) { Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection"); - return kFALSE; + return 0; } if (!DirectoryExists(griddir)) { Error("CopyLocalDataset", "Data directory %s not existing.", griddir); - return kFALSE; + return 0; } TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern); printf("Running command: %s\n", command.Data()); @@ -1059,7 +1145,7 @@ Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *patte Int_t nfound = res->GetEntries(); if (!nfound) { Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern); - return kFALSE; + return 0; } printf("... found %d files. Copying locally ...\n", nfound); @@ -1080,6 +1166,7 @@ Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *patte TString cdir = gSystem->WorkingDirectory(); gSystem->MakeDirectory(outputdir); gSystem->ChangeDirectory(outputdir); + Int_t ncopied = 0; for (Int_t i=0; iAt(i); turl = map->GetValue("turl")->GetName(); @@ -1092,34 +1179,34 @@ Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *patte TString targetFileName(filename); if (strlen(archivefile) > 0) { - // TODO here the archive in which the file resides should be determined - // however whereis returns only a guid, and guid2lfn does not work - // Therefore we use the one provided as argument for now - source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile); - targetFileName = archivefile; +// TODO here the archive in which the file resides should be determined +// however whereis returns only a guid, and guid2lfn does not work +// Therefore we use the one provided as argument for now + source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile); + targetFileName = archivefile; } if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) { - Bool_t success = kTRUE; - if (additionalArchives) - for (Int_t j=0; jGetEntriesFast(); j++) - { - TString target; - target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName()); - gSystem->MakeDirectory(gSystem->DirName(target)); - success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data())); - } + Bool_t success = kTRUE; + if (additionalArchives) { + for (Int_t j=0; jGetEntriesFast(); j++) { + TString target; + target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName()); + gSystem->MakeDirectory(gSystem->DirName(target)); + success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data())); + } + } - if (success) { - if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data())); - out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl; - } + if (success) { + if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data())); + out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl; + ncopied++; + } } } gSystem->ChangeDirectory(cdir); delete res; - if (additionalArchives) - delete additionalArchives; - return kTRUE; + delete additionalArchives; + return ncopied; } //______________________________________________________________________________ @@ -1141,6 +1228,10 @@ Bool_t AliAnalysisAlien::CreateDataset(const char *pattern) // Compose the 'find' command arguments TString format; TString command; + TString delimiter = pattern; + delimiter.Strip(); + if (delimiter.Contains(" ")) delimiter = ""; + else delimiter = " "; TString options = "-x collection "; if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles); else options += Form("-l %d ", gMaxEntries); // Protection for the find command @@ -1160,7 +1251,7 @@ Bool_t AliAnalysisAlien::CreateDataset(const char *pattern) if (!DirectoryExists(path)) { Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data()); return kFALSE; - } + } // CdWork(); if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml"; else file = Form("%s.xml", gSystem->BaseName(path)); @@ -1171,7 +1262,7 @@ Bool_t AliAnalysisAlien::CreateDataset(const char *pattern) command = "find "; command += Form("%s -o %d ",options.Data(), nstart); command += path; - command += " "; + command += delimiter; command += pattern; command += conditions; printf("command: %s\n", command.Data()); @@ -1247,7 +1338,7 @@ Bool_t AliAnalysisAlien::CreateDataset(const char *pattern) while ((os=(TObjString*)next())) { nstart = 0; stage = 0; - path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data()); + path = Form("%s/%s", fGridDataDir.Data(), os->GetString().Data()); if (!DirectoryExists(path)) continue; // CdWork(); if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml"; @@ -1260,6 +1351,7 @@ Bool_t AliAnalysisAlien::CreateDataset(const char *pattern) command = "find "; command += Form("%s -o %d ",options.Data(), nstart); command += path; + command += delimiter; command += pattern; command += conditions; TGridResult *res = gGrid->Command(command); @@ -1375,7 +1467,7 @@ Bool_t AliAnalysisAlien::CreateDataset(const char *pattern) } else { // Process a full run range. for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) { - format = Form("%%s/%s ", fRunPrefix.Data()); + format = Form("%%s/%s", fRunPrefix.Data()); nstart = 0; stage = 0; path = Form(format.Data(), fGridDataDir.Data(), irun); @@ -1399,6 +1491,7 @@ Bool_t AliAnalysisAlien::CreateDataset(const char *pattern) command = "find "; command += Form("%s -o %d ",options.Data(), nstart); command += path; + command += delimiter; command += pattern; command += conditions; TGridResult *res = gGrid->Command(command); @@ -1530,6 +1623,7 @@ Bool_t AliAnalysisAlien::CreateJDL() { // Generate a JDL file according to current settings. The name of the file is // specified by fJDLName. + AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager(); Bool_t error = kFALSE; TObjArray *arr = 0; Bool_t copy = kTRUE; @@ -1547,7 +1641,7 @@ Bool_t AliAnalysisAlien::CreateJDL() workdir += fGridWorkingDir; if (generate) { TObjString *os; - if (!fInputFiles) { + if (!fInputFiles && !fMCLoop) { Error("CreateJDL()", "Define some input files for your analysis."); error = kTRUE; } @@ -1583,13 +1677,16 @@ Bool_t AliAnalysisAlien::CreateJDL() if (!fUser.IsNull()) { fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data())); fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data())); - } - fGridJDL->SetExecutable(fExecutable, "This is the startup script"); - TString mergeExec = fExecutable; + } + TString executable = fExecutable; + if (!executable.BeginsWith("/")) + executable.Prepend(Form("%s/", workdir.Data())); + fGridJDL->SetExecutable(executable, "This is the startup script"); + TString mergeExec = executable; mergeExec.ReplaceAll(".sh", "_merge.sh"); fMergingJDL->SetExecutable(mergeExec, "This is the startup script"); mergeExec.ReplaceAll(".sh", ".C"); - fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers"); + fMergingJDL->AddToInputSandbox(Form("LF:%s", mergeExec.Data()), "List of input files to be uploaded to workers"); if (!fArguments.IsNull()) fGridJDL->SetArguments(fArguments, "Arguments for the executable command"); if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir); @@ -1607,7 +1704,7 @@ Bool_t AliAnalysisAlien::CreateJDL() fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed)); fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job"); } - if (fSplitMaxInputFileNumber > 0) { + if (fSplitMaxInputFileNumber > 0 && !fMCLoop) { fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber)); fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob"); } @@ -1646,8 +1743,10 @@ Bool_t AliAnalysisAlien::CreateJDL() } delete arr; } - fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data"); - fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node"); + if (!fMCLoop) { + fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data"); + fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node"); + } fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data"); fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node"); fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers"); @@ -1659,7 +1758,8 @@ Bool_t AliAnalysisAlien::CreateJDL() arr = fAdditionalLibs.Tokenize(" "); TIter next(arr); while ((os=(TObjString*)next())) { - if (os->GetString().Contains(".so")) continue; + if (os->GetString().Contains(".so") || + os->GetString().Contains(".dylib")) continue; fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data())); fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data())); } @@ -1673,27 +1773,37 @@ Bool_t AliAnalysisAlien::CreateJDL() fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName())); } } + const char *comment = "List of output files and archives"; if (fOutputArchive.Length()) { - arr = fOutputArchive.Tokenize(" "); + TString outputArchive = fOutputArchive; + if (!fRegisterExcludes.IsNull()) { + arr = fRegisterExcludes.Tokenize(" "); + TIter next1(arr); + while ((os=(TObjString*)next1())) { + outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),""); + outputArchive.ReplaceAll(os->GetString(),""); + } + delete arr; + } + arr = outputArchive.Tokenize(" "); TIter next(arr); Bool_t first = kTRUE; - const char *comment = "Files to be archived"; - const char *comment1 = comment; while ((os=(TObjString*)next())) { - if (!first) comment = NULL; if (!os->GetString().Contains("@") && fCloseSE.Length()) - fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment); + fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data())); else - fGridJDL->AddToOutputArchive(os->GetString(), comment); + fGridJDL->AddToSet("Output", os->GetString()); + if (first) fGridJDL->AddToSetDescription("Output", comment); first = kFALSE; } delete arr; // Output archive for the merging jdl - TString outputArchive; if (TestBit(AliAnalysisGrid::kDefaultOutputs)) { outputArchive = "log_archive.zip:std*@disk=1 "; // Add normal output files, extra files + terminate files - TString files = GetListOfFiles("outextter"); + TString files; + if (IsMergeAOD()) files = GetListOfFiles("outaodextter"); + else files = GetListOfFiles("outextter"); // Do not register files in fRegisterExcludes if (!fRegisterExcludes.IsNull()) { arr = fRegisterExcludes.Tokenize(" "); @@ -1705,7 +1815,11 @@ Bool_t AliAnalysisAlien::CreateJDL() delete arr; } files.ReplaceAll(".root", "*.root"); - outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas); + + if (mgr->IsCollectThroughput()) + outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",files.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas); + else + outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas); } else { TString files = fOutputArchive; files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use @@ -1713,15 +1827,14 @@ Bool_t AliAnalysisAlien::CreateJDL() } arr = outputArchive.Tokenize(" "); TIter next2(arr); - comment = comment1; first = kTRUE; while ((os=(TObjString*)next2())) { - if (!first) comment = NULL; TString currentfile = os->GetString(); if (!currentfile.Contains("@") && fCloseSE.Length()) - fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment); + fMergingJDL->AddToSet("Output", Form("%s@%s",currentfile.Data(), fCloseSE.Data())); else - fMergingJDL->AddToOutputArchive(currentfile, comment); + fMergingJDL->AddToSet("Output", currentfile); + if (first) fMergingJDL->AddToSetDescription("Output", comment); first = kFALSE; } delete arr; @@ -1729,7 +1842,6 @@ Bool_t AliAnalysisAlien::CreateJDL() arr = fOutputFiles.Tokenize(","); TIter next(arr); Bool_t first = kTRUE; - const char *comment = "Files to be saved"; while ((os=(TObjString*)next())) { // Ignore ouputs in jdl that are also in outputarchive TString sout = os->GetString(); @@ -1741,15 +1853,17 @@ Bool_t AliAnalysisAlien::CreateJDL() if (fRegisterExcludes.Contains(sout)) continue; if (!first) comment = NULL; if (!os->GetString().Contains("@") && fCloseSE.Length()) - fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment); + fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data())); else - fGridJDL->AddToOutputSandbox(os->GetString(), comment); - first = kFALSE; + fGridJDL->AddToSet("Output", os->GetString()); + if (first) fGridJDL->AddToSetDescription("Output", comment); if (fMergeExcludes.Contains(sout)) continue; if (!os->GetString().Contains("@") && fCloseSE.Length()) - fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment); + fMergingJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data())); else - fMergingJDL->AddToOutputSandbox(os->GetString(), comment); + fMergingJDL->AddToSet("Output", os->GetString()); + if (first) fMergingJDL->AddToSetDescription("Output", comment); + first = kFALSE; } delete arr; fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job"); @@ -1810,7 +1924,8 @@ Bool_t AliAnalysisAlien::CreateJDL() TObjString *os; TIter next(arr); while ((os=(TObjString*)next())) { - if (os->GetString().Contains(".so")) continue; + if (os->GetString().Contains(".so") || + os->GetString().Contains(".dylib")) continue; Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data()); if (FileExists(os->GetString())) gGrid->Rm(os->GetString()); // TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data())); @@ -1842,7 +1957,7 @@ Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy) // Writes one or more JDL's corresponding to findex. If findex is negative, // all run numbers are considered in one go (jdl). For non-negative indices // they correspond to the indices in the array fInputFiles. - if (!fInputFiles) return kFALSE; + if (!fInputFiles && !fMCLoop) return kFALSE; TObject *os; TString workdir; if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory(); @@ -1875,7 +1990,7 @@ Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory"); else { fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory"); - fMergingJDL->SetOutputDirectory(fGridOutputDir); +// fMergingJDL->SetOutputDirectory(fGridOutputDir); } } else { // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix @@ -1902,7 +2017,7 @@ Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy) } TString sjdl2 = fMergingJDL->Generate(); Int_t index, index1; - sjdl.ReplaceAll("\"LF:", "\n \"LF:"); + sjdl.ReplaceAll("\",\"", "\",\n \""); sjdl.ReplaceAll("(member", "\n (member"); sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_"); sjdl.ReplaceAll("{", "{\n "); @@ -1910,7 +2025,7 @@ Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy) sjdl.ReplaceAll("{\n \n", "{\n"); sjdl.ReplaceAll("\n\n", "\n"); sjdl.ReplaceAll("OutputDirectory", "OutputDir"); - sjdl1.ReplaceAll("\"LF:", "\n \"LF:"); + sjdl1.ReplaceAll("\",\"", "\",\n \""); sjdl1.ReplaceAll("(member", "\n (member"); sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_"); sjdl1.ReplaceAll("{", "{\n "); @@ -1918,7 +2033,7 @@ Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy) sjdl1.ReplaceAll("{\n \n", "{\n"); sjdl1.ReplaceAll("\n\n", "\n"); sjdl1.ReplaceAll("OutputDirectory", "OutputDir"); - sjdl2.ReplaceAll("\"LF:", "\n \"LF:"); + sjdl2.ReplaceAll("\",\"", "\",\n \""); sjdl2.ReplaceAll("(member", "\n (member"); sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_"); sjdl2.ReplaceAll("{", "{\n "); @@ -2088,30 +2203,31 @@ Bool_t AliAnalysisAlien::FileExists(const char *lfn) Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname) { // Returns true if directory exists. Can be also a path. +// Since there is not API in TAlien, we use the Cd trick: if (!gGrid) return kFALSE; - // Check if dirname is a path - TString dirstripped = dirname; - dirstripped = dirstripped.Strip(); - dirstripped = dirstripped.Strip(TString::kTrailing, '/'); - TString dir = gSystem->BaseName(dirstripped); - dir += "/"; - TString path = gSystem->DirName(dirstripped); - TGridResult *res = gGrid->Ls(path, "-F"); - if (!res) return kFALSE; - TIter next(res); - TMap *map; - TObject *obj; - while ((map=dynamic_cast(next()))) { - obj = map->GetValue("name"); - if (!obj) break; - if (dir == obj->GetName()) { - delete res; - return kTRUE; - } + // Backup current path + TString cpath = gGrid->Pwd(); + TString command = "cd "; + TString sdir(dirname); + sdir.ReplaceAll("alien://", ""); + command += sdir; + TGridResult *res = gGrid->Command(command); + if (!res) { + gGrid->Cd(cpath); + return kFALSE; + } + TMap *map = (TMap*)res->At(0); + if (!map) { + gGrid->Cd(cpath); + delete res; + return kFALSE; } + TString sval = map->GetValue("__result__")->GetName(); + Bool_t retval = (Bool_t)sval.Atoi(); + gGrid->Cd(cpath); delete res; - return kFALSE; -} + return retval; +} //______________________________________________________________________________ void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags) @@ -2227,13 +2343,28 @@ TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const Int_t count = 0; // Read the input list of files and add them to the chain TString line; - TChain *chain = new TChain(treeName); - TChain *chainFriend = 0; - if (!fFriendChainName.IsNull()) chainFriend = new TChain(treeName); + TString streeName(treeName); + if (IsUseMCchain()) streeName = "TE"; + TChain *chain = new TChain(streeName); + TList *friends = new TList(); + TChain *cfriend = 0; + if (!fFriendChainName.IsNull()) { + TObjArray *list = fFriendChainName.Tokenize(" "); + TIter next(list); + TObjString *str; + while((str=(TObjString*)next())) { + cfriend = new TChain(streeName, str->GetName()); + friends->Add(cfriend); + chain->AddFriend(cfriend); + } + delete list; + } + TString bpath; + TIter nextfriend(friends); while (in.good()) { in >> line; - if (line.IsNull()) continue; + if (line.IsNull() || line.BeginsWith("#")) continue; if (count++ == fNtestFiles) break; TString esdFile(line); TFile *file = TFile::Open(esdFile); @@ -2241,17 +2372,25 @@ TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const chain->Add(esdFile); file->Close(); if (!fFriendChainName.IsNull()) { - esdFile.ReplaceAll("AliAOD.root", fFriendChainName.Data()); - esdFile.ReplaceAll("AliAODs.root", fFriendChainName.Data()); - file = TFile::Open(esdFile); - if (file && !file->IsZombie()) { - file->Close(); - chainFriend->Add(esdFile); - } else { - Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data()); - return 0; - } - } + if (esdFile.Index("#") > -1) + esdFile.Remove(esdFile.Index("#")); + bpath = gSystem->DirName(esdFile); + bpath += "/"; + TString fileFriend; + nextfriend.Reset(); + while ((cfriend=(TChain*)nextfriend())) { + fileFriend = bpath; + fileFriend += cfriend->GetTitle(); + file = TFile::Open(fileFriend); + if (file && !file->IsZombie()) { + file->Close(); + cfriend->Add(fileFriend); + } else { + Fatal("GetChainForTestMode", "Cannot open friend file: %s", fileFriend.Data()); + return 0; + } + } + } } else { Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data()); } @@ -2260,11 +2399,10 @@ TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const if (!chain->GetListOfFiles()->GetEntries()) { Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data()); delete chain; - delete chainFriend; + friends->Delete(); + delete friends; return NULL; } -// chain->ls(); - if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend); return chain; } @@ -2332,7 +2470,39 @@ Bool_t AliAnalysisAlien::IsSingleOutput() const // Check if single-ouput option is on. return (!fOutputSingle.IsNull()); } - + +//______________________________________________________________________________ +Long64_t AliAnalysisAlien::RunMacroAndExtractLibs(const char* macro, const char *args, TString &libs) +{ +// Tries to run the specified macro and return the libraries that it loads. + TString expname; + if (strlen(macro)) expname = gSystem->ExpandPathName(macro); + if (expname.IsNull() || gSystem->AccessPathName(expname)) { + ::Error("RunMacroAndExtractLibs","Cannot find macro %s in current directory", macro); + return -1; + } + TString oldlibs = gSystem->GetLibraries(); + TMacro m(expname); + Int_t error = 0; + Long64_t retval = m.Exec(args, &error); + if (error != TInterpreter::kNoError) + { + ::Error("RunMacroAndExtractLibs", "Macro interpretation %s failed", macro); + return -1; + } + libs = gSystem->GetLibraries(); + libs.ReplaceAll(oldlibs, ""); + libs.Strip(TString::kLeading); + TObjArray *libTokens = libs.Tokenize(" "); + libs = ""; + for (Int_t i=0; iGetEntries(); i++) { + if (!libs.IsNull()) libs += " "; + libs += gSystem->BaseName(libTokens->At(i)->GetName()); + } + delete libTokens; + return retval; +} + //______________________________________________________________________________ void AliAnalysisAlien::Print(Option_t *) const { @@ -2350,8 +2520,8 @@ void AliAnalysisAlien::Print(Option_t *) const printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n"); if (fProofReset>1) printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n"); - if (!fRootVersionForProof.IsNull()) - printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data()); + if (!fROOTVersion.IsNull()) + printf("= ROOT version requested________________________ %s\n", fROOTVersion.Data()); else printf("= ROOT version requested________________________ default\n"); printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data()); @@ -2383,7 +2553,7 @@ void AliAnalysisAlien::Print(Option_t *) const \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n"); } printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO"); - printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO"); + printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO:Print"); printf("= Production mode:______________________________ %d\n", fProductionMode); printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data()); printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data()); @@ -2392,8 +2562,17 @@ void AliAnalysisAlien::Print(Option_t *) const printf("= User running the plugin: _____________________ %s\n", fUser.Data()); printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data()); printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data()); - printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data()); - printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data()); + TString basedatadir = fGridDataDir; + TString pattern = fDataPattern; + pattern.Strip(); + Int_t ind = pattern.Index(" "); + if (ind>=0) { + basedatadir += "/%run%/"; + basedatadir += pattern(0, ind); + pattern = pattern(ind+1, pattern.Length()); + } + printf("= Data base directory path requested: __________ %s\n", basedatadir.Data()); + printf("= Data search pattern: _________________________ %s\n", pattern.Data()); printf("= Input data format: ___________________________ %s\n", fInputFormat.Data()); if (fRunNumbers.Length()) printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data()); @@ -2474,7 +2653,7 @@ void AliAnalysisAlien::SetDefaults() fMaxMergeFiles = 100; fRunNumbers = ""; fExecutable = "analysis.sh"; - fExecutableCommand = "root -b -q"; + fExecutableCommand = "root -b -q -x"; fArguments = ""; fExecutableArgs = ""; fAnalysisMacro = "myAnalysis.C"; @@ -2503,6 +2682,36 @@ void AliAnalysisAlien::SetDefaults() fOverwriteMode = 1; } +//______________________________________________________________________________ +void AliAnalysisAlien::SetFriendChainName(const char *name, const char *libnames) +{ + // Set file name for the chain of friends and optionally additional libs to be loaded. + // Libs should be separated by blancs. + fFriendChainName = name; + fFriendChainName.ReplaceAll(",", " "); + fFriendChainName.Strip(); + fFriendChainName.ReplaceAll(" ", " "); + + fFriendLibs = libnames; + if (fFriendLibs.Length()) { + if(!fFriendLibs.Contains(".so") && + !fFriendLibs.Contains(".dylib")) + Fatal("SetFriendChainName()", "You should provide explicit library names (with extension)"); + fFriendLibs.ReplaceAll(",", " "); + fFriendLibs.Strip(); + fFriendLibs.ReplaceAll(" ", " "); + } +} + +//______________________________________________________________________________ +void AliAnalysisAlien::SetRootVersionForProof(const char *version) +{ +// Obsolete method. Use SetROOTVersion instead + Warning("SetRootVersionForProof", "Obsolete. Use SetROOTVersion instead"); + if (fROOTVersion.IsNull()) SetROOTVersion(version); + else Error("SetRootVersionForProof", "ROOT version already set to %s", fROOTVersion.Data()); +} + //______________________________________________________________________________ Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl) { @@ -2554,19 +2763,26 @@ Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *alie // Check if this is the last stage to be done. Bool_t laststage = (nfiles=fMaxMergeStages) laststage = kTRUE; + Int_t jobId = 0; if (laststage) { printf("### Submiting final merging stage %d\n", stage); TString finalJDL = jdl; finalJDL.ReplaceAll(".jdl", "_final.jdl"); TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage); - Int_t jobId = SubmitSingleJob(query); - if (!jobId) return kFALSE; + jobId = SubmitSingleJob(query); } else { printf("### Submiting merging stage %d\n", stage); TString query = Form("submit %s %s %d", jdl, aliendir, stage); - Int_t jobId = SubmitSingleJob(query); - if (!jobId) return kFALSE; + jobId = SubmitSingleJob(query); } + if (!jobId) return kFALSE; + + if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" "); + fGridJobIDs.Append(Form("%d", jobId)); + if (!fGridStages.IsNull()) fGridStages.Append(" "); + fGridStages.Append(Form("%s_merge_stage%d", + laststage ? "final" : "partial", stage)); + return kTRUE; } @@ -2608,10 +2824,51 @@ Int_t AliAnalysisAlien::SubmitSingleJob(const char *query) ::Error("SubmitSingleJob", "Your query %s could not be submitted", query); return 0; } - printf(" Job id: %s\n", jobId.Data()); - return atoi(jobId); + Int_t ijobId = jobId.Atoi(); + printf(" Job id: '%s' (%d)\n", jobId.Data(), ijobId); + return ijobId; } +//______________________________________________________________________________ +Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection) +{ +// Merges a collection of output files using concatenation. + TString scoll(collection); + if (!scoll.Contains(".xml")) return kFALSE; + TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection)); + if (!coll) { + ::Error("MergeInfo", "Input XML %s collection empty.", collection); + return kFALSE; + } + // Iterate grid collection + TString outtmp; + Bool_t merged = kFALSE; + Int_t ifile = 0; + while (coll->Next()) { + TString fname = gSystem->DirName(coll->GetTURL()); + fname += "/"; + fname += output; + outtmp = Form("%d_%s", ifile, output); + if (!TFile::Cp(fname, outtmp)) { + ::Error("MergeInfo", "Could not copy %s", fname.Data()); + continue; + } + ifile++; + if (ifile<2) { + gSystem->Exec(Form("cp %s lastmerged", outtmp.Data())); + continue; + } + gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data())); + gSystem->Exec("cp tempmerged lastmerged"); + } + if (ifile) { + gSystem->Exec(Form("cp lastmerged %s", output)); + gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output)); + merged = kTRUE; + } + return merged; +} + //______________________________________________________________________________ Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage) { @@ -2630,6 +2887,7 @@ Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, In Int_t countChunk = 0; Int_t countZero = nmaxmerge; Bool_t merged = kTRUE; + Bool_t isGrid = kTRUE; Int_t index = outputFile.Index("@"); if (index > 0) outputFile.Remove(index); TString inputFile = outputFile; @@ -2649,6 +2907,35 @@ Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, In fname += inputFile; listoffiles->Add(new TNamed(fname.Data(),"")); } + } else if (sbasedir.Contains(".txt")) { + // The file having the .txt extension is expected to contain a list of + // folders where the output files will be looked. For alien folders, + // the full folder LFN is expected (starting with alien://) + // Assume lfn's on each line + TString line; + ifstream in; + in.open(sbasedir); + if (in.fail()) { + ::Error("MergeOutput", "File %s cannot be opened. Merging stopped." ,sbasedir.Data()); + return kTRUE; + } + Int_t nfiles = 0; + while (in.good()) { + in >> line; + if (line.IsNull() || line.BeginsWith("#")) continue; + line.Strip(); + if (!line.Contains("alien:")) isGrid = kFALSE; + line += "/"; + line += outputFile; + nfiles++; + listoffiles->Add(new TNamed(line.Data(),"")); + } + in.close(); + if (!nfiles) { + ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data()); + delete listoffiles; + return kFALSE; + } } else { command = Form("find %s/ *%s", basedir, inputFile.Data()); printf("command: %s\n", command.Data()); @@ -2742,7 +3029,7 @@ Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, In // Loop 'find' results and get next LFN if (countZero == nmaxmerge) { // First file in chunk - create file merger and add previous chunk if any. - fm = new TFileMerger(kTRUE); + fm = new TFileMerger(isGrid); fm->SetFastMethod(kTRUE); if (previousChunk.Length()) fm->AddFile(previousChunk.Data()); outputChunk = outputFile; @@ -2782,7 +3069,7 @@ Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, In } // Merging stage different than 0. // Move to the begining of the requested chunk. - fm = new TFileMerger(kTRUE); + fm = new TFileMerger(isGrid); fm->SetFastMethod(kTRUE); while ((nextfile=next())) fm->AddFile(nextfile->GetName()); delete listoffiles; @@ -2894,11 +3181,13 @@ Bool_t AliAnalysisAlien::MergeOutputs() merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles); if (!merged) { Error("MergeOutputs", "Terminate() will NOT be executed"); - return kFALSE; + delete list; + return kFALSE; } TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile); if (fileOpened) fileOpened->Close(); } + delete list; return kTRUE; } @@ -2952,13 +3241,6 @@ void AliAnalysisAlien::SetOutputArchive(const char *list) fOutputArchive = list; } -//______________________________________________________________________________ -void AliAnalysisAlien::SetPreferedSE(const char */*se*/) -{ -// Setting a prefered output SE is not allowed anymore. - Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()"); -} - //______________________________________________________________________________ void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value) { @@ -3029,9 +3311,9 @@ Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEn } } // Do we need to change the ROOT version ? The success of this cannot be checked. - if (!fRootVersionForProof.IsNull() && !testMode) { - gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");", - fProofCluster.Data(), fRootVersionForProof.Data())); + if (!fROOTVersion.IsNull() && !testMode) { + gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"VO_ALICE@ROOT::%s\");", + fProofCluster.Data(), fROOTVersion.Data())); } // Connect to PROOF and check the status Long_t proof = 0; @@ -3092,13 +3374,19 @@ Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEn TString extraLibs; Bool_t parMode = kFALSE; if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice"; - // Parse the extra libs for .so + // Parse the extra libs for .so or .dylib if (fAdditionalLibs.Length()) { - TObjArray *list = fAdditionalLibs.Tokenize(" "); + TString additionalLibs = fAdditionalLibs; + additionalLibs.Strip(); + if (additionalLibs.Length() && fFriendLibs.Length()) + additionalLibs += " "; + additionalLibs += fFriendLibs; + TObjArray *list = additionalLibs.Tokenize(" "); TIter next(list); TObjString *str; while((str=(TObjString*)next())) { - if (str->GetString().Contains(".so")) { + if (str->GetString().Contains(".so") || + str->GetString().Contains(".dylib") ) { if (parMode) { Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName()); break; @@ -3106,6 +3394,7 @@ Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEn TString stmp = str->GetName(); if (stmp.BeginsWith("lib")) stmp.Remove(0,3); stmp.ReplaceAll(".so",""); + stmp.ReplaceAll(".dylib",""); if (!extraLibs.IsNull()) extraLibs += ":"; extraLibs += stmp; continue; @@ -3120,10 +3409,10 @@ Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEn } if (list) delete list; } - if (!extraLibs.IsNull()) { - Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data()); - optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data())); - } + if (!extraLibs.IsNull()) { + Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data()); + optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data())); + } // Check extra includes if (!fIncludePath.IsNull()) { TString includePath = fIncludePath; @@ -3185,7 +3474,8 @@ Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEn if (list) delete list; } } else { - if (fAdditionalLibs.Contains(".so") && !testMode) { + if ((fAdditionalLibs.Contains(".so") || fAdditionalLibs.Contains(".dylib")) && + !testMode) { Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \ \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()"); return kFALSE; @@ -3254,7 +3544,10 @@ Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEn } // Compose the output archive. fOutputArchive = "log_archive.zip:std*@disk=1 "; - fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas); + if (mgr->IsCollectThroughput()) + fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",fOutputFiles.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas); + else + fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas); } // if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE"); if (TestBit(AliAnalysisGrid::kOffline)) { @@ -3328,9 +3621,9 @@ Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEn } // Check if submitting is managed by LPM manager if (fProductionMode) { - TString prodfile = fJDLName; - prodfile.ReplaceAll(".jdl", ".prod"); - WriteProductionFile(prodfile); + //TString prodfile = fJDLName; + //prodfile.ReplaceAll(".jdl", ".prod"); + //WriteProductionFile(prodfile); Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished."); return kFALSE; } @@ -3338,6 +3631,8 @@ Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEn gGrid->Cd(fGridOutputDir); TGridResult *res; TString jobID = ""; + fGridJobIDs = ""; + fGridStages = ""; if (!fRunNumbers.Length() && !fRunRange[0]) { // Submit a given xml or a set of runs res = gGrid->Command(Form("submit %s", fJDLName.Data())); @@ -3355,6 +3650,12 @@ Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEn \n_______________________________________________________________________", fJDLName.Data(), cjobId); jobID = cjobId; + if (jobID.Atoi()) { + if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" "); + fGridJobIDs.Append(jobID); + if (!fGridStages.IsNull()) fGridStages.Append(" "); + fGridStages.Append("full"); + } } delete res; } else { @@ -3364,12 +3665,19 @@ Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEn } else { // Submit for a range of enumeration of runs. if (!Submit()) return kFALSE; + jobID = fGridJobIDs; } - Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \ - \n You may exit at any time and terminate the job later using the option \ - \n ##################################################################################", jobID.Data()); - gSystem->Exec("aliensh"); + if (fDropToShell) { + Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \ + \n You may exit at any time and terminate the job later using the option \ + \n ##################################################################################", jobID.Data()); + gSystem->Exec("aliensh"); + } else { + Info("StartAnalysis", "\n#### SUBMITTED JOB %s TO ALIEN QUEUE #### \ + \n Remember to terminate the job later using the option \ + \n ##################################################################################", jobID.Data()); + } return kTRUE; } @@ -3393,10 +3701,12 @@ const char *AliAnalysisAlien::GetListOfFiles(const char *type) return files.Data(); } if (mgr->GetOutputEventHandler()) { - aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName(); - TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs(); - if (!extraaod.IsNull()) { - aodfiles += ","; + aodfiles = ""; + if (mgr->GetOutputEventHandler()->GetFillAOD()) + aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName(); + TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs(kTRUE); + if (!extraaod.IsNull() && mgr->GetOutputEventHandler()->GetFillExtension()) { + if (!aodfiles.IsNull()) aodfiles += ","; aodfiles += extraaod; } } @@ -3414,11 +3724,11 @@ const char *AliAnalysisAlien::GetListOfFiles(const char *type) if (!(strcmp(filename, "default"))) continue; if (outputfiles.Contains(filename)) continue; if (aodfiles.Contains(filename)) continue; - if (!outputfiles.IsNull()) outputfiles += ","; + if (!outputfiles.IsNull() && strlen(filename)) outputfiles += ","; outputfiles += filename; } if (stype.Contains("out")) { - if (!files.IsNull()) files += ","; + if (!files.IsNull() && !outputfiles.IsNull()) files += ","; files += outputfiles; if (stype == "out") return files.Data(); } @@ -3440,7 +3750,7 @@ const char *AliAnalysisAlien::GetListOfFiles(const char *type) } delete fextra; if (stype.Contains("ext")) { - if (!files.IsNull()) files += ","; + if (!files.IsNull() && !sextra.IsNull()) files += ","; files += sextra; } } @@ -3537,12 +3847,20 @@ Bool_t AliAnalysisAlien::SubmitMerging() if (!fRunNumbers.Length() && !fRunRange[0]) break; } if (!ntosubmit) return kTRUE; - Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\ - \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob resubmit ERROR_ALL')\ - \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \ - \n output will be written to your alien output directory, while separate stages in . \ - \n ################################################################################################################"); - gSystem->Exec("aliensh"); + if (fDropToShell) { + Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\ + \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob resubmit ERROR_ALL')\ + \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \ + \n output will be written to your alien output directory, while separate stages in . \ + \n ################################################################################################################"); + gSystem->Exec("aliensh"); + } else { + Info("StartAnalysis", "\n #### STARTED MERGING JOBS FOR YOU #### \ + \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob resubmit ERROR_ALL') \ + \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \ + \n output will be written to your alien output directory, while separate stages in . \ + \n ################################################################################################################"); + } return kTRUE; } @@ -3610,6 +3928,10 @@ Bool_t AliAnalysisAlien::SubmitNext() \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \ \n_______________________________________________________________________", fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data()); + if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" "); + fGridJobIDs.Append(cjobId1); + if (!fGridStages.IsNull()) fGridStages.Append(" "); + fGridStages.Append("full"); jobID += cjobId1; jobID += " "; lastmaster = cjobId1.Atoi(); @@ -3701,16 +4023,21 @@ void AliAnalysisAlien::WriteAnalysisMacro() TString func = fAnalysisMacro; TString type = "ESD"; TString comment = "// Analysis using "; - if (IsUseMCchain()) { - type = "MC"; - comment += "MC"; - } else { - if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD"; - if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) { - type = "AOD"; - comment += "AOD"; - } - } + if (fMCLoop) { + type = "MCGEN"; + comment += "MCGEN"; + } else { + if (IsUseMCchain()) { + type = "MC"; + comment += "MC"; + } else { + if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD"; + if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) { + type = "AOD"; + comment += "AOD"; + } + } + } if (type!="AOD" && fFriendChainName!="") { Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD"); return; @@ -3726,6 +4053,10 @@ void AliAnalysisAlien::WriteAnalysisMacro() out << " TStopwatch timer;" << endl; out << " timer.Start();" << endl << endl; // Change temp directory to current one + if (!IsLocalTest()) { + out << "// connect to AliEn and make the chain" << endl; + out << " if (!TGrid::Connect(\"alien://\")) return;" << endl; + } out << "// Set temporary merging directory to current one" << endl; out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl; out << "// Set temporary compilation directory to current one" << endl; @@ -3748,7 +4079,7 @@ void AliAnalysisAlien::WriteAnalysisMacro() TIter next(list); TObjString *str; while((str=(TObjString*)next())) { - if (str->GetString().Contains(".so")) + if (str->GetString().Contains(".so") || str->GetString().Contains(".dylib")) out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl; } if (list) delete list; @@ -3837,18 +4168,33 @@ void AliAnalysisAlien::WriteAnalysisMacro() if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl; out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl; out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl; + if (fMCLoop && !fGeneratorLibs.IsNull()) { + out << "// MC generator libraries" << endl; + TObjArray *list = fGeneratorLibs.Tokenize(" "); + TIter next(list); + TObjString *str; + while((str=(TObjString*)next())) { + out << " gSystem->Load(\"" << str->GetName() << "\");" << endl; + } + delete list; + } if (fAdditionalLibs.Length()) { out << "// Add aditional AliRoot libraries" << endl; - TObjArray *list = fAdditionalLibs.Tokenize(" "); + TString additionalLibs = fAdditionalLibs; + additionalLibs.Strip(); + if (additionalLibs.Length() && fFriendLibs.Length()) + additionalLibs += " "; + additionalLibs += fFriendLibs; + TObjArray *list = additionalLibs.Tokenize(" "); TIter next(list); TObjString *str; while((str=(TObjString*)next())) { - if (str->GetString().Contains(".so")) + if (str->GetString().Contains(".so") || str->GetString().Contains(".dylib")) out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl; if (str->GetString().Contains(".par")) out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl; } - if (list) delete list; + delete list; } out << endl; out << "// analysis source to be compiled at runtime (if any)" << endl; @@ -3875,10 +4221,6 @@ void AliAnalysisAlien::WriteAnalysisMacro() out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl; out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl; } - if (!IsLocalTest()) { - out << "// connect to AliEn and make the chain" << endl; - out << " if (!TGrid::Connect(\"alien://\")) return;" << endl; - } out << "// read the analysis manager from file" << endl; TString analysisFile = fExecutable; analysisFile.ReplaceAll(".sh", ".root"); @@ -3893,6 +4235,12 @@ void AliAnalysisAlien::WriteAnalysisMacro() else out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl; out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl; + if (!fFriendChainName.IsNull()) + out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\",\"" << fFriendLibs << "\");" << endl; + if (IsUseMCchain()) + out << " plugin->SetUseMCchain();" << endl; + if (fMCLoop) + out << " plugin->SetMCLoop(kTRUE);" << endl; out << " mgr->SetGridHandler(plugin);" << endl; if (AliAnalysisManager::GetAnalysisManager()) { out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl; @@ -3914,15 +4262,25 @@ void AliAnalysisAlien::WriteAnalysisMacro() } } if (!IsLocalTest()) { - out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl; - out << " mgr->StartAnalysis(\"localfile\", chain);" << endl; + if (fMCLoop) { + out << " mgr->SetCacheSize(0);" << endl; + out << " mgr->EventLoop(" << fNMCevents << ");" << endl; + } else { + out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl; + out << " mgr->StartAnalysis(\"localfile\", chain);" << endl; + } } else { - out << " mgr->StartAnalysis(\"localfile\");" << endl; + if (fMCLoop) { + out << " mgr->SetCacheSize(0);" << endl; + out << " mgr->EventLoop(" << fNMCevents << ");" << endl; + } else { + out << " mgr->StartAnalysis(\"localfile\");" << endl; + } } out << " timer.Stop();" << endl; out << " timer.Print();" << endl; out << "}" << endl << endl; - if (!IsLocalTest()) { + if (!IsLocalTest() && !fMCLoop) { out <<"//________________________________________________________________________________" << endl; out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl; out << "{" << endl; @@ -3932,9 +4290,13 @@ void AliAnalysisAlien::WriteAnalysisMacro() if (IsUseMCchain()) { out << " TString treename = \"TE\";" << endl; } else { - out << " TString treename = type;" << endl; - out << " treename.ToLower();" << endl; - out << " treename += \"Tree\";" << endl; + if (!fTreeName.IsNull()) { + out << " TString treename = \"" << fTreeName << "\";" << endl; + } else { + out << " TString treename = type;" << endl; + out << " treename.ToLower();" << endl; + out << " treename += \"Tree\";" << endl; + } } out << " printf(\"***************************************\\n\");" << endl; out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl; @@ -3946,8 +4308,20 @@ void AliAnalysisAlien::WriteAnalysisMacro() out << " }" << endl; out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl; out << " TChain *chain = new TChain(treename);" << endl; - if(fFriendChainName!="") { - out << " TChain *chainFriend = new TChain(treename);" << endl; + if(!fFriendChainName.IsNull()) { + out << " TList *friends = new TList();" << endl; + out << " TIter nextfriend(friends);" << endl; + out << " TChain *cfriend = 0;" << endl; + TObjArray *list = fFriendChainName.Tokenize(" "); + TIter next(list); + TObjString *str; + while((str=(TObjString*)next())) { + out << " cfriend = new TChain(treename, \"" << str->GetName() << "\");" << endl; + out << " friends->Add(cfriend);" << endl; + out << " chain->AddFriend(cfriend);" << endl; + } + delete list; +// out << " TChain *chainFriend = new TChain(treename);" << endl; } out << " coll->Reset();" << endl; out << " while (coll->Next()) {" << endl; @@ -3961,20 +4335,31 @@ void AliAnalysisAlien::WriteAnalysisMacro() out << " }" << endl; out << " }" << endl; out << " chain->Add(filename);" << endl; - if(fFriendChainName!="") { - out << " TString fileFriend=coll->GetTURL(\"\");" << endl; - out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<Add(fileFriend.Data());" << endl; + if(!fFriendChainName.IsNull()) { + out << " TString bpath=coll->GetTURL(\"\");" << endl; + out << " if (bpath.Index(\"#\") > -1) bpath.Remove(bpath.Index(\"#\"));" << endl; + out << " bpath = gSystem->DirName(bpath);" << endl; + out << " bpath += \"/\";" << endl; + out << " TString fileFriend;" << endl; + out << " nextfriend.Reset();" << endl; + out << " while ((cfriend=(TChain*)nextfriend())) {" << endl; + out << " fileFriend = bpath;" << endl; + out << " fileFriend += cfriend->GetTitle();" << endl; + out << " TFile *file = TFile::Open(fileFriend);" << endl; + out << " if (file) {" << endl; + out << " file->Close();" << endl; + out << " cfriend->Add(fileFriend.Data());" << endl; + out << " } else {" << endl; + out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl; + out << " return 0;" << endl; + out << " }" << endl; + out << " }" << endl; } out << " }" << endl; out << " if (!chain->GetNtrees()) {" << endl; out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl; out << " return NULL;" << endl; out << " }" << endl; - if(fFriendChainName!="") { - out << " chain->AddFriend(chainFriend);" << endl; - } out << " return chain;" << endl; out << "}" << endl << endl; } @@ -4044,6 +4429,7 @@ void AliAnalysisAlien::WriteMergingMacro() Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?"); return; } + AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager(); TString mergingMacro = fExecutable; mergingMacro.ReplaceAll(".sh","_merge.C"); if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data()); @@ -4087,7 +4473,7 @@ void AliAnalysisAlien::WriteMergingMacro() TIter next(list); TObjString *str; while((str=(TObjString*)next())) { - if (str->GetString().Contains(".so")) + if (str->GetString().Contains(".so") || str->GetString().Contains(".dylib")) out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl; } if (list) delete list; @@ -4177,13 +4563,28 @@ void AliAnalysisAlien::WriteMergingMacro() if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl; out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl; out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl; + if (fMCLoop && !fGeneratorLibs.IsNull()) { + out << "// MC generator libraries" << endl; + TObjArray *list = fGeneratorLibs.Tokenize(" "); + TIter next(list); + TObjString *str; + while((str=(TObjString*)next())) { + out << " gSystem->Load(\"" << str->GetName() << "\");" << endl; + } + delete list; + } if (fAdditionalLibs.Length()) { out << "// Add aditional AliRoot libraries" << endl; - TObjArray *list = fAdditionalLibs.Tokenize(" "); + TString additionalLibs = fAdditionalLibs; + additionalLibs.Strip(); + if (additionalLibs.Length() && fFriendLibs.Length()) + additionalLibs += " "; + additionalLibs += fFriendLibs; + TObjArray *list = additionalLibs.Tokenize(" "); TIter next(list); TObjString *str; while((str=(TObjString*)next())) { - if (str->GetString().Contains(".so")) + if (str->GetString().Contains(".so") || str->GetString().Contains(".dylib")) out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl; } if (list) delete list; @@ -4212,20 +4613,36 @@ void AliAnalysisAlien::WriteMergingMacro() out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl; } // Change temp directory to current one + out << "// Connect to AliEn" << endl; + out << " if (!TGrid::Connect(\"alien://\")) return;" << endl; out << "// Set temporary merging directory to current one" << endl; out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl; out << "// Set temporary compilation directory to current one" << endl; out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl; - out << "// Connect to AliEn" << endl; - out << " if (!TGrid::Connect(\"alien://\")) return;" << endl; out << " TString outputDir = dir;" << endl; - out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl; + if (IsMergeAOD()) + out << " TString outputFiles = \"" << GetListOfFiles("outaod") << "\";" << endl; + else + out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl; out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl; out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl; out << " TIter *iter = new TIter(list);" << endl; out << " TObjString *str;" << endl; out << " TString outputFile;" << endl; out << " Bool_t merged = kTRUE;" << endl; + TString analysisFile = fExecutable; + analysisFile.ReplaceAll(".sh", ".root"); + out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\"" + << analysisFile << "\");" << endl; + out << " if (!mgr) {" << endl; + out << " printf(\"ERROR: Analysis manager could not be extracted from file \");" << endl; + out << " return;" << endl; + out << " }" << endl; + if (IsLocalTest()) { + out << " printf(\"===================================\\n\");" << endl; + out << " printf(\"Testing merging...\\n\");" << endl; + out << " printf(\"===================================\\n\");" << endl; + } out << " while((str=(TObjString*)iter->Next())) {" << endl; out << " outputFile = str->GetString();" << endl; out << " if (outputFile.Contains(\"*\")) continue;" << endl; @@ -4233,7 +4650,7 @@ void AliAnalysisAlien::WriteMergingMacro() out << " if (index > 0) outputFile.Remove(index);" << endl; out << " // Skip already merged outputs" << endl; out << " if (!gSystem->AccessPathName(outputFile)) {" << endl; - out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl; + out << " printf(\"Output file <%s> found. Not merging again.\\n\",outputFile.Data());" << endl; out << " continue;" << endl; out << " }" << endl; out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl; @@ -4243,22 +4660,27 @@ void AliAnalysisAlien::WriteMergingMacro() out << " return;" << endl; out << " }" << endl; out << " }" << endl; + if (mgr && mgr->IsCollectThroughput() && !IsLocalTest()) { + out << " TString infolog = \"" << mgr->GetFileInfoLog() << "\";" << endl; + out << " AliAnalysisAlien::MergeInfo(infolog, outputDir);" << endl; + } out << " // all outputs merged, validate" << endl; out << " ofstream out;" << endl; out << " out.open(\"outputs_valid\", ios::out);" << endl; out << " out.close();" << endl; out << " // read the analysis manager from file" << endl; - TString analysisFile = fExecutable; - analysisFile.ReplaceAll(".sh", ".root"); - out << " if (!outputDir.Contains(\"Stage\")) return;" << endl; - out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\"" - << analysisFile << "\");" << endl; - out << " if (!mgr) return;" << endl; + if (IsLocalTest()) { + out << " printf(\"===================================\\n\");" << endl; + out << " printf(\"Testing Terminate()...\\n\");" << endl; + out << " printf(\"===================================\\n\");" << endl; + } else { + out << " if (!outputDir.Contains(\"Stage\")) return;" << endl; + } out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl; out << " mgr->SetSkipTerminate(kFALSE);" << endl; out << " mgr->PrintStatus();" << endl; - if (AliAnalysisManager::GetAnalysisManager()) { - if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) { + if (mgr) { + if (mgr->GetDebugLevel()>3) { out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl; } else { if (TestBit(AliAnalysisGrid::kTest)) @@ -4372,6 +4794,10 @@ Bool_t AliAnalysisAlien::SetupPar(const char *package) void AliAnalysisAlien::WriteExecutable() { // Generate the alien executable script. + // Patch executable with -x to catch error code + if (fExecutableCommand.Contains("root") && + fExecutableCommand.Contains("-q") && + !fExecutableCommand.Contains("-x")) fExecutableCommand += " -x"; if (!TestBit(AliAnalysisGrid::kSubmit)) { ofstream out; out.open(fExecutable.Data(), ios::out); @@ -4425,13 +4851,10 @@ void AliAnalysisAlien::WriteExecutable() if (copy) { CdWork(); TString workdir = gGrid->GetHomeDirectory(); - TString bindir = Form("%s/bin", workdir.Data()); - if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p"); workdir += fGridWorkingDir; - TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data()); + TString executable = TString::Format("%s/%s", workdir.Data(), fExecutable.Data()); if (FileExists(executable)) gGrid->Rm(executable); Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data()); -// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data())); if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(), executable.Data())) Fatal("","Terminating"); } @@ -4502,15 +4925,12 @@ void AliAnalysisAlien::WriteMergeExecutable() if (copy) { CdWork(); TString workdir = gGrid->GetHomeDirectory(); - TString bindir = Form("%s/bin", workdir.Data()); - if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p"); workdir += fGridWorkingDir; - TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data()); + TString executable = TString::Format("%s/%s", workdir.Data(), mergeExec.Data()); if (FileExists(executable)) gGrid->Rm(executable); Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data()); -// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data())); - if (!copyLocal2Alien("WriteMergeExecutable", - mergeExec.Data(), executable.Data())) Fatal("","Terminating"); + if (!copyLocal2Alien("WriteMergeExecutable",mergeExec.Data(), + executable.Data())) Fatal("","Terminating"); } } @@ -4610,7 +5030,7 @@ void AliAnalysisAlien::WriteValidationScript(Bool_t merge) out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl; out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl; out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl; - out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl; + out << "glibcErr=`grep -Ei '\\*\\*\\* glibc detected \\*\\*\\*' stderr`" << endl; out << "" << endl; out << "if [ \"$parArch\" != \"\" ] ; then" << endl; @@ -4645,7 +5065,7 @@ void AliAnalysisAlien::WriteValidationScript(Bool_t merge) TString outputFiles = fOutputFiles; if (merge && !fTerminateFiles.IsNull()) { - outputFiles += ","; + if (!outputFiles.IsNull()) outputFiles += ","; outputFiles += fTerminateFiles; } TObjArray *arr = outputFiles.Tokenize(",");