#include "TChain.h"
#include "TObjString.h"
#include "TObjArray.h"
+#include "TMacro.h"
#include "TGrid.h"
#include "TGridResult.h"
#include "TGridCollection.h"
#include "TGridJobStatus.h"
#include "TFileMerger.h"
#include "AliAnalysisManager.h"
+#include "AliAnalysisTaskCfg.h"
#include "AliVEventHandler.h"
#include "AliAnalysisDataContainer.h"
#include "AliMultiInputEventHandler.h"
fJDLName(),
fTerminateFiles(),
fMergeExcludes(),
+ fRegisterExcludes(),
fIncludePath(),
fCloseSE(),
fFriendChainName(),
fMergeDirName(),
fInputFiles(0),
fPackages(0),
+ fModules(0),
fProofParam()
{
// Dummy ctor.
fJDLName(),
fTerminateFiles(),
fMergeExcludes(),
+ fRegisterExcludes(),
fIncludePath(),
fCloseSE(),
fFriendChainName(),
fMergeDirName(),
fInputFiles(0),
fPackages(0),
+ fModules(0),
fProofParam()
{
// Default ctor.
fJDLName(other.fJDLName),
fTerminateFiles(other.fTerminateFiles),
fMergeExcludes(other.fMergeExcludes),
+ fRegisterExcludes(other.fRegisterExcludes),
fIncludePath(other.fIncludePath),
fCloseSE(other.fCloseSE),
fFriendChainName(other.fFriendChainName),
fMergeDirName(other.fMergeDirName),
fInputFiles(0),
fPackages(0),
+ fModules(0),
fProofParam()
{
// Copy ctor.
while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
fPackages->SetOwner();
}
+ if (other.fModules) {
+ fModules = new TObjArray();
+ fModules->SetOwner();
+ TIter next(other.fModules);
+ AliAnalysisTaskCfg *mod, *crt;
+ while ((crt=(AliAnalysisTaskCfg*)next())) {
+ mod = new AliAnalysisTaskCfg(*crt);
+ fModules->Add(mod);
+ }
+ }
}
//______________________________________________________________________________
AliAnalysisAlien::~AliAnalysisAlien()
{
// Destructor.
- if (fGridJDL) delete fGridJDL;
- if (fMergingJDL) delete fMergingJDL;
- if (fInputFiles) delete fInputFiles;
- if (fPackages) delete fPackages;
+ delete fGridJDL;
+ delete fMergingJDL;
+ delete fInputFiles;
+ delete fPackages;
+ delete fModules;
fProofParam.DeleteAll();
}
fJDLName = other.fJDLName;
fTerminateFiles = other.fTerminateFiles;
fMergeExcludes = other.fMergeExcludes;
+ fRegisterExcludes = other.fRegisterExcludes;
fIncludePath = other.fIncludePath;
fCloseSE = other.fCloseSE;
fFriendChainName = other.fFriendChainName;
while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
fPackages->SetOwner();
}
+ if (other.fModules) {
+ fModules = new TObjArray();
+ fModules->SetOwner();
+ TIter next(other.fModules);
+ AliAnalysisTaskCfg *mod, *crt;
+ while ((crt=(AliAnalysisTaskCfg*)next())) {
+ mod = new AliAnalysisTaskCfg(*crt);
+ fModules->Add(mod);
+ }
+ }
}
return *this;
}
+//______________________________________________________________________________
+void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
+{
+// Add a single additional library to be loaded. Extension must be present.
+ TString lib(name);
+ if (!lib.Contains(".")) {
+ Error("AddAdditionalLibrary", "Extension not defined for %s", name);
+ return;
+ }
+ if (fAdditionalLibs.Contains(name)) {
+ Warning("AddAdditionalLibrary", "Library %s already added.", name);
+ return;
+ }
+ if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
+ fAdditionalLibs += lib;
+}
+
+//______________________________________________________________________________
+void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
+{
+// Adding a module. Checks if already existing. Becomes owned by this.
+ if (!module) return;
+ if (GetModule(module->GetName())) {
+ Error("AddModule", "A module having the same name %s already added", module->GetName());
+ return;
+ }
+ if (!fModules) {
+ fModules = new TObjArray();
+ fModules->SetOwner();
+ }
+ fModules->Add(module);
+}
+
+//______________________________________________________________________________
+void AliAnalysisAlien::AddModules(TObjArray *list)
+{
+// Adding a list of modules. Checks if already existing. Becomes owned by this.
+ TIter next(list);
+ AliAnalysisTaskCfg *module;
+ while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
+}
+
+//______________________________________________________________________________
+Bool_t AliAnalysisAlien::CheckDependencies()
+{
+// Check if all dependencies are satisfied. Reorder modules if needed.
+ Int_t nmodules = GetNmodules();
+ if (!nmodules) {
+ Warning("CheckDependencies", "No modules added yet to check their dependencies");
+ return kTRUE;
+ }
+ AliAnalysisTaskCfg *mod = 0;
+ AliAnalysisTaskCfg *dep = 0;
+ TString depname;
+ Int_t i, j, k;
+ for (i=0; i<nmodules; i++) {
+ mod = (AliAnalysisTaskCfg*) fModules->At(i);
+ Int_t ndeps = mod->GetNdeps();
+ Int_t istart = i;
+ for (j=0; j<ndeps; j++) {
+ depname = mod->GetDependency(j);
+ dep = GetModule(depname);
+ if (!dep) {
+ Error("CheckDependencies","Dependency %s not added for module %s",
+ depname.Data(), mod->GetName());
+ return kFALSE;
+ }
+ if (dep->NeedsDependency(mod->GetName())) {
+ Error("CheckDependencies","Modules %s and %s circularly depend on each other",
+ mod->GetName(), dep->GetName());
+ return kFALSE;
+ }
+ Int_t idep = fModules->IndexOf(dep);
+ // The dependency task must come first
+ if (idep>i) {
+ // Remove at idep and move all objects below up one slot
+ // down to index i included.
+ fModules->RemoveAt(idep);
+ for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
+ fModules->AddAt(dep, i++);
+ }
+ //Redo from istart if dependencies were inserted
+ if (i>istart) i=istart-1;
+ }
+ }
+ return kTRUE;
+}
+
+//______________________________________________________________________________
+AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
+{
+// Create the analysis manager and optionally execute the macro in filename.
+ AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
+ if (mgr) return mgr;
+ mgr = new AliAnalysisManager(name);
+ mgr->SetGridHandler((AliAnalysisGrid*)this);
+ if (strlen(filename)) {
+ TString line = gSystem->ExpandPathName(filename);
+ line.Prepend(".x ");
+ gROOT->ProcessLine(line.Data());
+ }
+ return mgr;
+}
+
+//______________________________________________________________________________
+Int_t AliAnalysisAlien::GetNmodules() const
+{
+// Get number of modules.
+ if (!fModules) return 0;
+ return fModules->GetEntries();
+}
+
+//______________________________________________________________________________
+AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
+{
+// Get a module by name.
+ if (!fModules) return 0;
+ return (AliAnalysisTaskCfg*)fModules->FindObject(name);
+}
+
+//______________________________________________________________________________
+Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
+{
+// Load a given module.
+ if (mod->IsLoaded()) return kTRUE;
+ AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
+ if (!mgr) {
+ Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
+ return kFALSE;
+ }
+ Int_t ndeps = mod->GetNdeps();
+ TString depname;
+ for (Int_t j=0; j<ndeps; j++) {
+ depname = mod->GetDependency(j);
+ AliAnalysisTaskCfg *dep = GetModule(depname);
+ if (!dep) {
+ Error("LoadModule","Dependency %s not existing for module %s",
+ depname.Data(), mod->GetName());
+ return kFALSE;
+ }
+ if (!LoadModule(dep)) {
+ Error("LoadModule","Dependency %s for module %s could not be loaded",
+ depname.Data(), mod->GetName());
+ return kFALSE;
+ }
+ }
+ // Load libraries for the module
+ if (!mod->CheckLoadLibraries()) {
+ Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
+ return kFALSE;
+ }
+ // Check if a custom file name was requested
+ if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
+
+ // Check if a custom terminate file name was requested
+ if (strlen(mod->GetTerminateFileName())) {
+ if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
+ fTerminateFiles += mod->GetTerminateFileName();
+ }
+
+ // Execute the macro
+ if (mod->ExecuteMacro()<0) {
+ Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
+ mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
+ return kFALSE;
+ }
+ // Configure dependencies
+ if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
+ Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
+ mod->GetConfigMacro()->GetTitle(), mod->GetName());
+ return kFALSE;
+ }
+ // Adjust extra libraries
+ Int_t nlibs = mod->GetNlibs();
+ TString lib;
+ for (Int_t i=0; i<nlibs; i++) {
+ lib = mod->GetLibrary(i);
+ if (fAdditionalLibs.Contains(lib)) continue;
+ lib = Form("lib%s.so", lib.Data());
+ if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
+ fAdditionalLibs += lib;
+ }
+ return kTRUE;
+}
+
+//______________________________________________________________________________
+Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
+{
+// Generate the full train.
+ fAdditionalLibs = "";
+ if (!LoadModules()) return kFALSE;
+ AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
+ if (!mgr->InitAnalysis()) return kFALSE;
+ mgr->RunLocalInit();
+ mgr->PrintStatus();
+ Int_t productionMode = fProductionMode;
+ SetProductionMode();
+ TString macro = fAnalysisMacro;
+ TString executable = fExecutable;
+ TString validation = fValidationScript;
+ TString execCommand = fExecutableCommand;
+ SetAnalysisMacro(Form("%s.C", name));
+ SetExecutable(Form("%s.sh", name));
+// SetExecutableCommand("aliroot -b -q ");
+ SetValidationScript(Form("%s_validation.sh", name));
+ StartAnalysis();
+ SetProductionMode(productionMode);
+ fAnalysisMacro = macro;
+ fExecutable = executable;
+ fExecutableCommand = execCommand;
+ fValidationScript = validation;
+ return kTRUE;
+}
+
+//______________________________________________________________________________
+Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
+{
+// Generate test macros for a single module or for the full train.
+ fAdditionalLibs = "";
+ if (strlen(modname)) {
+ if (!CheckDependencies()) return kFALSE;
+ AliAnalysisTaskCfg *mod = GetModule(modname);
+ if (!mod) {
+ Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
+ return kFALSE;
+ }
+ if (!LoadModule(mod)) return kFALSE;
+ } else if (!LoadModules()) return kFALSE;
+ AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
+ if (!mgr->InitAnalysis()) return kFALSE;
+ mgr->RunLocalInit();
+ mgr->PrintStatus();
+ SetLocalTest(kTRUE);
+ Int_t productionMode = fProductionMode;
+ SetProductionMode();
+ TString macro = fAnalysisMacro;
+ TString executable = fExecutable;
+ TString validation = fValidationScript;
+ TString execCommand = fExecutableCommand;
+ SetAnalysisMacro(Form("%s.C", name));
+ SetExecutable(Form("%s.sh", name));
+// SetExecutableCommand("aliroot -b -q ");
+ SetValidationScript(Form("%s_validation.sh", name));
+ WriteAnalysisFile();
+ WriteAnalysisMacro();
+ WriteExecutable();
+ WriteValidationScript();
+ SetLocalTest(kFALSE);
+ SetProductionMode(productionMode);
+ fAnalysisMacro = macro;
+ fExecutable = executable;
+ fExecutableCommand = execCommand;
+ fValidationScript = validation;
+ return kTRUE;
+}
+
+//______________________________________________________________________________
+Bool_t AliAnalysisAlien::LoadModules()
+{
+// Load all modules by executing the AddTask macros. Checks first the dependencies.
+ fAdditionalLibs = "";
+ Int_t nmodules = GetNmodules();
+ if (!nmodules) {
+ Warning("LoadModules", "No module to be loaded");
+ return kTRUE;
+ }
+ AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
+ if (!mgr) {
+ Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
+ return kFALSE;
+ }
+ if (!CheckDependencies()) return kFALSE;
+ nmodules = GetNmodules();
+ AliAnalysisTaskCfg *mod;
+ for (Int_t imod=0; imod<nmodules; imod++) {
+ mod = (AliAnalysisTaskCfg*)fModules->At(imod);
+ if (!LoadModule(mod)) return kFALSE;
+ }
+ return kTRUE;
+}
+
//______________________________________________________________________________
void AliAnalysisAlien::SetRunPrefix(const char *prefix)
{
{
// Check if file copying is possible.
if (fProductionMode) return kTRUE;
+ TString salienpath(alienpath);
+ if (salienpath.Contains(" ")) {
+ Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
+ return kFALSE;
+ }
if (!Connect()) {
Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
return kFALSE;
Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
return kFALSE;
}
- TFile f("plugin_test_copy", "RECREATE");
+ TString stest = "plugin_test_copy";
+ TFile f(stest, "RECREATE");
// User may not have write permissions to current directory
if (f.IsZombie()) {
Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
return kFALSE;
}
f.Close();
- if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
- if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
+ if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
+ if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
\n# 1. Make sure you have write permissions there. If this is the case: \
\n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
\n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
\n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
\n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
- gSystem->Unlink(f.GetName());
+ gSystem->Unlink(stest.Data());
return kFALSE;
}
- gSystem->Unlink(f.GetName());
- gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
+ gSystem->Unlink(stest.Data());
+ gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
Info("CheckFileCopy", "### ...SUCCESS ###");
return kTRUE;
}
return kTRUE;
}
+//______________________________________________________________________________
+Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
+{
+// Copy data from the given grid directory according a pattern and make a local
+// dataset.
+// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
+ if (!Connect()) {
+ Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
+ return kFALSE;
+ }
+ if (!DirectoryExists(griddir)) {
+ Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
+ return kFALSE;
+ }
+ TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
+ printf("Running command: %s\n", command.Data());
+ TGridResult *res = gGrid->Command(command);
+ Int_t nfound = res->GetEntries();
+ if (!nfound) {
+ Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
+ return kFALSE;
+ }
+ printf("... found %d files. Copying locally ...\n", nfound);
+
+ // archives
+ TObjArray* additionalArchives = 0;
+ if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
+ additionalArchives = TString(archivefile).Tokenize(";");
+ archivefile = additionalArchives->At(0)->GetName();
+ additionalArchives->RemoveAt(0);
+ additionalArchives->Compress();
+ }
+
+ // Copy files locally
+ ofstream out;
+ out.open(output, ios::out);
+ TMap *map;
+ TString turl, dirname, filename, temp;
+ TString cdir = gSystem->WorkingDirectory();
+ gSystem->MakeDirectory(outputdir);
+ gSystem->ChangeDirectory(outputdir);
+ for (Int_t i=0; i<nfound; i++) {
+ map = (TMap*)res->At(i);
+ turl = map->GetValue("turl")->GetName();
+ filename = gSystem->BaseName(turl.Data());
+ dirname = gSystem->DirName(turl.Data());
+ dirname = gSystem->BaseName(dirname.Data());
+ gSystem->MakeDirectory(dirname);
+
+ TString source(turl);
+ TString targetFileName(filename);
+
+ if (strlen(archivefile) > 0) {
+ // TODO here the archive in which the file resides should be determined
+ // however whereis returns only a guid, and guid2lfn does not work
+ // Therefore we use the one provided as argument for now
+ source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
+ targetFileName = archivefile;
+ }
+ if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
+ Bool_t success = kTRUE;
+ if (additionalArchives)
+ for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++)
+ {
+ TString target;
+ target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
+ gSystem->MakeDirectory(gSystem->DirName(target));
+ success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
+ }
+
+ if (success) {
+ if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
+ out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
+ }
+ }
+ }
+ gSystem->ChangeDirectory(cdir);
+ delete res;
+ if (additionalArchives)
+ delete additionalArchives;
+ return kTRUE;
+}
+
//______________________________________________________________________________
Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
{
analysisFile.ReplaceAll(".sh", ".root");
fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
- if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
- fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
if (fAdditionalLibs.Length()) {
arr = fAdditionalLibs.Tokenize(" ");
TIter next(arr);
}
}
if (fOutputArchive.Length()) {
- arr = fOutputArchive.Tokenize(" ");
+ TString outputArchive = fOutputArchive;
+ if (!fRegisterExcludes.IsNull()) {
+ arr = fRegisterExcludes.Tokenize(" ");
+ TIter next1(arr);
+ while ((os=(TObjString*)next1())) {
+ outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
+ outputArchive.ReplaceAll(os->GetString(),"");
+ }
+ delete arr;
+ }
+ arr = outputArchive.Tokenize(" ");
TIter next(arr);
Bool_t first = kTRUE;
const char *comment = "Files to be archived";
}
delete arr;
// Output archive for the merging jdl
- TString outputArchive;
if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
outputArchive = "log_archive.zip:std*@disk=1 ";
// Add normal output files, extra files + terminate files
TString files = GetListOfFiles("outextter");
- // Do not register merge excludes
- if (!fMergeExcludes.IsNull()) {
- arr = fMergeExcludes.Tokenize(" ");
+ // Do not register files in fRegisterExcludes
+ if (!fRegisterExcludes.IsNull()) {
+ arr = fRegisterExcludes.Tokenize(" ");
TIter next1(arr);
while ((os=(TObjString*)next1())) {
files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
sout.ReplaceAll(".root", "");
if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
if (fOutputArchive.Contains(sout)) continue;
+ // Ignore fRegisterExcludes
+ if (fRegisterExcludes.Contains(sout)) continue;
if (!first) comment = NULL;
if (!os->GetString().Contains("@") && fCloseSE.Length())
fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
in.open(fFileForTestMode);
Int_t count = 0;
// Read the input list of files and add them to the chain
- TString line;
- TChain *chain = new TChain(treeName);
- while (in.good())
- {
+ TString line;
+ TChain *chain = new TChain(treeName);
+ TChain *chainFriend = 0;
+ if (!fFriendChainName.IsNull()) chainFriend = new TChain(treeName);
+ while (in.good())
+ {
in >> line;
if (line.IsNull()) continue;
if (count++ == fNtestFiles) break;
TString esdFile(line);
TFile *file = TFile::Open(esdFile);
- if (file) {
- if (!file->IsZombie()) chain->Add(esdFile);
+ if (file && !file->IsZombie()) {
+ chain->Add(esdFile);
file->Close();
+ if (!fFriendChainName.IsNull()) {
+ if (esdFile.Index("#") > -1)
+ esdFile.Remove(esdFile.Index("#"));
+ esdFile = gSystem->DirName(esdFile);
+ esdFile += "/" + fFriendChainName;
+ file = TFile::Open(esdFile);
+ if (file && !file->IsZombie()) {
+ file->Close();
+ chainFriend->Add(esdFile);
+ } else {
+ Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data());
+ return 0;
+ }
+ }
} else {
Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
}
- }
- in.close();
- if (!chain->GetListOfFiles()->GetEntries()) {
+ }
+ in.close();
+ if (!chain->GetListOfFiles()->GetEntries()) {
Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
delete chain;
+ delete chainFriend;
return NULL;
- }
+ }
// chain->ls();
- return chain;
+ if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend);
+ return chain;
}
//______________________________________________________________________________
printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
+ printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
printf("=====================================================================\n");
printf("= Job price: ___________________________________ %d\n", fPrice);
}
// Copy the file in the output directory
printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
- TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
+// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
// Check if this is the last stage to be done.
// Loop 'find' results and get next LFN
if (countZero == nmaxmerge) {
// First file in chunk - create file merger and add previous chunk if any.
- fm = new TFileMerger(kFALSE);
+ fm = new TFileMerger(kTRUE);
fm->SetFastMethod(kTRUE);
if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
outputChunk = outputFile;
}
// Merging stage different than 0.
// Move to the begining of the requested chunk.
- fm = new TFileMerger(kFALSE);
+ fm = new TFileMerger(kTRUE);
fm->SetFastMethod(kTRUE);
while ((nextfile=next())) fm->AddFile(nextfile->GetName());
delete listoffiles;
gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
}
}
- if (fMergeExcludes.Length() &&
- fMergeExcludes.Contains(outputFile.Data())) continue;
+ if (fMergeExcludes.Contains(outputFile.Data()) ||
+ fRegisterExcludes.Contains(outputFile.Data())) continue;
// Perform a 'find' command in the output directory, looking for registered outputs
merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
if (!merged) {
Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
return kFALSE;
}
- WriteAnalysisFile();
+ WriteAnalysisFile();
WriteAnalysisMacro();
WriteExecutable();
WriteValidationScript();
}
// Check if submitting is managed by LPM manager
if (fProductionMode) {
- TString prodfile = fJDLName;
- prodfile.ReplaceAll(".jdl", ".prod");
- WriteProductionFile(prodfile);
+ //TString prodfile = fJDLName;
+ //prodfile.ReplaceAll(".jdl", ".prod");
+ //WriteProductionFile(prodfile);
Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
return kFALSE;
}
outputFile = str->GetString();
Int_t index = outputFile.Index("@");
if (index > 0) outputFile.Remove(index);
- if (!fMergeExcludes.Contains(outputFile)) break;
+ if (!fMergeExcludes.Contains(outputFile) &&
+ !fRegisterExcludes.Contains(outputFile)) break;
}
delete list;
Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
workdir += fGridWorkingDir;
Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
- TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
}
out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
+ }
+ if (!IsLocalTest()) {
+ out << "// connect to AliEn and make the chain" << endl;
+ out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
}
- out << "// connect to AliEn and make the chain" << endl;
- out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
out << "// read the analysis manager from file" << endl;
TString analysisFile = fExecutable;
analysisFile.ReplaceAll(".sh", ".root");
out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
<< analysisFile << "\");" << endl;
out << " if (!mgr) return;" << endl;
+ if (IsLocalTest()) {
+ out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
+ out << " plugin->SetRunMode(\"test\");" << endl;
+ if (fFileForTestMode.IsNull())
+ out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
+ else
+ out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
+ out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
+ if (!fFriendChainName.IsNull())
+ out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\");" << endl;
+ out << " mgr->SetGridHandler(plugin);" << endl;
+ if (AliAnalysisManager::GetAnalysisManager()) {
+ out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
+ out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
+ } else {
+ out << " mgr->SetDebugLevel(10);" << endl;
+ out << " mgr->SetNSysInfo(100);" << endl;
+ }
+ }
out << " mgr->PrintStatus();" << endl;
if (AliAnalysisManager::GetAnalysisManager()) {
if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
}
}
- if (IsUsingTags()) {
- out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
- } else {
+ if (!IsLocalTest()) {
out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
+ out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
+ } else {
+ out << " mgr->StartAnalysis(\"localfile\");" << endl;
}
- out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
out << " timer.Stop();" << endl;
out << " timer.Print();" << endl;
out << "}" << endl << endl;
- if (IsUsingTags()) {
- out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
- out << "{" << endl;
- out << "// Create a chain using tags from the xml file." << endl;
- out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
- out << " if (!coll) {" << endl;
- out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
- out << " return NULL;" << endl;
- out << " }" << endl;
- out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
- out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
- out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
- out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
- out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
- out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
- out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
- out << " // Check if the cuts configuration file was provided" << endl;
- out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
- out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
- out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
- out << " }" << endl;
- if (fFriendChainName=="") {
- out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
- } else {
- out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
- out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
- out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
- }
- out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
- out << " chain->ls();" << endl;
- out << " return chain;" << endl;
- out << "}" << endl << endl;
- if (gSystem->AccessPathName("ConfigureCuts.C")) {
- TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
- msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
- msg += " AliLHCTagCuts *lhcCuts,\n";
- msg += " AliDetectorTagCuts *detCuts,\n";
- msg += " AliEventTagCuts *evCuts)";
- Info("WriteAnalysisMacro", "%s", msg.Data());
- }
- }
- if (!IsUsingTags() || fFriendChainName!="") {
+ if (!IsLocalTest()) {
out <<"//________________________________________________________________________________" << endl;
out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
out << "{" << endl;
out << " chain->Add(filename);" << endl;
if(fFriendChainName!="") {
out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
- out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
- out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
- out << " chainFriend->Add(fileFriend.Data());" << endl;
+ out << " if (fileFriend.Index(\"#\") > -1) fileFriend.Remove(fileFriend.Index(\"#\"));" << endl;
+ out << " fileFriend = gSystem->DirName(fileFriend);" << endl;
+ out << " fileFriend += \"/\";" << endl;
+ out << " fileFriend += \"" << fFriendChainName << "\";";
+ out << " TFile *file = TFile::Open(fileFriend);" << endl;
+ out << " if (file) {" << endl;
+ out << " file->Close();" << endl;
+ out << " chainFriend->Add(fileFriend.Data());" << endl;
+ out << " } else {" << endl;
+ out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
+ out << " return 0;" << endl;
+ out << " }" << endl;
}
out << " }" << endl;
out << " if (!chain->GetNtrees()) {" << endl;
TString workdir = gGrid->GetHomeDirectory();
workdir += fGridWorkingDir;
if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
- if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
- if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
- Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
- if (!copyLocal2Alien("WriteAnalysisMacro","ConfigureCuts.C",
- Form("%s/ConfigureCuts.C", workdir.Data()))) Fatal("","Terminating");
-// TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
- }
Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
}
TString mergingMacro = fExecutable;
mergingMacro.ReplaceAll(".sh","_merge.C");
- if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
+ if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
if (!TestBit(AliAnalysisGrid::kSubmit)) {
ofstream out;
out.open(mergingMacro.Data(), ios::out);
out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
out << " TString outputDir = dir;" << endl;
out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
- out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
+ out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
out << " TIter *iter = new TIter(list);" << endl;
out << " TObjString *str;" << endl;
out << "free -m" << endl;
out << "echo \"=========================================\"" << endl << endl;
out << fExecutableCommand << " ";
- out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
- out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
+ out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
+ out << "RET=$?" << endl;
+ out << "if [ \"$RET\" != \"0\" ];then" << endl;
+ out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
+ out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
+ out << " let sig=\"$RET - 128\""<<endl;
+ out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
+ out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
+ out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
+ out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
+ out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
+ out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
+ out << " fi"<<endl;
+ out << " exit $RET"<< endl;
+ out << "fi" << endl << endl ;
+ out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
out << "echo \"############## memory after: ##############\"" << endl;
out << "free -m" << endl;
}
else
out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
out << fExecutableCommand << " " << "$ARG" << endl;
+ out << "RET=$?" << endl;
+ out << "if [ \"$RET\" != \"0\" ];then" << endl;
+ out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
+ out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
+ out << " let sig=\"$RET - 128\""<<endl;
+ out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
+ out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
+ out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
+ out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
+ out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
+ out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
+ out << " fi"<<endl;
+ out << " exit $RET"<< endl;
+ out << "fi" << endl << endl ;
out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
out << "echo \"############## memory after: ##############\"" << endl;
out << "free -m" << endl;
git://git.uio.no / u / mrichter / AliRoot.git / blobdiff