#include "Riostream.h"
#include "TEnv.h"
+#include "TBits.h"
#include "TError.h"
#include "TROOT.h"
#include "TSystem.h"
}
// Work directory not existing - create it
gGrid->Cd(homedir);
- if (gGrid->Mkdir(workdir)) {
+ if (gGrid->Mkdir(workdir, "-p")) {
gGrid->Cd(fGridWorkingDir);
Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
} else {
return kTRUE;
}
// Process declared files
- Bool_t is_collection = kFALSE;
- Bool_t is_xml = kFALSE;
- Bool_t use_tags = kFALSE;
+ Bool_t isCollection = kFALSE;
+ Bool_t isXml = kFALSE;
+ Bool_t useTags = kFALSE;
Bool_t checked = kFALSE;
CdWork();
TString file;
CheckDataType(file, iscoll, isxml, usetags);
if (!checked) {
checked = kTRUE;
- is_collection = iscoll;
- is_xml = isxml;
- use_tags = usetags;
- TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
+ isCollection = iscoll;
+ isXml = isxml;
+ useTags = usetags;
+ TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
} else {
- if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
+ if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
Error("CheckInputData", "Some conflict was found in the types of inputs");
return kFALSE;
}
Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
return kFALSE;
}
- if (is_collection) {
+ if (isCollection) {
Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
return kFALSE;
}
- if (checked && !is_xml) {
+ if (checked && !isXml) {
Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
return kFALSE;
}
TString path;
if (!checked) {
checked = kTRUE;
- use_tags = fDataPattern.Contains("tag");
- TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
+ useTags = fDataPattern.Contains("tag");
+ TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
}
- if (use_tags != fDataPattern.Contains("tag")) {
+ if (useTags != fDataPattern.Contains("tag")) {
Error("CheckInputData", "Cannot mix input files using/not using tags");
return kFALSE;
}
TString msg = "\n##### file: ";
msg += path;
msg += " type: xml_collection;";
- if (use_tags) msg += " using_tags: Yes";
+ if (useTags) msg += " using_tags: Yes";
else msg += " using_tags: No";
Info("CheckDataType", msg.Data());
if (fNrunsPerMaster<2) {
TString msg = "\n##### file: ";
msg += path;
msg += " type: xml_collection;";
- if (use_tags) msg += " using_tags: Yes";
+ if (useTags) msg += " using_tags: Yes";
else msg += " using_tags: No";
Info("CheckDataType", msg.Data());
if (fNrunsPerMaster<2) {
if (res) delete res;
// Write standard output to file
gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
- Bool_t has_grep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
- Bool_t null_file = kFALSE;
- if (!has_grep) {
+ Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
+ Bool_t nullFile = kFALSE;
+ if (!hasGrep) {
Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
} else {
- null_file = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
- if (null_file) {
+ nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
+ if (nullFile) {
Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
return kFALSE;
}
return kTRUE;
}
// Several runs
- Bool_t null_result = kTRUE;
+ Bool_t nullResult = kTRUE;
if (fRunNumbers.Length()) {
TObjArray *arr = fRunNumbers.Tokenize(" ");
TObjString *os;
if (res) delete res;
// Write standard output to file
gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
- Bool_t has_grep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
- Bool_t null_file = kFALSE;
- if (!has_grep) {
+ Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
+ Bool_t nullFile = kFALSE;
+ if (!hasGrep) {
Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
} else {
- null_file = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
- if (null_file) {
+ nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
+ if (nullFile) {
Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
fRunNumbers.ReplaceAll(os->GetString().Data(), "");
continue;
}
}
- null_result = kFALSE;
+ nullResult = kFALSE;
}
if (TestBit(AliAnalysisGrid::kTest)) break;
// Check if there is one run per master job.
}
}
delete arr;
- if (null_result) {
+ if (nullResult) {
Error("CreateDataset", "No valid dataset corresponding to the query!");
return kFALSE;
}
if (res) delete res;
// Write standard output to file
gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
- Bool_t has_grep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
- Bool_t null_file = kFALSE;
- if (!has_grep) {
+ Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
+ Bool_t nullFile = kFALSE;
+ if (!hasGrep) {
Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
} else {
- null_file = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
- if (null_file) {
+ nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
+ if (nullFile) {
Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
continue;
}
}
- null_result = kFALSE;
+ nullResult = kFALSE;
}
if (TestBit(AliAnalysisGrid::kTest)) break;
// Check if there is one run per master job.
}
}
}
- if (null_result) {
+ if (nullResult) {
Error("CreateDataset", "No valid dataset corresponding to the query!");
return kFALSE;
}
} else {
if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
if (!DirectoryExists(fGridOutputDir)) {
- if (gGrid->Mkdir(fGridOutputDir)) {
+ if (gGrid->Mkdir(fGridOutputDir,"-p")) {
Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
} else {
Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
if (!fArguments.IsNull())
fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
- fMergingJDL->SetArguments("$1");
+ fMergingJDL->SetArguments("$1 $2 $3");
fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
first = kTRUE;
while ((os=(TObjString*)next2())) {
if (!first) comment = NULL;
- if (!os->GetString().Contains("@") && fCloseSE.Length())
- fMergingJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
+ TString currentfile = os->GetString();
+ currentfile.ReplaceAll(".root", "*.root");
+ currentfile.ReplaceAll(".zip", "-Stage$2_$3.zip");
+ if (!currentfile.Contains("@") && fCloseSE.Length())
+ fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
else
- fMergingJDL->AddToOutputArchive(os->GetString(), comment);
+ fMergingJDL->AddToOutputArchive(currentfile, comment);
first = kFALSE;
}
delete arr;
} else {
if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
- if (gGrid->Mkdir(fGridOutputDir)) {
+ if (gGrid->Mkdir(fGridOutputDir,"-p")) {
Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
} else {
Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
} else {
fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
- fMergingJDL->SetOutputDirectory(Form("%s/$1", fGridOutputDir.Data()), "Output directory");
+ fMergingJDL->SetOutputDirectory(Form("$1", fGridOutputDir.Data()), "Output directory");
}
}
sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
index = sjdl.Index("JDLVariables");
if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
+ sjdl += "Workdirectorysize = {\"5000MB\"};";
sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
- sjdl1.Prepend(Form("Jobtag = {\n \"comment:Merging_%s\"\n};\n", fJobTag.Data()));
+ sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", fJobTag.Data()));
+ sjdl1.Prepend("# Generated merging jdl\n# $1 = full alien path to output directory to be merged\n# $2 = merging stage\n# $3 = merged chunk\n");
index = sjdl1.Index("JDLVariables");
if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
+ sjdl1 += "Workdirectorysize = {\"5000MB\"};";
// Write jdl to file
ofstream out;
out.open(fJDLName.Data(), ios::out);
}
//______________________________________________________________________________
-void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
+void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
{
// Check input data type.
- is_collection = kFALSE;
- is_xml = kFALSE;
- use_tags = kFALSE;
+ isCollection = kFALSE;
+ isXml = kFALSE;
+ useTags = kFALSE;
if (!gGrid) {
Error("CheckDataType", "No connection to grid");
return;
}
- is_collection = IsCollection(lfn);
+ isCollection = IsCollection(lfn);
TString msg = "\n##### file: ";
msg += lfn;
- if (is_collection) {
+ if (isCollection) {
msg += " type: raw_collection;";
// special treatment for collections
- is_xml = kFALSE;
+ isXml = kFALSE;
// check for tag files in the collection
TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
if (!res) {
return;
}
TString file = typeStr;
- use_tags = file.Contains(".tag");
- if (use_tags) msg += " using_tags: Yes";
+ useTags = file.Contains(".tag");
+ if (useTags) msg += " using_tags: Yes";
else msg += " using_tags: No";
Info("CheckDataType", msg.Data());
return;
}
TString slfn(lfn);
slfn.ToLower();
- is_xml = slfn.Contains(".xml");
- if (is_xml) {
+ isXml = slfn.Contains(".xml");
+ if (isXml) {
// Open xml collection and check if there are tag files inside
msg += " type: xml_collection;";
TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
map = (TMap*)map->GetValue("");
TString file;
if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
- use_tags = file.Contains(".tag");
+ useTags = file.Contains(".tag");
delete coll;
- if (use_tags) msg += " using_tags: Yes";
+ if (useTags) msg += " using_tags: Yes";
else msg += " using_tags: No";
Info("CheckDataType", msg.Data());
return;
}
- use_tags = slfn.Contains(".tag");
+ useTags = slfn.Contains(".tag");
if (slfn.Contains(".root")) msg += " type: root file;";
else msg += " type: unknown file;";
- if (use_tags) msg += " using_tags: Yes";
+ if (useTags) msg += " using_tags: Yes";
else msg += " using_tags: No";
Info("CheckDataType", msg.Data());
}
}
//______________________________________________________________________________
-Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge)
+Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, Bool_t submit, const char *jdl)
+{
+// Static method that checks the status of merging. This can submit merging jobs that did not produced the expected
+// output. If <submit> is false (checking) returns true only when the final merged file was found. If submit is true returns
+// true if the jobs were successfully submitted.
+ Int_t countOrig = 0;
+ Int_t countStage = 0;
+ Int_t stage = 0;
+ Int_t i;
+ Bool_t doneFinal = kFALSE;
+ TBits chunksDone;
+ TString saliendir(aliendir);
+ TString sfilename, stmp;
+ saliendir.ReplaceAll("//","/");
+ saliendir = saliendir.Strip(TString::kTrailing, '/');
+ if (!gGrid) {
+ ::Error("GetNregisteredFiles", "You need to be connected to AliEn.");
+ return kFALSE;
+ }
+ sfilename = filename;
+ sfilename.ReplaceAll(".root", "*.root");
+ printf("Checking directory <%s> for merged files <%s> ...\n", aliendir, sfilename.Data());
+ TString command = Form("find %s/ *%s", saliendir.Data(), sfilename.Data());
+ TGridResult *res = gGrid->Command(command);
+ if (!res) {
+ ::Error("GetNregisteredFiles","Error: No result for the find command\n");
+ return kFALSE;
+ }
+ TIter nextmap(res);
+ TMap *map = 0;
+ while ((map=(TMap*)nextmap())) {
+ TString turl = map->GetValue("turl")->GetName();
+ if (!turl.Length()) {
+ // Nothing found
+ delete res;
+ return kFALSE;
+ }
+ turl.ReplaceAll("alien://", "");
+ turl.ReplaceAll(saliendir, "");
+ sfilename = gSystem->BaseName(turl);
+ turl = turl.Strip(TString::kLeading, '/');
+ // Now check to what the file corresponds to:
+ // original output - aliendir/%03d/filename
+ // merged file (which stage) - aliendir/filename-Stage%02d_%04d
+ // final merged file - aliendir/filename
+ if (sfilename == turl) {
+ if (sfilename == filename) {
+ doneFinal = kTRUE;
+ } else {
+ // check stage
+ Int_t index = sfilename.Index("Stage");
+ if (index<0) continue;
+ stmp = sfilename(index+5,2);
+ Int_t istage = atoi(stmp);
+ stmp = sfilename(index+8,4);
+ Int_t ijob = atoi(stmp);
+ if (istage<stage) continue; // Ignore lower stages
+ if (istage>stage) {
+ countStage = 0;
+ chunksDone.ResetAllBits();
+ stage = istage;
+ }
+ countStage++;
+ chunksDone.SetBitNumber(ijob);
+ }
+ } else {
+ countOrig++;
+ }
+ if (doneFinal) {
+ delete res;
+ printf("=> Removing files from previous stages...\n");
+ gGrid->Rm(Form("%s/*Stage*.root", aliendir));
+ return kTRUE;
+ }
+ }
+ delete res;
+ // Compute number of jobs that were submitted for the current stage
+ Int_t ntotstage = countOrig;
+ for (i=1; i<=stage; i++) {
+ if (ntotstage%nperchunk) ntotstage = (ntotstage/nperchunk)+1;
+ else ntotstage = (ntotstage/nperchunk);
+ }
+ // Now compare with the number of set bits in the chunksDone array
+ Int_t nmissing = (stage>0)?(ntotstage - countStage):0;
+ // Print the info
+ printf("*** Found %d original files\n", countOrig);
+ if (stage==0) printf("*** No merging completed so far.\n");
+ else printf("*** Found %d out of %d files merged for stage %d\n", countStage, ntotstage, stage);
+ if (nmissing) printf("*** Number of merged files missing for this stage: %d -> check merging job completion\n", nmissing);
+ if (!submit) return doneFinal;
+ // Sumbit merging jobs for all missing chunks for the current stage.
+ TString query = Form("submit %s %s", jdl, aliendir);
+ Int_t ichunk = -1;
+ if (nmissing) {
+ for (i=0; i<nmissing; i++) {
+ ichunk = chunksDone.FirstNullBit(ichunk+1);
+ Int_t jobId = SubmitSingleJob(Form("%s %d %d", query.Data(), stage, ichunk));
+ if (!jobId) return kFALSE;
+ }
+ return kTRUE;
+ }
+ // Submit next stage of merging
+ if (stage==0) countStage = countOrig;
+ Int_t nchunks = (countStage/nperchunk);
+ if (countStage%nperchunk) nchunks += 1;
+ for (i=0; i<nchunks; i++) {
+ Int_t jobId = SubmitSingleJob(Form("%s %d %d", query.Data(), stage+1, i));
+ if (!jobId) return kFALSE;
+ }
+ return kTRUE;
+}
+
+//______________________________________________________________________________
+Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
+{
+// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
+ if (!gGrid) return 0;
+ printf("=> %s ------> ",query);
+ TGridResult *res = gGrid->Command(query);
+ if (!res) return 0;
+ TString jobId = res->GetKey(0,"jobId");
+ delete res;
+ if (jobId.IsNull()) {
+ printf("submission failed. Reason:\n");
+ gGrid->Stdout();
+ gGrid->Stderr();
+ ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
+ return 0;
+ }
+ printf(" Job id: %s\n", jobId.Data());
+ return atoi(jobId);
+}
+
+//______________________________________________________________________________
+Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage, Int_t ichunk)
{
-// Merge all registered outputs from basedir.
- TString output_file = output;
+// Merge given output files from basedir. The file merger will merge nmaxmerge
+// files in a group. Merging can be done in stages:
+// stage=0 : will merge all existing files in a single stage
+// stage=1 : does a find command for all files that do NOT contain the string "Stage".
+// If their number is bigger that nmaxmerge, only the files from
+// ichunk*nmaxmerge to ichunk*(nmaxmerge+1)-1 will get merged as output_stage_<ichunk>
+// stage=n : does a find command for files named <output>Stage<stage-1>_*. If their number is bigger than
+// nmaxmerge, merge just the chunk ichunk, otherwise write the merged output to the file
+// named <output>.
+ TString outputFile = output;
TString command;
- TString output_chunk;
- TString previous_chunk = "";
- Int_t count_chunk = 0;
- Int_t count_zero = nmaxmerge;
+ TString outputChunk;
+ TString previousChunk = "";
+ Int_t countChunk = 0;
+ Int_t countZero = nmaxmerge;
Bool_t merged = kTRUE;
- Int_t index = output_file.Index("@");
- if (index > 0) output_file.Remove(index);
- command = Form("find %s/ *%s", basedir, output_file.Data());
+ Int_t index = outputFile.Index("@");
+ if (index > 0) outputFile.Remove(index);
+ TString inputFile = outputFile;
+ if (stage>1) inputFile.ReplaceAll(".root", Form("-Stage%02d_*.root", stage-1));
+ command = Form("find %s/ *%s", basedir, inputFile.Data());
printf("command: %s\n", command.Data());
TGridResult *res = gGrid->Command(command);
if (!res) {
- printf("Error: No result for the find command\n");
+ ::Error("MergeOutput","No result for the find command\n");
return kFALSE;
}
TFileMerger *fm = 0;
TIter nextmap(res);
TMap *map = 0;
- // Check if there is a merge operation to resume
- output_chunk = output_file;
- output_chunk.ReplaceAll(".root", "_*.root");
+ // Check if there is a merge operation to resume. Works only for stage 0 or 1.
+ outputChunk = outputFile;
+ outputChunk.ReplaceAll(".root", "_*.root");
// Check for existent temporary merge files
// Check overwrite mode and remove previous partial results if needed
- if (!gSystem->Exec(Form("ls %s 2>/dev/null", output_chunk.Data()))) {
- while (1) {
- // Skip as many input files as in a chunk
- for (Int_t counter=0; counter<nmaxmerge; counter++) map = (TMap*)nextmap();
- if (!map) {
- ::Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
+ // Preserve old merging functionality for stage 0.
+ if (stage==0) {
+ if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
+ while (1) {
+ // Skip as many input files as in a chunk
+ for (Int_t counter=0; counter<nmaxmerge; counter++) map = (TMap*)nextmap();
+ if (!map) {
+ ::Error("MergeOutput", "Cannot resume merging for <%s>, nentries=%d", outputFile.Data(), res->GetSize());
+ delete res;
+ return kFALSE;
+ }
+ outputChunk = outputFile;
+ outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
+ countChunk++;
+ if (gSystem->AccessPathName(outputChunk)) continue;
+ // Merged file with chunks up to <countChunk> found
+ ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
+ previousChunk = outputChunk;
+ break;
+ }
+ }
+ countZero = nmaxmerge;
+
+ while ((map=(TMap*)nextmap())) {
+ // Loop 'find' results and get next LFN
+ if (countZero == nmaxmerge) {
+ // First file in chunk - create file merger and add previous chunk if any.
+ fm = new TFileMerger(kFALSE);
+ fm->SetFastMethod(kTRUE);
+ if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
+ outputChunk = outputFile;
+ outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
+ }
+ // If last file found, put merged results in the output file
+ if (map == res->Last()) outputChunk = outputFile;
+ TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
+ if (!objs || !objs->GetString().Length()) {
+ // Nothing found - skip this output
delete res;
+ delete fm;
return kFALSE;
+ }
+ // Add file to be merged and decrement chunk counter.
+ fm->AddFile(objs->GetString());
+ countZero--;
+ if (countZero==0 || map == res->Last()) {
+ if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
+ // Nothing found - skip this output
+ ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
+ delete res;
+ delete fm;
+ return kFALSE;
+ }
+ fm->OutputFile(outputChunk);
+ // Merge the outputs, then go to next chunk
+ if (!fm->Merge()) {
+ ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
+ delete res;
+ delete fm;
+ return kFALSE;
+ } else {
+ ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
+ gSystem->Unlink(previousChunk);
+ }
+ if (map == res->Last()) {
+ delete res;
+ delete fm;
+ break;
+ }
+ countChunk++;
+ countZero = nmaxmerge;
+ previousChunk = outputChunk;
}
- output_chunk = output_file;
- output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
- count_chunk++;
- if (gSystem->AccessPathName(output_chunk)) continue;
- // Merged file with chunks up to <count_chunk> found
- printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
- previous_chunk = output_chunk;
- break;
}
- }
- count_zero = nmaxmerge;
-
+ return merged;
+ }
+ // Merging stage different than 0.
+ // Move to the begining of the requested chunk.
+ outputChunk = outputFile;
+ if (nmaxmerge < res->GetSize()) {
+ if (ichunk*nmaxmerge >= res->GetSize()) {
+ ::Error("MergeOutput", "Cannot merge merge chunk %d grouping %d files from %d total.", ichunk, nmaxmerge, res->GetSize());
+ delete res;
+ return kFALSE;
+ }
+ for (Int_t counter=0; counter<ichunk*nmaxmerge; counter++) map = (TMap*)nextmap();
+ outputChunk.ReplaceAll(".root", Form("-Stage%02d_%04d.root", stage, ichunk));
+ }
+ countZero = nmaxmerge;
+ fm = new TFileMerger(kFALSE);
+ fm->SetFastMethod(kTRUE);
while ((map=(TMap*)nextmap())) {
- // Loop 'find' results and get next LFN
- if (count_zero == nmaxmerge) {
- // First file in chunk - create file merger and add previous chunk if any.
- fm = new TFileMerger(kFALSE);
- fm->SetFastMethod(kTRUE);
- if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
- output_chunk = output_file;
- output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
- }
- // If last file found, put merged results in the output file
- if (map == res->Last()) output_chunk = output_file;
+ // Loop 'find' results and get next LFN
TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
if (!objs || !objs->GetString().Length()) {
// Nothing found - skip this output
}
// Add file to be merged and decrement chunk counter.
fm->AddFile(objs->GetString());
- count_zero--;
- if (count_zero==0 || map == res->Last()) {
- fm->OutputFile(output_chunk);
- if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
- // Nothing found - skip this output
- ::Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
- delete res;
- delete fm;
- return kFALSE;
- }
- // Merge the outputs, then go to next chunk
- if (!fm->Merge()) {
- ::Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
- delete res;
- delete fm;
- merged = kFALSE;
- return kFALSE;
- } else {
- ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
- gSystem->Unlink(previous_chunk);
- }
- if (map == res->Last()) {
- delete res;
- delete fm;
- break;
- }
- count_chunk++;
- count_zero = nmaxmerge;
- previous_chunk = output_chunk;
- }
+ countZero--;
+ if (countZero==0) break;
+ }
+ delete res;
+ if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
+ // Nothing found - skip this output
+ ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
+ delete fm;
+ return kFALSE;
}
- return merged;
+ fm->OutputFile(outputChunk);
+ // Merge the outputs
+ if (!fm->Merge()) {
+ ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
+ delete fm;
+ return kFALSE;
+ } else {
+ ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
+ }
+ delete fm;
+ return kTRUE;
}
//______________________________________________________________________________
gEnv->SetValue("XNet.ReconnectTimeout",10);
gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
}
+ // Make sure we change the temporary directory
+ gSystem->Setenv("TMPDIR", gSystem->pwd());
TObjArray *list = fOutputFiles.Tokenize(",");
TIter next(list);
TObjString *str;
- TString output_file;
+ TString outputFile;
Bool_t merged = kTRUE;
while((str=(TObjString*)next())) {
- output_file = str->GetString();
- Int_t index = output_file.Index("@");
- if (index > 0) output_file.Remove(index);
- TString output_chunk = output_file;
- output_chunk.ReplaceAll(".root", "_*.root");
+ outputFile = str->GetString();
+ Int_t index = outputFile.Index("@");
+ if (index > 0) outputFile.Remove(index);
+ TString outputChunk = outputFile;
+ outputChunk.ReplaceAll(".root", "_*.root");
// Skip already merged outputs
- if (!gSystem->AccessPathName(output_file)) {
+ if (!gSystem->AccessPathName(outputFile)) {
if (fOverwriteMode) {
- Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", output_file.Data());
- gSystem->Unlink(output_file);
- if (!gSystem->Exec(Form("ls %s 2>/dev/null", output_chunk.Data()))) {
+ Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
+ gSystem->Unlink(outputFile);
+ if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
- output_chunk.Data());
- gSystem->Exec(Form("rm -f %s", output_chunk.Data()));
+ outputChunk.Data());
+ gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
}
} else {
- Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
+ Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
continue;
}
} else {
- if (!gSystem->Exec(Form("ls %s 2>/dev/null", output_chunk.Data()))) {
+ if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
- output_chunk.Data());
- gSystem->Exec(Form("rm -f %s", output_chunk.Data()));
+ outputChunk.Data());
+ gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
}
}
if (fMergeExcludes.Length() &&
- fMergeExcludes.Contains(output_file.Data())) continue;
+ fMergeExcludes.Contains(outputFile.Data())) continue;
// Perform a 'find' command in the output directory, looking for registered outputs
- merged = MergeOutput(output_file, fGridOutputDir, fMaxMergeFiles);
+ merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
if (!merged) {
Error("MergeOutputs", "Terminate() will NOT be executed");
return kFALSE;
- }
+ }
+ TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
+ if (fileOpened) fileOpened->Close();
}
return kTRUE;
}
// Use the output files connected to output containers from the analysis manager
// rather than the files defined by SetOutputFiles
if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
- Info("SetDefaultOutputs", "Plugin will use the output files taken from \
- analysis manager");
+ Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
}
TObjArray *list = fOutputFiles.Tokenize(",");
TIter next(list);
TObjString *str;
- TString output_file;
+ TString outputFile;
while((str=(TObjString*)next())) {
- output_file = str->GetString();
- Int_t index = output_file.Index("@");
- if (index > 0) output_file.Remove(index);
- if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
+ outputFile = str->GetString();
+ Int_t index = outputFile.Index("@");
+ if (index > 0) outputFile.Remove(index);
+ if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
}
delete list;
gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
TString mergeJDLName = fExecutable;
mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
Int_t ntosubmit = fInputFiles->GetEntries();
- printf("### Submitting %d merging jobs...\n", ntosubmit);
for (Int_t i=0; i<ntosubmit; i++) {
- TString query;
TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
runOutDir.ReplaceAll(".xml", "");
- if (fOutputToRunNo)
- query = Form("submit %s %s", mergeJDLName.Data(), runOutDir.Data());
- else
- query = Form("submit %s %03d", mergeJDLName.Data(), i);
- printf("********* %s\n",query.Data());
- TGridResult *res = gGrid->Command(query);
- if (res) {
- const char *cjobId = res->GetKey(0,"jobId");
- if (!cjobId) {
- gGrid->Stdout();
- gGrid->Stderr();
- Error("StartAnalysis", "Your JDL %s could not be submitted", mergeJDLName.Data());
- return kFALSE;
- } else {
- Info("StartAnalysis", "\n_______________________________________________________________________ \
- \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
- \n_______________________________________________________________________",
- mergeJDLName.Data(), cjobId);
- }
- delete res;
- } else {
- Error("SubmitMerging", "No grid result after submission !!! Bailing out...");
- return kFALSE;
- }
+ if (fOutputToRunNo) {
+ // The output directory is the run number
+ printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
+ runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
+ } else {
+ // The output directory is the master number in 3 digits format
+ printf("### Submitting merging job for master <%03d>\n", i);
+ runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
+ }
+ // Check now the number of merging stages.
+ TObjArray *list = fOutputFiles.Tokenize(",");
+ TIter next(list);
+ TObjString *str;
+ TString outputFile;
+ while((str=(TObjString*)next())) {
+ outputFile = str->GetString();
+ Int_t index = outputFile.Index("@");
+ if (index > 0) outputFile.Remove(index);
+ if (!fMergeExcludes.Contains(outputFile)) break;
+ }
+ delete list;
+ Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, kTRUE, mergeJDLName);
+ if (!done) return kFALSE;
}
if (!ntosubmit) return kTRUE;
Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR MERGING JOBS HAVE FINISHED. #### \
TDirectory *cdir = gDirectory;
TFile *file = TFile::Open(analysisFile, "RECREATE");
if (file) {
- // Skip task Terminate calls for the grid job
- mgr->SetSkipTerminate(kTRUE);
+ // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
+ if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
// Unless merging makes no sense
if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
mgr->Write();
if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
out << "// Load analysis framework libraries" << endl;
+ TString setupPar = "AliAnalysisAlien::SetupPar";
if (!fPackages) {
out << " gSystem->Load(\"libSTEERBase\");" << endl;
out << " gSystem->Load(\"libESD\");" << endl;
TIter next(fPackages);
TObject *obj;
TString pkgname;
- TString setupPar = "AliAnalysisAlien::SetupPar";
while ((obj=next())) {
pkgname = obj->GetName();
if (pkgname == "STEERBase" ||
while((str=(TObjString*)next())) {
if (str->GetString().Contains(".so"))
out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
+ if (str->GetString().Contains(".par"))
+ out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
}
if (list) delete list;
}
out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",10);" << endl;
out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
}
+ // Change temp directory to current one
+ out << "// Set temporary merging directory to current one" << endl;
+ out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
out << "// connect to AliEn and make the chain" << endl;
out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
if (IsUsingTags()) {
out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
} else {
- if(fFriendChainName!="AliAOD.VertexingHF.root") {
- out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
- } else {
- out << " // Check if the macro to create the chain was provided" << endl;
- out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
- out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
- out << " return;" << endl;
- out << " }" << endl;
- out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
- out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
- }
+ out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
}
out << "// read the analysis manager from file" << endl;
TString analysisFile = fExecutable;
out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
out << " };" << endl;
out << " if (!mgr) {" << endl;
- out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
+ out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file " << analysisFile <<"\");" << endl;
out << " return;" << endl;
out << " }" << endl << endl;
out << " mgr->PrintStatus();" << endl;
TString func = mergingMacro;
TString comment;
func.ReplaceAll(".C", "");
- out << "void " << func.Data() << "(const char *dir)" << endl;
+ out << "void " << func.Data() << "(const char *dir, Int_t stage=0, Int_t ichunk=0)" << endl;
out << "{" << endl;
out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
out << " TStopwatch timer;" << endl;
out << " timer.Start();" << endl << endl;
- out << "// load base root libraries" << endl;
- out << " gSystem->Load(\"libTree\");" << endl;
- out << " gSystem->Load(\"libGeom\");" << endl;
- out << " gSystem->Load(\"libVMC\");" << endl;
- out << " gSystem->Load(\"libPhysics\");" << endl << endl;
- out << " gSystem->Load(\"libMinuit\");" << endl << endl;
+ if (!fExecutableCommand.Contains("aliroot")) {
+ out << "// load base root libraries" << endl;
+ out << " gSystem->Load(\"libTree\");" << endl;
+ out << " gSystem->Load(\"libGeom\");" << endl;
+ out << " gSystem->Load(\"libVMC\");" << endl;
+ out << " gSystem->Load(\"libPhysics\");" << endl << endl;
+ out << " gSystem->Load(\"libMinuit\");" << endl << endl;
+ }
if (fAdditionalRootLibs.Length()) {
// in principle libtree /lib geom libvmc etc. can go into this list, too
out << "// Add aditional libraries" << endl;
out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
out << "// Load analysis framework libraries" << endl;
if (!fPackages) {
- out << " gSystem->Load(\"libSTEERBase\");" << endl;
- out << " gSystem->Load(\"libESD\");" << endl;
- out << " gSystem->Load(\"libAOD\");" << endl;
+ if (!fExecutableCommand.Contains("aliroot")) {
+ out << " gSystem->Load(\"libSTEERBase\");" << endl;
+ out << " gSystem->Load(\"libESD\");" << endl;
+ out << " gSystem->Load(\"libAOD\");" << endl;
+ }
out << " gSystem->Load(\"libANALYSIS\");" << endl;
out << " gSystem->Load(\"libANALYSISalice\");" << endl;
out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",10);" << endl;
out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
- }
+ }
+ // Change temp directory to current one
+ out << "// Set temporary merging directory to current one" << endl;
+ out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
out << "// Connect to AliEn" << endl;
out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
- out << " TString outputDir = \"" << fGridOutputDir << "/\";" << endl;
- out << " outputDir += dir;" << endl;
+ out << " Bool_t laststage = kFALSE;" << endl;
+ out << " TString outputDir = dir;" << endl;
out << " TString outputFiles = \"" << fOutputFiles << "\";" << endl;
out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
out << " mergeExcludes += \"" << AliAnalysisManager::GetAnalysisManager()->GetExtraFiles() << "\";" << endl;
- out << " TObjArray *list = outputFiles.Tokenize(\" \");" << endl;
+ out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
out << " TIter *iter = new TIter(list);" << endl;
out << " TObjString *str;" << endl;
- out << " TString output_file;" << endl;
+ out << " TString outputFile;" << endl;
out << " Bool_t merged = kTRUE;" << endl;
out << " while((str=(TObjString*)iter->Next())) {" << endl;
- out << " output_file = str->GetString();" << endl;
- out << " Int_t index = output_file.Index(\"@\");" << endl;
- out << " if (index > 0) output_file.Remove(index);" << endl;
+ out << " outputFile = str->GetString();" << endl;
+ out << " if (outputFile.Contains(\"*\")) continue;" << endl;
+ out << " Int_t index = outputFile.Index(\"@\");" << endl;
+ out << " if (index > 0) outputFile.Remove(index);" << endl;
out << " // Skip already merged outputs" << endl;
- out << " if (!gSystem->AccessPathName(output_file)) {" << endl;
- out << " printf(\"Output file <%s> found. Not merging again.\",output_file.Data());" << endl;
+ out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
+ out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
out << " continue;" << endl;
out << " }" << endl;
- out << " if (mergeExcludes.Contains(output_file.Data())) continue;" << endl;
- out << " merged = AliAnalysisAlien::MergeOutput(output_file, outputDir, " << fMaxMergeFiles << ");" << endl;
+ out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
+ out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage, ichunk);" << endl;
out << " if (!merged) {" << endl;
- out << " printf(\"ERROR: Cannot merge %s\\n\", output_file.Data());" << endl;
+ out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
+ out << " return;" << endl;
out << " }" << endl;
+ out << " // Check if this was the last stage. If yes, run terminate for the tasks." << endl;
+ out << " if (!gSystem->AccessPathName(outputFile)) laststage = kTRUE;" << endl;
out << " }" << endl;
- out << "// read the analysis manager from file" << endl;
+ out << " // all outputs merged, validate" << endl;
+ out << " ofstream out;" << endl;
+ out << " out.open(\"outputs_valid\", ios::out);" << endl;
+ out << " out.close();" << endl;
+ out << " // read the analysis manager from file" << endl;
TString analysisFile = fExecutable;
analysisFile.ReplaceAll(".sh", ".root");
+ out << " if (!laststage) return;" << endl;
out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
- out << " if (!file) return;" << endl;
+ out << " if (!file) return;" << endl;
out << " TIter nextkey(file->GetListOfKeys());" << endl;
out << " AliAnalysisManager *mgr = 0;" << endl;
out << " TKey *key;" << endl;
//______________________________________________________________________________
Bool_t AliAnalysisAlien::SetupPar(const char *package)
{
-// Compile the par file archive pointed by <package>. This must be present in the current durectory.
+// Compile the par file archive pointed by <package>. This must be present in the current directory.
// Note that for loading the compiled library. The current directory should have precedence in
// LD_LIBRARY_PATH
TString pkgdir = package;
CdWork();
TString workdir = gGrid->GetHomeDirectory();
TString bindir = Form("%s/bin", workdir.Data());
- if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
+ if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
workdir += fGridWorkingDir;
TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
if (FileExists(executable)) gGrid->Rm(executable);
if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
TString mergeMacro = fExecutable;
mergeMacro.ReplaceAll(".sh", "_merge.C");
- out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
+ out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2,$3)\"" << endl;
out << fExecutableCommand << " " << "$ARG" << endl;
out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
out << "echo \"############## memory after: ##############\"" << endl;
CdWork();
TString workdir = gGrid->GetHomeDirectory();
TString bindir = Form("%s/bin", workdir.Data());
- if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
+ if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
workdir += fGridWorkingDir;
TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
if (FileExists(executable)) gGrid->Rm(executable);
Error("WriteValidationScript", "Alien connection required");
return;
}
- TString out_stream = "";
- if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
+ TString outStream = "";
+ if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
if (!TestBit(AliAnalysisGrid::kSubmit)) {
ofstream out;
out.open(validationScript, ios::out);
out << "fi" << endl << endl;
out << "cd $validateout;" << endl;
out << "validateworkdir=`pwd`;" << endl << endl;
- out << "echo \"*******************************************************\"" << out_stream << endl;
- out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
+ out << "echo \"*******************************************************\"" << outStream << endl;
+ out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
out << "" << endl;
- out << "echo \"* Time: $validatetime \"" << out_stream << endl;
- out << "echo \"* Dir: $validateout\"" << out_stream << endl;
- out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
- out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
- out << "ls -la ./" << out_stream << endl;
- out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
+ out << "echo \"* Time: $validatetime \"" << outStream << endl;
+ out << "echo \"* Dir: $validateout\"" << outStream << endl;
+ out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
+ out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
+ out << "ls -la ./" << outStream << endl;
+ out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
out << "##################################################" << endl;
-
- out << "" << endl;
- out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
- out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
- out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
out << "" << endl;
out << "if [ ! -f stderr ] ; then" << endl;
out << " error=1" << endl;
- out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
- out << " echo \"Error = $error\" " << out_stream << endl;
+ out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
+ out << " echo \"Error = $error\" " << outStream << endl;
out << "fi" << endl;
+ out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
+ out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
+ out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
+ out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
+ out << "" << endl;
+
out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
out << " error=1" << endl;
- out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
- out << " echo \"$parArch\" " << out_stream << endl;
- out << " echo \"Error = $error\" " << out_stream << endl;
+ out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
+ out << " echo \"$parArch\" " << outStream << endl;
+ out << " echo \"Error = $error\" " << outStream << endl;
out << "fi" << endl;
out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
out << " error=1" << endl;
- out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
- out << " echo \"$segViol\" " << out_stream << endl;
- out << " echo \"Error = $error\" " << out_stream << endl;
+ out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
+ out << " echo \"$segViol\" " << outStream << endl;
+ out << " echo \"Error = $error\" " << outStream << endl;
out << "fi" << endl;
out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
out << " error=1" << endl;
- out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
- out << " echo \"$segFault\" " << out_stream << endl;
- out << " echo \"Error = $error\" " << out_stream << endl;
+ out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
+ out << " echo \"$segFault\" " << outStream << endl;
+ out << " echo \"Error = $error\" " << outStream << endl;
+ out << "fi" << endl;
+
+ out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
+ out << " error=1" << endl;
+ out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
+ out << " echo \"$glibcErr\" " << outStream << endl;
+ out << " echo \"Error = $error\" " << outStream << endl;
out << "fi" << endl;
// Part dedicated to the specific analyses running into the train
TObjArray *arr = fOutputFiles.Tokenize(",");
TIter next1(arr);
- TString output_file;
+ TString outputFile;
AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
TString extra = mgr->GetExtraFiles();
while ((os=(TObjString*)next1())) {
- output_file = os->GetString();
- Int_t index = output_file.Index("@");
- if (index > 0) output_file.Remove(index);
- if (merge && fMergeExcludes.Contains(output_file)) continue;
- if (extra.Contains(output_file)) continue;
- out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
+ if (merge) break;
+ outputFile = os->GetString();
+ Int_t index = outputFile.Index("@");
+ if (index > 0) outputFile.Remove(index);
+ if (merge && fMergeExcludes.Contains(outputFile)) continue;
+ if (extra.Contains(outputFile)) continue;
+ if (outputFile.Contains("*")) continue;
+ out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
out << " error=1" << endl;
- out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
- out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
+ out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
+ out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
out << "fi" << endl;
}
delete arr;
- if (!merge) {
- out << "if ! [ -f outputs_valid ] ; then" << endl;
- out << " error=1" << endl;
- out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
- out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
- out << "fi" << endl;
- }
+ out << "if ! [ -f outputs_valid ] ; then" << endl;
+ out << " error=1" << endl;
+ out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
+ out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
+ out << "fi" << endl;
out << "if [ $error = 0 ] ; then" << endl;
- out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
+ out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
if (!IsKeepLogs()) {
out << " echo \"* === Logs std* will be deleted === \"" << endl;
- out_stream = "";
+ outStream = "";
out << " rm -f std*" << endl;
}
out << "fi" << endl;
- out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
- out << "echo \"*******************************************************\"" << out_stream << endl;
+ out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
+ out << "echo \"*******************************************************\"" << outStream << endl;
out << "cd -" << endl;
out << "exit $error" << endl;
}
git://git.uio.no / u / mrichter / AliRoot.git / blobdiff