]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Correcly adds prefix to AddRunNumber list of entries (S.Carrazza)
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
7c2cd90a 27#include "TBits.h"
0f389141 28#include "TError.h"
c57f56b7 29#include "TROOT.h"
30#include "TSystem.h"
31#include "TFile.h"
830acc4c 32#include "TFileCollection.h"
3bdcb562 33#include "TChain.h"
c57f56b7 34#include "TObjString.h"
35#include "TObjArray.h"
36#include "TGrid.h"
37#include "TGridResult.h"
38#include "TGridCollection.h"
39#include "TGridJDL.h"
d2a409b2 40#include "TGridJobStatusList.h"
41#include "TGridJobStatus.h"
c57f56b7 42#include "TFileMerger.h"
43#include "AliAnalysisManager.h"
bb885a9e 44#include "AliVEventHandler.h"
45#include "AliAnalysisDataContainer.h"
65eb22f8 46#include "AliMultiInputEventHandler.h"
c57f56b7 47
48ClassImp(AliAnalysisAlien)
49
50//______________________________________________________________________________
51AliAnalysisAlien::AliAnalysisAlien()
52 :AliAnalysisGrid(),
53 fGridJDL(NULL),
0f389141 54 fMergingJDL(NULL),
c57f56b7 55 fPrice(0),
56 fTTL(0),
57 fSplitMaxInputFileNumber(0),
58 fMaxInitFailed(0),
59 fMasterResubmitThreshold(0),
bb885a9e 60 fNtestFiles(0),
319593fb 61 fNrunsPerMaster(0),
16a4353c 62 fMaxMergeFiles(0),
b3e07543 63 fMaxMergeStages(0),
d2a409b2 64 fNsubmitted(0),
a3e84053 65 fProductionMode(0),
cd11251e 66 fOutputToRunNo(0),
0f389141 67 fMergeViaJDL(0),
68 fFastReadOption(0),
e1c22e21 69 fOverwriteMode(1),
149d288c 70 fNreplicas(2),
3bdcb562 71 fNproofWorkers(0),
27734f0e 72 fNproofWorkersPerSlave(0),
3bdcb562 73 fProofReset(0),
c57f56b7 74 fRunNumbers(),
75 fExecutable(),
0a1c1f7f 76 fExecutableCommand(),
c57f56b7 77 fArguments(),
631c0b05 78 fExecutableArgs(),
c57f56b7 79 fAnalysisMacro(),
80 fAnalysisSource(),
d3b18c4c 81 fValidationScript(),
d5c6455a 82 fAdditionalRootLibs(),
c57f56b7 83 fAdditionalLibs(),
84 fSplitMode(),
85 fAPIVersion(),
86 fROOTVersion(),
87 fAliROOTVersion(),
648174cf 88 fExternalPackages(),
c57f56b7 89 fUser(),
90 fGridWorkingDir(),
91 fGridDataDir(),
92 fDataPattern(),
93 fGridOutputDir(),
94 fOutputArchive(),
95 fOutputFiles(),
96 fInputFormat(),
e7c71df0 97 fDatasetName(),
c57f56b7 98 fJDLName(),
d3b18c4c 99 fTerminateFiles(),
bb885a9e 100 fMergeExcludes(),
f965131e 101 fIncludePath(),
bb885a9e 102 fCloseSE(),
0df6ccf2 103 fFriendChainName(),
c6cb3634 104 fJobTag(),
648174cf 105 fOutputSingle(),
5fce53f4 106 fRunPrefix(),
3bdcb562 107 fProofCluster(),
108 fProofDataSet(),
109 fFileForTestMode(),
110 fRootVersionForProof(),
111 fAliRootMode(),
f47d5cb4 112 fMergeDirName(),
4e5c5506 113 fInputFiles(0),
114 fPackages(0)
c57f56b7 115{
116// Dummy ctor.
117 SetDefaults();
118}
119
120//______________________________________________________________________________
121AliAnalysisAlien::AliAnalysisAlien(const char *name)
122 :AliAnalysisGrid(name),
123 fGridJDL(NULL),
0f389141 124 fMergingJDL(NULL),
c57f56b7 125 fPrice(0),
126 fTTL(0),
127 fSplitMaxInputFileNumber(0),
128 fMaxInitFailed(0),
129 fMasterResubmitThreshold(0),
bb885a9e 130 fNtestFiles(0),
319593fb 131 fNrunsPerMaster(0),
16a4353c 132 fMaxMergeFiles(0),
b3e07543 133 fMaxMergeStages(0),
d2a409b2 134 fNsubmitted(0),
a3e84053 135 fProductionMode(0),
cd11251e 136 fOutputToRunNo(0),
0f389141 137 fMergeViaJDL(0),
138 fFastReadOption(0),
e1c22e21 139 fOverwriteMode(1),
149d288c 140 fNreplicas(2),
3bdcb562 141 fNproofWorkers(0),
27734f0e 142 fNproofWorkersPerSlave(0),
3bdcb562 143 fProofReset(0),
c57f56b7 144 fRunNumbers(),
145 fExecutable(),
0a1c1f7f 146 fExecutableCommand(),
c57f56b7 147 fArguments(),
631c0b05 148 fExecutableArgs(),
c57f56b7 149 fAnalysisMacro(),
150 fAnalysisSource(),
d3b18c4c 151 fValidationScript(),
d5c6455a 152 fAdditionalRootLibs(),
c57f56b7 153 fAdditionalLibs(),
154 fSplitMode(),
155 fAPIVersion(),
156 fROOTVersion(),
157 fAliROOTVersion(),
648174cf 158 fExternalPackages(),
c57f56b7 159 fUser(),
160 fGridWorkingDir(),
161 fGridDataDir(),
162 fDataPattern(),
163 fGridOutputDir(),
164 fOutputArchive(),
165 fOutputFiles(),
166 fInputFormat(),
e7c71df0 167 fDatasetName(),
c57f56b7 168 fJDLName(),
d3b18c4c 169 fTerminateFiles(),
bb885a9e 170 fMergeExcludes(),
f965131e 171 fIncludePath(),
bb885a9e 172 fCloseSE(),
0df6ccf2 173 fFriendChainName(),
c6cb3634 174 fJobTag(),
648174cf 175 fOutputSingle(),
5fce53f4 176 fRunPrefix(),
3bdcb562 177 fProofCluster(),
178 fProofDataSet(),
179 fFileForTestMode(),
180 fRootVersionForProof(),
181 fAliRootMode(),
f47d5cb4 182 fMergeDirName(),
4e5c5506 183 fInputFiles(0),
184 fPackages(0)
c57f56b7 185{
186// Default ctor.
187 SetDefaults();
188}
189
190//______________________________________________________________________________
191AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
192 :AliAnalysisGrid(other),
193 fGridJDL(NULL),
0f389141 194 fMergingJDL(NULL),
c57f56b7 195 fPrice(other.fPrice),
196 fTTL(other.fTTL),
197 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
198 fMaxInitFailed(other.fMaxInitFailed),
199 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 200 fNtestFiles(other.fNtestFiles),
319593fb 201 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 202 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 203 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 204 fNsubmitted(other.fNsubmitted),
a3e84053 205 fProductionMode(other.fProductionMode),
cd11251e 206 fOutputToRunNo(other.fOutputToRunNo),
0f389141 207 fMergeViaJDL(other.fMergeViaJDL),
208 fFastReadOption(other.fFastReadOption),
209 fOverwriteMode(other.fOverwriteMode),
149d288c 210 fNreplicas(other.fNreplicas),
3bdcb562 211 fNproofWorkers(other.fNproofWorkers),
27734f0e 212 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 213 fProofReset(other.fProofReset),
c57f56b7 214 fRunNumbers(other.fRunNumbers),
215 fExecutable(other.fExecutable),
0a1c1f7f 216 fExecutableCommand(other.fExecutableCommand),
c57f56b7 217 fArguments(other.fArguments),
631c0b05 218 fExecutableArgs(other.fExecutableArgs),
c57f56b7 219 fAnalysisMacro(other.fAnalysisMacro),
220 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 221 fValidationScript(other.fValidationScript),
d5c6455a 222 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 223 fAdditionalLibs(other.fAdditionalLibs),
224 fSplitMode(other.fSplitMode),
225 fAPIVersion(other.fAPIVersion),
226 fROOTVersion(other.fROOTVersion),
227 fAliROOTVersion(other.fAliROOTVersion),
648174cf 228 fExternalPackages(other.fExternalPackages),
c57f56b7 229 fUser(other.fUser),
230 fGridWorkingDir(other.fGridWorkingDir),
231 fGridDataDir(other.fGridDataDir),
232 fDataPattern(other.fDataPattern),
233 fGridOutputDir(other.fGridOutputDir),
234 fOutputArchive(other.fOutputArchive),
235 fOutputFiles(other.fOutputFiles),
236 fInputFormat(other.fInputFormat),
e7c71df0 237 fDatasetName(other.fDatasetName),
c57f56b7 238 fJDLName(other.fJDLName),
d3b18c4c 239 fTerminateFiles(other.fTerminateFiles),
bb885a9e 240 fMergeExcludes(other.fMergeExcludes),
f965131e 241 fIncludePath(other.fIncludePath),
bb885a9e 242 fCloseSE(other.fCloseSE),
0df6ccf2 243 fFriendChainName(other.fFriendChainName),
c6cb3634 244 fJobTag(other.fJobTag),
648174cf 245 fOutputSingle(other.fOutputSingle),
5fce53f4 246 fRunPrefix(other.fRunPrefix),
3bdcb562 247 fProofCluster(other.fProofCluster),
248 fProofDataSet(other.fProofDataSet),
249 fFileForTestMode(other.fFileForTestMode),
250 fRootVersionForProof(other.fRootVersionForProof),
251 fAliRootMode(other.fAliRootMode),
f47d5cb4 252 fMergeDirName(other.fMergeDirName),
4e5c5506 253 fInputFiles(0),
254 fPackages(0)
c57f56b7 255{
256// Copy ctor.
257 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 258 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 259 fRunRange[0] = other.fRunRange[0];
260 fRunRange[1] = other.fRunRange[1];
c57f56b7 261 if (other.fInputFiles) {
262 fInputFiles = new TObjArray();
263 TIter next(other.fInputFiles);
264 TObject *obj;
265 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
266 fInputFiles->SetOwner();
267 }
4e5c5506 268 if (other.fPackages) {
269 fPackages = new TObjArray();
270 TIter next(other.fPackages);
271 TObject *obj;
272 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
273 fPackages->SetOwner();
274 }
c57f56b7 275}
276
277//______________________________________________________________________________
278AliAnalysisAlien::~AliAnalysisAlien()
279{
280// Destructor.
281 if (fGridJDL) delete fGridJDL;
0f389141 282 if (fMergingJDL) delete fMergingJDL;
c57f56b7 283 if (fInputFiles) delete fInputFiles;
4e5c5506 284 if (fPackages) delete fPackages;
c57f56b7 285}
286
287//______________________________________________________________________________
288AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
289{
290// Assignment.
291 if (this != &other) {
292 AliAnalysisGrid::operator=(other);
293 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 294 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 295 fPrice = other.fPrice;
296 fTTL = other.fTTL;
297 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
298 fMaxInitFailed = other.fMaxInitFailed;
299 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 300 fNtestFiles = other.fNtestFiles;
a3e84053 301 fNrunsPerMaster = other.fNrunsPerMaster;
302 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 303 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 304 fNsubmitted = other.fNsubmitted;
305 fProductionMode = other.fProductionMode;
cd11251e 306 fOutputToRunNo = other.fOutputToRunNo;
0f389141 307 fMergeViaJDL = other.fMergeViaJDL;
308 fFastReadOption = other.fFastReadOption;
309 fOverwriteMode = other.fOverwriteMode;
149d288c 310 fNreplicas = other.fNreplicas;
3bdcb562 311 fNproofWorkers = other.fNproofWorkers;
27734f0e 312 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 313 fProofReset = other.fProofReset;
c57f56b7 314 fRunNumbers = other.fRunNumbers;
315 fExecutable = other.fExecutable;
0a1c1f7f 316 fExecutableCommand = other.fExecutableCommand;
c57f56b7 317 fArguments = other.fArguments;
631c0b05 318 fExecutableArgs = other.fExecutableArgs;
c57f56b7 319 fAnalysisMacro = other.fAnalysisMacro;
320 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 321 fValidationScript = other.fValidationScript;
d5c6455a 322 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 323 fAdditionalLibs = other.fAdditionalLibs;
324 fSplitMode = other.fSplitMode;
325 fAPIVersion = other.fAPIVersion;
326 fROOTVersion = other.fROOTVersion;
327 fAliROOTVersion = other.fAliROOTVersion;
648174cf 328 fExternalPackages = other.fExternalPackages;
c57f56b7 329 fUser = other.fUser;
330 fGridWorkingDir = other.fGridWorkingDir;
331 fGridDataDir = other.fGridDataDir;
332 fDataPattern = other.fDataPattern;
333 fGridOutputDir = other.fGridOutputDir;
334 fOutputArchive = other.fOutputArchive;
335 fOutputFiles = other.fOutputFiles;
336 fInputFormat = other.fInputFormat;
e7c71df0 337 fDatasetName = other.fDatasetName;
c57f56b7 338 fJDLName = other.fJDLName;
d3b18c4c 339 fTerminateFiles = other.fTerminateFiles;
bb885a9e 340 fMergeExcludes = other.fMergeExcludes;
f965131e 341 fIncludePath = other.fIncludePath;
bb885a9e 342 fCloseSE = other.fCloseSE;
0df6ccf2 343 fFriendChainName = other.fFriendChainName;
c6cb3634 344 fJobTag = other.fJobTag;
648174cf 345 fOutputSingle = other.fOutputSingle;
5fce53f4 346 fRunPrefix = other.fRunPrefix;
3bdcb562 347 fProofCluster = other.fProofCluster;
348 fProofDataSet = other.fProofDataSet;
349 fFileForTestMode = other.fFileForTestMode;
350 fRootVersionForProof = other.fRootVersionForProof;
351 fAliRootMode = other.fAliRootMode;
f47d5cb4 352 fMergeDirName = other.fMergeDirName;
c57f56b7 353 if (other.fInputFiles) {
354 fInputFiles = new TObjArray();
355 TIter next(other.fInputFiles);
356 TObject *obj;
357 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
358 fInputFiles->SetOwner();
359 }
4e5c5506 360 if (other.fPackages) {
361 fPackages = new TObjArray();
362 TIter next(other.fPackages);
363 TObject *obj;
364 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
365 fPackages->SetOwner();
366 }
c57f56b7 367 }
368 return *this;
369}
370
74013535 371//______________________________________________________________________________
372void AliAnalysisAlien::SetRunPrefix(const char *prefix)
373{
374// Set the run number format. Can be a prefix or a format like "%09d"
375 fRunPrefix = prefix;
376 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
377}
378
f965131e 379//______________________________________________________________________________
380void AliAnalysisAlien::AddIncludePath(const char *path)
381{
382// Add include path in the remote analysis macro.
383 TString p(path);
384 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
385 else fIncludePath += Form("-I%s ", path);
386}
387
c57f56b7 388//______________________________________________________________________________
389void AliAnalysisAlien::AddRunNumber(Int_t run)
390{
391// Add a run number to the list of runs to be processed.
392 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 393 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 394}
395
98690895 396//______________________________________________________________________________
397void AliAnalysisAlien::AddRunList(const char* runList)
398{
399// Add several runs into the list of runs; they are expected to be separated by a blank character.
400 TString sList = runList;
401 TObjArray *list = sList.Tokenize(" ");
402 Int_t n = list->GetEntries();
403 for (Int_t i = 0; i < n; i++) {
404 TObjString *os = (TObjString*)list->At(i);
405 AddRunNumber(os->GetString().Atoi());
406 }
407 delete list;
408}
409
ee75cfc3 410//______________________________________________________________________________
411void AliAnalysisAlien::AddRunNumber(const char* run)
412{
413// Add a run number to the list of runs to be processed.
7aa5d539 414 TString runs = run;
415 TObjString *os;
416 TObjArray *arr = runs.Tokenize(" ");
417 TIter next(arr);
418 TString prefix;
419 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
420 while ((os=(TObjString*)next())){
421 if (fRunNumbers.Length()) fRunNumbers += " ";
422 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
423 }
424 delete arr;
ee75cfc3 425}
426
c57f56b7 427//______________________________________________________________________________
428void AliAnalysisAlien::AddDataFile(const char *lfn)
429{
430// Adds a data file to the input to be analysed. The file should be a valid LFN
431// or point to an existing file in the alien workdir.
432 if (!fInputFiles) fInputFiles = new TObjArray();
433 fInputFiles->Add(new TObjString(lfn));
434}
648174cf 435
436//______________________________________________________________________________
437void AliAnalysisAlien::AddExternalPackage(const char *package)
438{
439// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
440 if (fExternalPackages) fExternalPackages += " ";
441 fExternalPackages += package;
442}
443
c57f56b7 444//______________________________________________________________________________
445Bool_t AliAnalysisAlien::Connect()
446{
447// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
448 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 449 if (fProductionMode) return kTRUE;
c57f56b7 450 if (!gGrid) {
451 Info("Connect", "Trying to connect to AliEn ...");
452 TGrid::Connect("alien://");
453 }
454 if (!gGrid || !gGrid->IsConnected()) {
455 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
456 return kFALSE;
457 }
458 fUser = gGrid->GetUser();
459 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
460 return kTRUE;
461}
462
463//______________________________________________________________________________
464void AliAnalysisAlien::CdWork()
465{
466// Check validity of alien workspace. Create directory if possible.
467 if (!Connect()) {
468 Error("CdWork", "Alien connection required");
469 return;
470 }
471 TString homedir = gGrid->GetHomeDirectory();
472 TString workdir = homedir + fGridWorkingDir;
923e2ca5 473 if (DirectoryExists(workdir)) {
474 gGrid->Cd(workdir);
475 return;
476 }
477 // Work directory not existing - create it
478 gGrid->Cd(homedir);
b93f8109 479 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 480 gGrid->Cd(fGridWorkingDir);
d3b18c4c 481 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 482 } else {
d3b18c4c 483 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 484 workdir.Data(), homedir.Data());
485 fGridWorkingDir = "";
486 }
c57f56b7 487}
488
348be253 489//______________________________________________________________________________
490Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
491{
492// Check if file copying is possible.
d3b18c4c 493 if (fProductionMode) return kTRUE;
348be253 494 if (!Connect()) {
495 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
496 return kFALSE;
497 }
d3339be3 498 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
499 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 500 // Check if alien_CLOSE_SE is defined
501 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
502 if (!closeSE.IsNull()) {
503 Info("CheckFileCopy", "Your current close storage is pointing to: \
504 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
505 } else {
506 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
507 }
508 // Check if grid directory exists.
509 if (!DirectoryExists(alienpath)) {
510 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
511 return kFALSE;
512 }
d3339be3 513 TFile f("plugin_test_copy", "RECREATE");
348be253 514 // User may not have write permissions to current directory
515 if (f.IsZombie()) {
516 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
517 gSystem->WorkingDirectory());
518 return kFALSE;
519 }
520 f.Close();
d3339be3 521 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
522 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
523 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 524 \n# 1. Make sure you have write permissions there. If this is the case: \
525 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
526 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
527 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
528 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
529 gSystem->Unlink(f.GetName());
530 return kFALSE;
531 }
532 gSystem->Unlink(f.GetName());
d3339be3 533 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
534 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 535 return kTRUE;
536}
537
c57f56b7 538//______________________________________________________________________________
539Bool_t AliAnalysisAlien::CheckInputData()
540{
541// Check validity of input data. If necessary, create xml files.
d3b18c4c 542 if (fProductionMode) return kTRUE;
a8739e8a 543 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 544 if (!fGridDataDir.Length()) {
545 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
546 return kFALSE;
547 }
9f020952 548 if (fMergeViaJDL) {
549 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
550 return kFALSE;
551 }
d2a409b2 552 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 553 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
554 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 555 return kTRUE;
c57f56b7 556 }
557 // Process declared files
a2f5fc01 558 Bool_t isCollection = kFALSE;
559 Bool_t isXml = kFALSE;
560 Bool_t useTags = kFALSE;
c57f56b7 561 Bool_t checked = kFALSE;
d3b18c4c 562 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 563 TString file;
564 TString workdir = gGrid->GetHomeDirectory();
565 workdir += fGridWorkingDir;
566 if (fInputFiles) {
567 TObjString *objstr;
568 TIter next(fInputFiles);
569 while ((objstr=(TObjString*)next())) {
570 file = workdir;
571 file += "/";
572 file += objstr->GetString();
573 // Store full lfn path
574 if (FileExists(file)) objstr->SetString(file);
575 else {
576 file = objstr->GetName();
577 if (!FileExists(objstr->GetName())) {
578 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
579 objstr->GetName(), workdir.Data());
580 return kFALSE;
581 }
582 }
583 Bool_t iscoll, isxml, usetags;
584 CheckDataType(file, iscoll, isxml, usetags);
585 if (!checked) {
586 checked = kTRUE;
a2f5fc01 587 isCollection = iscoll;
588 isXml = isxml;
589 useTags = usetags;
590 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 591 } else {
a2f5fc01 592 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 593 Error("CheckInputData", "Some conflict was found in the types of inputs");
594 return kFALSE;
595 }
596 }
597 }
598 }
599 // Process requested run numbers
a8739e8a 600 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 601 // Check validity of alien data directory
602 if (!fGridDataDir.Length()) {
603 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
604 return kFALSE;
605 }
923e2ca5 606 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 607 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
608 return kFALSE;
609 }
a2f5fc01 610 if (isCollection) {
c57f56b7 611 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
612 return kFALSE;
613 }
614
a2f5fc01 615 if (checked && !isXml) {
c57f56b7 616 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
617 return kFALSE;
618 }
619 // Check validity of run number(s)
620 TObjArray *arr;
621 TObjString *os;
74013535 622 TString format;
319593fb 623 Int_t nruns = 0;
904f9f5f 624 TString schunk, schunk2;
c57f56b7 625 TString path;
626 if (!checked) {
627 checked = kTRUE;
a2f5fc01 628 useTags = fDataPattern.Contains("tag");
629 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 630 }
a2f5fc01 631 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 632 Error("CheckInputData", "Cannot mix input files using/not using tags");
633 return kFALSE;
634 }
635 if (fRunNumbers.Length()) {
a8739e8a 636 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 637 arr = fRunNumbers.Tokenize(" ");
638 TIter next(arr);
639 while ((os=(TObjString*)next())) {
640 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 641 if (!DirectoryExists(path)) {
642 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 643 continue;
c57f56b7 644 }
645 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
646 TString msg = "\n##### file: ";
647 msg += path;
648 msg += " type: xml_collection;";
a2f5fc01 649 if (useTags) msg += " using_tags: Yes";
c57f56b7 650 else msg += " using_tags: No";
23329835 651 Info("CheckDataType", "%s", msg.Data());
319593fb 652 if (fNrunsPerMaster<2) {
d2a409b2 653 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 654 } else {
655 nruns++;
656 if (((nruns-1)%fNrunsPerMaster) == 0) {
657 schunk = os->GetString();
658 }
659 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
660 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 661 AddDataFile(schunk);
319593fb 662 }
c57f56b7 663 }
664 delete arr;
a8739e8a 665 } else {
666 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
667 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 668 format = Form("%%s/%s ", fRunPrefix.Data());
669 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 670 if (!DirectoryExists(path)) {
a8739e8a 671 continue;
672 }
74013535 673 format = Form("%%s/%s.xml", fRunPrefix.Data());
674 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 675 TString msg = "\n##### file: ";
676 msg += path;
677 msg += " type: xml_collection;";
a2f5fc01 678 if (useTags) msg += " using_tags: Yes";
a8739e8a 679 else msg += " using_tags: No";
23329835 680 Info("CheckDataType", "%s", msg.Data());
319593fb 681 if (fNrunsPerMaster<2) {
74013535 682 format = Form("%s.xml", fRunPrefix.Data());
683 AddDataFile(Form(format.Data(),irun));
319593fb 684 } else {
685 nruns++;
686 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 687 schunk = Form(fRunPrefix.Data(),irun);
319593fb 688 }
74013535 689 format = Form("_%s.xml", fRunPrefix.Data());
690 schunk2 = Form(format.Data(), irun);
319593fb 691 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 692 schunk += schunk2;
d2a409b2 693 AddDataFile(schunk);
319593fb 694 }
a8739e8a 695 }
904f9f5f 696 if (!fInputFiles) {
697 schunk += schunk2;
698 AddDataFile(schunk);
699 }
c57f56b7 700 }
701 return kTRUE;
702}
703
704//______________________________________________________________________________
705Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
706{
707// Create dataset for the grid data directory + run number.
d3b18c4c 708 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 709 if (!Connect()) {
710 Error("CreateDataset", "Cannot create dataset with no grid connection");
711 return kFALSE;
712 }
713
714 // Cd workspace
d3b18c4c 715 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 716 TString workdir = gGrid->GetHomeDirectory();
717 workdir += fGridWorkingDir;
718
719 // Compose the 'find' command arguments
74013535 720 TString format;
c57f56b7 721 TString command;
722 TString options = "-x collection ";
bb885a9e 723 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 724 TString conditions = "";
725
726 TString file;
727 TString path;
319593fb 728 Int_t nruns = 0;
904f9f5f 729 TString schunk, schunk2;
ab254fd1 730 TGridCollection *cbase=0, *cadd=0;
d2a409b2 731 if (!fRunNumbers.Length() && !fRunRange[0]) {
732 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
733 // Make a single data collection from data directory.
734 path = fGridDataDir;
923e2ca5 735 if (!DirectoryExists(path)) {
d2a409b2 736 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
737 return kFALSE;
738 }
923e2ca5 739// CdWork();
d2a409b2 740 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
741 else file = Form("%s.xml", gSystem->BaseName(path));
0f389141 742 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
d2a409b2 743 command = "find ";
744 command += options;
745 command += path;
746 command += " ";
747 command += pattern;
748 command += conditions;
84fcd93f 749 printf("command: %s\n", command.Data());
d2a409b2 750 TGridResult *res = gGrid->Command(command);
751 if (res) delete res;
752 // Write standard output to file
753 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 754 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
755 Bool_t nullFile = kFALSE;
756 if (!hasGrep) {
d3339be3 757 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
758 } else {
a2f5fc01 759 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
760 if (nullFile) {
defd7a3a 761 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
762 return kFALSE;
763 }
d3339be3 764 }
0f389141 765 }
766 Bool_t fileExists = FileExists(file);
767 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 768 // Copy xml file to alien space
0f389141 769 if (fileExists) gGrid->Rm(file);
d2a409b2 770 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
771 if (!FileExists(file)) {
772 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
773 return kFALSE;
774 }
775 // Update list of files to be processed.
776 }
777 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
778 return kTRUE;
779 }
c57f56b7 780 // Several runs
a2f5fc01 781 Bool_t nullResult = kTRUE;
a8739e8a 782 if (fRunNumbers.Length()) {
783 TObjArray *arr = fRunNumbers.Tokenize(" ");
784 TObjString *os;
785 TIter next(arr);
786 while ((os=(TObjString*)next())) {
787 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 788 if (!DirectoryExists(path)) continue;
789// CdWork();
a8739e8a 790 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
791 else file = Form("%s.xml", os->GetString().Data());
319593fb 792 // If local collection file does not exist, create it via 'find' command.
d3339be3 793 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 794 command = "find ";
795 command += options;
796 command += path;
797 command += pattern;
798 command += conditions;
799 TGridResult *res = gGrid->Command(command);
800 if (res) delete res;
801 // Write standard output to file
802 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 803 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
804 Bool_t nullFile = kFALSE;
805 if (!hasGrep) {
d3339be3 806 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
807 } else {
a2f5fc01 808 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
809 if (nullFile) {
defd7a3a 810 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
811 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
812 continue;
813 }
d3339be3 814 }
a2f5fc01 815 nullResult = kFALSE;
d3339be3 816 }
a8739e8a 817 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 818 // Check if there is one run per master job.
819 if (fNrunsPerMaster<2) {
820 if (FileExists(file)) {
0f389141 821 if (fOverwriteMode) gGrid->Rm(file);
822 else {
823 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
824 continue;
825 }
319593fb 826 }
827 // Copy xml file to alien space
828 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
829 if (!FileExists(file)) {
830 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
831 delete arr;
832 return kFALSE;
833 }
834 } else {
835 nruns++;
836 if (((nruns-1)%fNrunsPerMaster) == 0) {
837 schunk = os->GetString();
838 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
839 } else {
840 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 841 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 842 cbase->Add(cadd);
843 delete cadd;
844 }
845 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
846 continue;
847 }
848 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 849 if (FileExists(schunk)) {
850 if (fOverwriteMode) gGrid->Rm(file);
851 else {
852 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
853 continue;
854 }
319593fb 855 }
84fcd93f 856 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 857 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 858 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 859 if (!FileExists(schunk)) {
860 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
861 delete arr;
862 return kFALSE;
863 }
d3339be3 864 }
a8739e8a 865 }
866 delete arr;
a2f5fc01 867 if (nullResult) {
d3339be3 868 Error("CreateDataset", "No valid dataset corresponding to the query!");
869 return kFALSE;
870 }
a8739e8a 871 } else {
872 // Process a full run range.
873 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 874 format = Form("%%s/%s ", fRunPrefix.Data());
875 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 876 if (!DirectoryExists(path)) continue;
877// CdWork();
74013535 878 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 879 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 880 else file = Form(format.Data(), irun);
0f389141 881 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
882 if (fOverwriteMode) gGrid->Rm(file);
883 else {
884 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
885 continue;
886 }
a8739e8a 887 }
319593fb 888 // If local collection file does not exist, create it via 'find' command.
d3339be3 889 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 890 command = "find ";
891 command += options;
892 command += path;
893 command += pattern;
894 command += conditions;
895 TGridResult *res = gGrid->Command(command);
896 if (res) delete res;
897 // Write standard output to file
898 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 899 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
900 Bool_t nullFile = kFALSE;
901 if (!hasGrep) {
d3339be3 902 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
903 } else {
a2f5fc01 904 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
905 if (nullFile) {
defd7a3a 906 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
907 continue;
908 }
d3339be3 909 }
a2f5fc01 910 nullResult = kFALSE;
319593fb 911 }
a8739e8a 912 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 913 // Check if there is one run per master job.
914 if (fNrunsPerMaster<2) {
915 if (FileExists(file)) {
0f389141 916 if (fOverwriteMode) gGrid->Rm(file);
917 else {
918 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
919 continue;
920 }
319593fb 921 }
922 // Copy xml file to alien space
923 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
924 if (!FileExists(file)) {
925 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
926 return kFALSE;
927 }
928 } else {
929 nruns++;
95e5b448 930 // Check if the collection for the chunk exist locally.
931 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 932 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
933 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
934 else continue;
935 }
84fcd93f 936 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 937 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 938 schunk = Form(fRunPrefix.Data(), irun);
319593fb 939 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
940 } else {
941 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
942 cbase->Add(cadd);
943 delete cadd;
944 }
74013535 945 format = Form("%%s_%s.xml", fRunPrefix.Data());
946 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 947 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 948 continue;
949 }
904f9f5f 950 schunk = schunk2;
319593fb 951 if (FileExists(schunk)) {
0f389141 952 if (fOverwriteMode) gGrid->Rm(schunk);
953 else {
954 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
955 continue;
956 }
319593fb 957 }
84fcd93f 958 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 959 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 960 if (FileExists(schunk)) {
0f389141 961 if (fOverwriteMode) gGrid->Rm(schunk);
962 else {
963 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
964 continue;
965 }
95e5b448 966 }
319593fb 967 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
968 if (!FileExists(schunk)) {
969 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
970 return kFALSE;
971 }
972 }
c57f56b7 973 }
a2f5fc01 974 if (nullResult) {
d3339be3 975 Error("CreateDataset", "No valid dataset corresponding to the query!");
976 return kFALSE;
977 }
a8739e8a 978 }
c57f56b7 979 return kTRUE;
980}
981
982//______________________________________________________________________________
983Bool_t AliAnalysisAlien::CreateJDL()
984{
985// Generate a JDL file according to current settings. The name of the file is
986// specified by fJDLName.
987 Bool_t error = kFALSE;
988 TObjArray *arr = 0;
989 Bool_t copy = kTRUE;
d3b18c4c 990 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 991 Bool_t generate = kTRUE;
992 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
993 if (!Connect()) {
994 Error("CreateJDL", "Alien connection required");
995 return kFALSE;
996 }
997 // Check validity of alien workspace
d3b18c4c 998 TString workdir;
999 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1000 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1001 workdir += fGridWorkingDir;
1002 if (generate) {
1003 TObjString *os;
1004 if (!fInputFiles) {
1005 Error("CreateJDL()", "Define some input files for your analysis.");
1006 error = kTRUE;
1007 }
1008 // Compose list of input files
1009 // Check if output files were defined
1010 if (!fOutputFiles.Length()) {
1011 Error("CreateJDL", "You must define at least one output file");
1012 error = kTRUE;
1013 }
1014 // Check if an output directory was defined and valid
1015 if (!fGridOutputDir.Length()) {
1016 Error("CreateJDL", "You must define AliEn output directory");
1017 error = kTRUE;
1018 } else {
d3b18c4c 1019 if (!fProductionMode) {
1020 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1021 if (!DirectoryExists(fGridOutputDir)) {
1022 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1023 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1024 } else {
1025 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1026 // error = kTRUE;
1027 }
67a3b254 1028 } else {
1029 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1030 }
1031 gGrid->Cd(workdir);
1032 }
c57f56b7 1033 }
1034 // Exit if any error up to now
1035 if (error) return kFALSE;
1036 // Set JDL fields
0f389141 1037 if (!fUser.IsNull()) {
1038 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1039 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1040 }
1041 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1042 TString mergeExec = fExecutable;
1043 mergeExec.ReplaceAll(".sh", "_merge.sh");
1044 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1045 mergeExec.ReplaceAll(".sh", ".C");
1046 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1047 if (!fArguments.IsNull())
1048 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1049 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1050 else {
1051 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1052 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1053 }
b3e07543 1054
1f0d1ca2 1055 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1056 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1057 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1058 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1059
0f389141 1060 if (fMaxInitFailed > 0) {
c57f56b7 1061 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1062 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1063 }
1064 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1065 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1066 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1067 }
1068 if (!IsOneStageMerging()) {
f47d5cb4 1069 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1070 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1071 }
1072 if (fSplitMode.Length()) {
c57f56b7 1073 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1074 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1075 }
1076 fMergingJDL->SetValue("Split", "\"se\"");
1077 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1078 if (!fAliROOTVersion.IsNull()) {
1079 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1080 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1081 }
1082 if (!fROOTVersion.IsNull()) {
c57f56b7 1083 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1084 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1085 }
1086 if (!fAPIVersion.IsNull()) {
c57f56b7 1087 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1088 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1089 }
648174cf 1090 if (!fExternalPackages.IsNull()) {
1091 arr = fExternalPackages.Tokenize(" ");
1092 TIter next(arr);
1093 while ((os=(TObjString*)next())) {
1094 TString pkgname = os->GetString();
1095 Int_t index = pkgname.Index("::");
1096 TString pkgversion = pkgname(index+2, pkgname.Length());
1097 pkgname.Remove(index);
1098 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1099 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1100 }
1101 delete arr;
1102 }
0f389141 1103 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1104 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1105 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1106 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1107 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1108 TString analysisFile = fExecutable;
1109 analysisFile.ReplaceAll(".sh", ".root");
1110 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1111 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1112 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
1113 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
1114 if (fAdditionalLibs.Length()) {
1115 arr = fAdditionalLibs.Tokenize(" ");
1116 TIter next(arr);
1117 while ((os=(TObjString*)next())) {
1118 if (os->GetString().Contains(".so")) continue;
1119 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1120 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1121 }
1122 delete arr;
1123 }
4e5c5506 1124 if (fPackages) {
1125 TIter next(fPackages);
1126 TObject *obj;
0f389141 1127 while ((obj=next())) {
4e5c5506 1128 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1129 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1130 }
4e5c5506 1131 }
c57f56b7 1132 if (fOutputArchive.Length()) {
1133 arr = fOutputArchive.Tokenize(" ");
1134 TIter next(arr);
0f389141 1135 Bool_t first = kTRUE;
1136 const char *comment = "Files to be archived";
1137 const char *comment1 = comment;
1138 while ((os=(TObjString*)next())) {
1139 if (!first) comment = NULL;
1140 if (!os->GetString().Contains("@") && fCloseSE.Length())
1141 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1142 else
1143 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1144 first = kFALSE;
1145 }
c57f56b7 1146 delete arr;
f790bc1b 1147 // Output archive for the merging jdl
1148 TString outputArchive;
1149 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1150 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1151 // Add normal output files, extra files + terminate files
1152 TString files = GetListOfFiles("outextter");
1153 // Do not register merge excludes
1154 if (!fMergeExcludes.IsNull()) {
1155 arr = fMergeExcludes.Tokenize(" ");
1156 TIter next1(arr);
1157 while ((os=(TObjString*)next1())) {
1158 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1159 files.ReplaceAll(os->GetString(),"");
1160 }
66d71516 1161 delete arr;
0f389141 1162 }
f790bc1b 1163 files.ReplaceAll(".root", "*.root");
b83f84f0 1164 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1165 } else {
95f84779 1166 TString files = fOutputArchive;
1167 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1168 outputArchive = files;
f790bc1b 1169 }
0f389141 1170 arr = outputArchive.Tokenize(" ");
705adb3e 1171 TIter next2(arr);
0f389141 1172 comment = comment1;
1173 first = kTRUE;
705adb3e 1174 while ((os=(TObjString*)next2())) {
0f389141 1175 if (!first) comment = NULL;
0cdf65a8 1176 TString currentfile = os->GetString();
0cdf65a8 1177 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1178 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1179 else
0cdf65a8 1180 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1181 first = kFALSE;
1182 }
1183 delete arr;
c57f56b7 1184 }
149d288c 1185 arr = fOutputFiles.Tokenize(",");
c57f56b7 1186 TIter next(arr);
0f389141 1187 Bool_t first = kTRUE;
f790bc1b 1188 const char *comment = "Files to be saved";
43da816a 1189 while ((os=(TObjString*)next())) {
e1eaf596 1190 // Ignore ouputs in jdl that are also in outputarchive
1191 TString sout = os->GetString();
f790bc1b 1192 sout.ReplaceAll("*", "");
1193 sout.ReplaceAll(".root", "");
e1eaf596 1194 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1195 if (fOutputArchive.Contains(sout)) continue;
0f389141 1196 if (!first) comment = NULL;
43da816a 1197 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1198 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1199 else
0f389141 1200 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1201 first = kFALSE;
f790bc1b 1202 if (fMergeExcludes.Contains(sout)) continue;
1203 if (!os->GetString().Contains("@") && fCloseSE.Length())
1204 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1205 else
1206 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1207 }
c57f56b7 1208 delete arr;
0f389141 1209 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1210 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1211 TString validationScript = fValidationScript;
0f389141 1212 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1213 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1214 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1215 if (fMasterResubmitThreshold) {
1216 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1217 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1218 }
d2a409b2 1219 // Write a jdl with 2 input parameters: collection name and output dir name.
1220 WriteJDL(copy);
c57f56b7 1221 }
1222 // Copy jdl to grid workspace
a8739e8a 1223 if (copy) {
b5e4aaa7 1224 // Check if an output directory was defined and valid
1225 if (!fGridOutputDir.Length()) {
1226 Error("CreateJDL", "You must define AliEn output directory");
1227 return kFALSE;
1228 } else {
1229 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1230 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1231 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1232 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1233 } else {
1234 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1235 return kFALSE;
1236 }
1237 }
1238 gGrid->Cd(workdir);
1239 }
648174cf 1240 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1241 TString mergeJDLName = fExecutable;
1242 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1243 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1244 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1245 if (fProductionMode) {
648174cf 1246 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1247 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1248 }
648174cf 1249 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1250 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1251 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
648174cf 1252 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1253 if (fMergeViaJDL) {
1254 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1255 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1256 }
648174cf 1257 }
c57f56b7 1258 if (fAdditionalLibs.Length()) {
e7c71df0 1259 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1260 TObjString *os;
1261 TIter next(arr);
1262 while ((os=(TObjString*)next())) {
c57f56b7 1263 if (os->GetString().Contains(".so")) continue;
4e5c5506 1264 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1265 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1266 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1267 }
1268 delete arr;
1269 }
4e5c5506 1270 if (fPackages) {
1271 TIter next(fPackages);
1272 TObject *obj;
1273 while ((obj=next())) {
fdbbc7be 1274 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1275 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1276 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1277 }
1278 }
c57f56b7 1279 }
1280 return kTRUE;
1281}
1282
a8739e8a 1283//______________________________________________________________________________
d2a409b2 1284Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1285{
1286// Writes one or more JDL's corresponding to findex. If findex is negative,
1287// all run numbers are considered in one go (jdl). For non-negative indices
1288// they correspond to the indices in the array fInputFiles.
1289 if (!fInputFiles) return kFALSE;
d3b18c4c 1290 TObject *os;
1291 TString workdir;
1292 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1293 workdir += fGridWorkingDir;
f47d5cb4 1294 TString stageName = "$2";
1295 if (fProductionMode) stageName = "$4";
1296 if (!fMergeDirName.IsNull()) {
1297 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1298 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1299 } else {
1300 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1301 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1302 }
d3b18c4c 1303 if (fProductionMode) {
a8739e8a 1304 TIter next(fInputFiles);
b3e07543 1305 while ((os=next())) {
d3b18c4c 1306 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1307 }
d3b18c4c 1308 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
d3b18c4c 1309 } else {
1310 if (!fRunNumbers.Length() && !fRunRange[0]) {
1311 // One jdl with no parameters in case input data is specified by name.
1312 TIter next(fInputFiles);
1313 while ((os=next()))
1314 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1315 if (!fOutputSingle.IsNull())
1316 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1317 else {
1318 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1319 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1320 }
1321 } else {
1322 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1323 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1324 if (!fOutputSingle.IsNull()) {
1325 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1326 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1327 } else {
1328 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1329 }
1330 }
a8739e8a 1331 }
1332
a8739e8a 1333 // Generate the JDL as a string
1334 TString sjdl = fGridJDL->Generate();
0f389141 1335 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1336 // Final merge jdl
1337 if (!fMergeDirName.IsNull()) {
1338 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1339 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1340 } else {
1341 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1342 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1343 }
b3e07543 1344 TString sjdl2 = fMergingJDL->Generate();
1345 Int_t index, index1;
a8739e8a 1346 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1347 sjdl.ReplaceAll("(member", "\n (member");
1348 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1349 sjdl.ReplaceAll("{", "{\n ");
1350 sjdl.ReplaceAll("};", "\n};");
1351 sjdl.ReplaceAll("{\n \n", "{\n");
1352 sjdl.ReplaceAll("\n\n", "\n");
1353 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1354 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1355 sjdl1.ReplaceAll("(member", "\n (member");
1356 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1357 sjdl1.ReplaceAll("{", "{\n ");
1358 sjdl1.ReplaceAll("};", "\n};");
1359 sjdl1.ReplaceAll("{\n \n", "{\n");
1360 sjdl1.ReplaceAll("\n\n", "\n");
1361 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1362 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1363 sjdl2.ReplaceAll("(member", "\n (member");
1364 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1365 sjdl2.ReplaceAll("{", "{\n ");
1366 sjdl2.ReplaceAll("};", "\n};");
1367 sjdl2.ReplaceAll("{\n \n", "{\n");
1368 sjdl2.ReplaceAll("\n\n", "\n");
1369 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1370 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1371 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1372 index = sjdl.Index("JDLVariables");
1373 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1374 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1375 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1376 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1377 index = fJobTag.Index(":");
1378 if (index < 0) index = fJobTag.Length();
1379 TString jobTag = fJobTag;
f47d5cb4 1380 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1381 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1382 if (fProductionMode) {
1383 sjdl1.Prepend("# Generated merging jdl (production mode) \
1384 \n# $1 = full alien path to output directory to be merged \
1385 \n# $2 = train number \
1386 \n# $3 = production (like LHC10b) \
1387 \n# $4 = merging stage \
1388 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1389 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1390 sjdl2.Prepend("# Generated merging jdl \
1391 \n# $1 = full alien path to output directory to be merged \
1392 \n# $2 = train number \
1393 \n# $3 = production (like LHC10b) \
1394 \n# $4 = merging stage \
1395 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1396 } else {
1397 sjdl1.Prepend("# Generated merging jdl \
1398 \n# $1 = full alien path to output directory to be merged \
1399 \n# $2 = merging stage \
03335324 1400 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1401 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1402 sjdl2.Prepend("# Generated merging jdl \
1403 \n# $1 = full alien path to output directory to be merged \
1404 \n# $2 = merging stage \
03335324 1405 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1406 }
0f389141 1407 index = sjdl1.Index("JDLVariables");
1408 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1409 index = sjdl2.Index("JDLVariables");
1410 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1411 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1412 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1413 index = sjdl2.Index("Split =");
1414 if (index>=0) {
1415 index1 = sjdl2.Index("\n", index);
1416 sjdl2.Remove(index, index1-index+1);
1417 }
1418 index = sjdl2.Index("SplitMaxInputFileNumber");
1419 if (index>=0) {
1420 index1 = sjdl2.Index("\n", index);
1421 sjdl2.Remove(index, index1-index+1);
1422 }
1423 index = sjdl2.Index("InputDataCollection");
1424 if (index>=0) {
1425 index1 = sjdl2.Index(";", index);
1426 sjdl2.Remove(index, index1-index+1);
1427 }
1428 index = sjdl2.Index("InputDataListFormat");
1429 if (index>=0) {
1430 index1 = sjdl2.Index("\n", index);
1431 sjdl2.Remove(index, index1-index+1);
1432 }
1433 index = sjdl2.Index("InputDataList");
1434 if (index>=0) {
1435 index1 = sjdl2.Index("\n", index);
1436 sjdl2.Remove(index, index1-index+1);
1437 }
f47d5cb4 1438 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1439 // Write jdl to file
a8739e8a 1440 ofstream out;
d2a409b2 1441 out.open(fJDLName.Data(), ios::out);
a8739e8a 1442 if (out.bad()) {
d3b18c4c 1443 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1444 return kFALSE;
1445 }
1446 out << sjdl << endl;
b3e07543 1447 out.close();
0f389141 1448 TString mergeJDLName = fExecutable;
1449 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1450 if (fMergeViaJDL) {
1451 ofstream out1;
1452 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1453 if (out1.bad()) {
d3b18c4c 1454 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1455 return kFALSE;
1456 }
1457 out1 << sjdl1 << endl;
b3e07543 1458 out1.close();
1459 ofstream out2;
1460 TString finalJDL = mergeJDLName;
1461 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1462 out2.open(finalJDL.Data(), ios::out);
1463 if (out2.bad()) {
1464 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1465 return kFALSE;
1466 }
1467 out2 << sjdl2 << endl;
1468 out2.close();
0f389141 1469 }
a8739e8a 1470
1471 // Copy jdl to grid workspace
1472 if (!copy) {
d3b18c4c 1473 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1474 } else {
d2a409b2 1475 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1476 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1477 TString finalJDL = mergeJDLName;
1478 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1479 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1480 if (fProductionMode) {
b5fe9cba 1481 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1482 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1483 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1484 }
d2a409b2 1485 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1486 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 1487 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 1488 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
5513444a 1489 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1490 if (fMergeViaJDL) {
b3e07543 1491 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
0f389141 1492 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
b3e07543 1493 TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
0f389141 1494 }
a8739e8a 1495 }
1496 return kTRUE;
1497}
1498
c57f56b7 1499//______________________________________________________________________________
5513444a 1500Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1501{
1502// Returns true if file exists.
5513444a 1503 if (!gGrid) return kFALSE;
2ea55496 1504 TString slfn = lfn;
1505 slfn.ReplaceAll("alien://","");
1506 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 1507 if (!res) return kFALSE;
1508 TMap *map = dynamic_cast<TMap*>(res->At(0));
1509 if (!map) {
1510 delete res;
1511 return kFALSE;
1512 }
1513 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1514 if (!objs || !objs->GetString().Length()) {
1515 delete res;
1516 return kFALSE;
1517 }
1518 delete res;
1519 return kTRUE;
1520}
1521
923e2ca5 1522//______________________________________________________________________________
1523Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1524{
1525// Returns true if directory exists. Can be also a path.
1526 if (!gGrid) return kFALSE;
1527 // Check if dirname is a path
1528 TString dirstripped = dirname;
1529 dirstripped = dirstripped.Strip();
1530 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1531 TString dir = gSystem->BaseName(dirstripped);
1532 dir += "/";
1533 TString path = gSystem->DirName(dirstripped);
1534 TGridResult *res = gGrid->Ls(path, "-F");
1535 if (!res) return kFALSE;
1536 TIter next(res);
1537 TMap *map;
1538 TObject *obj;
1539 while ((map=dynamic_cast<TMap*>(next()))) {
1540 obj = map->GetValue("name");
1541 if (!obj) break;
1542 if (dir == obj->GetName()) {
1543 delete res;
1544 return kTRUE;
1545 }
1546 }
1547 delete res;
1548 return kFALSE;
1549}
1550
c57f56b7 1551//______________________________________________________________________________
a2f5fc01 1552void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 1553{
1554// Check input data type.
a2f5fc01 1555 isCollection = kFALSE;
1556 isXml = kFALSE;
1557 useTags = kFALSE;
c57f56b7 1558 if (!gGrid) {
1559 Error("CheckDataType", "No connection to grid");
1560 return;
1561 }
a2f5fc01 1562 isCollection = IsCollection(lfn);
c57f56b7 1563 TString msg = "\n##### file: ";
1564 msg += lfn;
a2f5fc01 1565 if (isCollection) {
c57f56b7 1566 msg += " type: raw_collection;";
1567 // special treatment for collections
a2f5fc01 1568 isXml = kFALSE;
c57f56b7 1569 // check for tag files in the collection
1570 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1571 if (!res) {
1572 msg += " using_tags: No (unknown)";
23329835 1573 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1574 return;
1575 }
1576 const char* typeStr = res->GetKey(0, "origLFN");
1577 if (!typeStr || !strlen(typeStr)) {
1578 msg += " using_tags: No (unknown)";
23329835 1579 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1580 return;
1581 }
1582 TString file = typeStr;
a2f5fc01 1583 useTags = file.Contains(".tag");
1584 if (useTags) msg += " using_tags: Yes";
c57f56b7 1585 else msg += " using_tags: No";
23329835 1586 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1587 return;
1588 }
1589 TString slfn(lfn);
1590 slfn.ToLower();
a2f5fc01 1591 isXml = slfn.Contains(".xml");
1592 if (isXml) {
c57f56b7 1593 // Open xml collection and check if there are tag files inside
1594 msg += " type: xml_collection;";
1595 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1596 if (!coll) {
1597 msg += " using_tags: No (unknown)";
23329835 1598 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1599 return;
1600 }
1601 TMap *map = coll->Next();
1602 if (!map) {
1603 msg += " using_tags: No (unknown)";
23329835 1604 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1605 return;
1606 }
1607 map = (TMap*)map->GetValue("");
1608 TString file;
1609 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 1610 useTags = file.Contains(".tag");
c57f56b7 1611 delete coll;
a2f5fc01 1612 if (useTags) msg += " using_tags: Yes";
c57f56b7 1613 else msg += " using_tags: No";
23329835 1614 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1615 return;
1616 }
a2f5fc01 1617 useTags = slfn.Contains(".tag");
c57f56b7 1618 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 1619 else msg += " type: unknown file;";
a2f5fc01 1620 if (useTags) msg += " using_tags: Yes";
c57f56b7 1621 else msg += " using_tags: No";
23329835 1622 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1623}
1624
4e5c5506 1625//______________________________________________________________________________
1626void AliAnalysisAlien::EnablePackage(const char *package)
1627{
1628// Enables a par file supposed to exist in the current directory.
1629 TString pkg(package);
1630 pkg.ReplaceAll(".par", "");
1631 pkg += ".par";
1632 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 1633 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 1634 return;
1635 }
1636 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1637 Info("EnablePackage", "AliEn plugin will use .par packages");
1638 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1639 if (!fPackages) {
1640 fPackages = new TObjArray();
1641 fPackages->SetOwner();
1642 }
1643 fPackages->Add(new TObjString(pkg));
1644}
1645
3bdcb562 1646//______________________________________________________________________________
1647TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
1648{
1649// Make a tree from files having the location specified in fFileForTestMode.
1650// Inspired from JF's CreateESDChain.
1651 if (fFileForTestMode.IsNull()) {
1652 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
1653 return NULL;
1654 }
1655 if (gSystem->AccessPathName(fFileForTestMode)) {
1656 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
1657 return NULL;
1658 }
1659 // Open the file
1660 ifstream in;
1661 in.open(fFileForTestMode);
1662 Int_t count = 0;
1663 // Read the input list of files and add them to the chain
1664 TString line;
1665 TChain *chain = new TChain(treeName);
1666 while (in.good())
1667 {
1668 in >> line;
1669 if (line.IsNull()) continue;
1670 if (count++ == fNtestFiles) break;
1671 TString esdFile(line);
1672 TFile *file = TFile::Open(esdFile);
1673 if (file) {
1674 if (!file->IsZombie()) chain->Add(esdFile);
1675 file->Close();
1676 } else {
1677 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
1678 }
1679 }
1680 in.close();
1681 if (!chain->GetListOfFiles()->GetEntries()) {
1682 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
1683 delete chain;
1684 return NULL;
1685 }
1686// chain->ls();
1687 return chain;
1688}
1689
d2a409b2 1690//______________________________________________________________________________
1691const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1692{
1693// Get job status for all jobs with jobid>jobidstart.
1694 static char mstatus[20];
1695 mstatus[0] = '\0';
1696 nrunning = 0;
1697 nwaiting = 0;
1698 nerror = 0;
1699 ndone = 0;
1700 TGridJobStatusList *list = gGrid->Ps("");
1701 if (!list) return mstatus;
1702 Int_t nentries = list->GetSize();
1703 TGridJobStatus *status;
1704 Int_t pid;
1705 for (Int_t ijob=0; ijob<nentries; ijob++) {
1706 status = (TGridJobStatus *)list->At(ijob);
8ddad121 1707 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 1708 if (pid<jobidstart) continue;
1709 if (pid == lastid) {
8ddad121 1710 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 1711 }
1712 switch (status->GetStatus()) {
1713 case TGridJobStatus::kWAITING:
1714 nwaiting++; break;
1715 case TGridJobStatus::kRUNNING:
1716 nrunning++; break;
1717 case TGridJobStatus::kABORTED:
1718 case TGridJobStatus::kFAIL:
1719 case TGridJobStatus::kUNKNOWN:
1720 nerror++; break;
1721 case TGridJobStatus::kDONE:
1722 ndone++;
1723 }
1724 }
1725 list->Delete();
1726 delete list;
1727 return mstatus;
1728}
1729
c57f56b7 1730//______________________________________________________________________________
1731Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1732{
1733// Returns true if file is a collection. Functionality duplicated from
1734// TAlien::Type() because we don't want to directly depend on TAlien.
1735 if (!gGrid) {
1736 Error("IsCollection", "No connection to grid");
1737 return kFALSE;
1738 }
1739 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1740 if (!res) return kFALSE;
1741 const char* typeStr = res->GetKey(0, "type");
1742 if (!typeStr || !strlen(typeStr)) return kFALSE;
1743 if (!strcmp(typeStr, "collection")) return kTRUE;
1744 delete res;
1745 return kFALSE;
1746}
1747
fe2d7fc2 1748//______________________________________________________________________________
1749Bool_t AliAnalysisAlien::IsSingleOutput() const
1750{
1751// Check if single-ouput option is on.
1752 return (!fOutputSingle.IsNull());
1753}
1754
16a4353c 1755//______________________________________________________________________________
1756void AliAnalysisAlien::Print(Option_t *) const
1757{
1758// Print current plugin settings.
84fcd93f 1759 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 1760 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1761 if (mgr && mgr->IsProofMode()) {
1762 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
1763 if (TestBit(AliAnalysisGrid::kTest))
1764 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
1765 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
1766 if (!fProofDataSet.IsNull())
1767 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
1768 if (fProofReset==1)
1769 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1770 if (fProofReset>1)
1771 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1772 if (!fRootVersionForProof.IsNull())
1773 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
1774 else
1775 printf("= ROOT version requested________________________ default\n");
1776 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
1777 if (!fAliRootMode.IsNull())
1778 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
1779 if (fNproofWorkers)
1780 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
1781 if (fNproofWorkersPerSlave)
1782 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
1783 if (TestSpecialBit(kClearPackages))
1784 printf("= ClearPackages requested...\n");
1785 if (fIncludePath.Data())
1786 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1787 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1788 if (fPackages && fPackages->GetEntries()) {
1789 TIter next(fPackages);
1790 TObject *obj;
1791 TString list;
1792 while ((obj=next())) list += obj->GetName();
1793 printf("= Par files to be used: ________________________ %s\n", list.Data());
1794 }
1795 if (TestSpecialBit(kProofConnectGrid))
1796 printf("= Requested PROOF connection to grid\n");
1797 return;
1798 }
e1c22e21 1799 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
1800 if (fOverwriteMode) {
1801 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
1802 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
1803 }
348be253 1804 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
1805 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 1806 printf("= Production mode:______________________________ %d\n", fProductionMode);
1807 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1808 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1809 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1810 if (fUser.Length())
84fcd93f 1811 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1812 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1813 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1814 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1815 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1816 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1817 if (fRunNumbers.Length())
84fcd93f 1818 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1819 if (fRunRange[0])
74013535 1820 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 1821 if (!fRunRange[0] && !fRunNumbers.Length()) {
1822 TIter next(fInputFiles);
1823 TObject *obj;
1824 TString list;
1825 while ((obj=next())) list += obj->GetName();
84fcd93f 1826 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1827 }
1828 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1829 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1830 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1831 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1832 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
d3b18c4c 1833 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 1834 printf("=====================================================================\n");
1835 printf("= Job price: ___________________________________ %d\n", fPrice);
1836 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1837 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1838 if (fMaxInitFailed>0)
84fcd93f 1839 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1840 if (fMasterResubmitThreshold>0)
84fcd93f 1841 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 1842 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 1843 if (fNrunsPerMaster>0)
84fcd93f 1844 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1845 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1846 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1847 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1848 if (fArguments.Length())
84fcd93f 1849 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1850 if (fExecutableArgs.Length())
1851 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1852 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1853 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1854 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1855 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1856 if (fDatasetName)
84fcd93f 1857 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1858 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1859 if (fIncludePath.Data())
84fcd93f 1860 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1861 if (fCloseSE.Length())
84fcd93f 1862 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1863 if (fFriendChainName.Length())
84fcd93f 1864 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 1865 if (fPackages && fPackages->GetEntries()) {
16a4353c 1866 TIter next(fPackages);
1867 TObject *obj;
1868 TString list;
1869 while ((obj=next())) list += obj->GetName();
84fcd93f 1870 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1871 }
1872}
1873
c57f56b7 1874//______________________________________________________________________________
1875void AliAnalysisAlien::SetDefaults()
1876{
1877// Set default values for everything. What cannot be filled will be left empty.
1878 if (fGridJDL) delete fGridJDL;
1879 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 1880 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 1881 fPrice = 1;
1882 fTTL = 30000;
1883 fSplitMaxInputFileNumber = 100;
1884 fMaxInitFailed = 0;
1885 fMasterResubmitThreshold = 0;
bb885a9e 1886 fNtestFiles = 10;
149d288c 1887 fNreplicas = 2;
a8739e8a 1888 fRunRange[0] = 0;
1889 fRunRange[1] = 0;
8e3bd06a 1890 fRunPrefix = "%d";
319593fb 1891 fNrunsPerMaster = 1;
16a4353c 1892 fMaxMergeFiles = 100;
c57f56b7 1893 fRunNumbers = "";
1894 fExecutable = "analysis.sh";
0a1c1f7f 1895 fExecutableCommand = "root -b -q";
c57f56b7 1896 fArguments = "";
631c0b05 1897 fExecutableArgs = "";
c57f56b7 1898 fAnalysisMacro = "myAnalysis.C";
1899 fAnalysisSource = "";
1900 fAdditionalLibs = "";
1901 fSplitMode = "se";
1902 fAPIVersion = "";
1903 fROOTVersion = "";
1904 fAliROOTVersion = "";
1905 fUser = ""; // Your alien user name
1906 fGridWorkingDir = "";
1907 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1908 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1909 fFriendChainName = "";
c57f56b7 1910 fGridOutputDir = "output";
b83f84f0 1911 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 1912 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1913 fInputFormat = "xml-single";
1914 fJDLName = "analysis.jdl";
c6cb3634 1915 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1916 fMergeExcludes = "";
0f389141 1917 fMergeViaJDL = 0;
348be253 1918 SetUseCopy(kTRUE);
1919 SetCheckCopy(kTRUE);
149d288c 1920 SetDefaultOutputs(kTRUE);
e1c22e21 1921 fOverwriteMode = 1;
c57f56b7 1922}
1923
0f389141 1924//______________________________________________________________________________
b3e07543 1925Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 1926{
b3e07543 1927// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
1928 // First check if the result is already in the output directory.
1929 if (FileExists(Form("%s/%s",aliendir,filename))) {
1930 printf("Final merged results found. Not merging again.\n");
7c2cd90a 1931 return kFALSE;
1932 }
b3e07543 1933 // Now check the last stage done.
1934 Int_t stage = 0;
1935 while (1) {
1936 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
1937 stage++;
1938 }
1939 // Next stage of merging
1940 stage++;
1941 TString pattern = "*root_archive.zip";
1942 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
1943 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
1944 if (res) delete res;
1945 // Write standard output to file
1946 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
1947 // Count the number of files inside
1948 ifstream ifile;
1949 ifile.open(Form("Stage_%d.xml",stage));
1950 if (!ifile.good()) {
1951 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 1952 return kFALSE;
b3e07543 1953 }
1954 TString line;
1955 Int_t nfiles = 0;
1956 while (!ifile.eof()) {
1957 ifile >> line;
1958 if (line.Contains("/event")) nfiles++;
7c2cd90a 1959 }
b3e07543 1960 ifile.close();
1961 if (!nfiles) {
1962 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
1963 return kFALSE;
1964 } else {
1965 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 1966 }
b3e07543 1967 // Copy the file in the output directory
1968 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
1969 TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
1970 // Check if this is the last stage to be done.
1971 Bool_t laststage = (nfiles<nperchunk);
1972 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
1973 if (laststage) {
1974 printf("### Submiting final merging stage %d\n", stage);
1975 TString finalJDL = jdl;
1976 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 1977 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 1978 Int_t jobId = SubmitSingleJob(query);
1979 if (!jobId) return kFALSE;
1980 } else {
1981 printf("### Submiting merging stage %d\n", stage);
03335324 1982 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 1983 Int_t jobId = SubmitSingleJob(query);
1984 if (!jobId) return kFALSE;
7c2cd90a 1985 }
b3e07543 1986 return kTRUE;
1987}
7c2cd90a 1988
1989//______________________________________________________________________________
1990Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
1991{
1992// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
1993 if (!gGrid) return 0;
1994 printf("=> %s ------> ",query);
1995 TGridResult *res = gGrid->Command(query);
1996 if (!res) return 0;
1997 TString jobId = res->GetKey(0,"jobId");
1998 delete res;
1999 if (jobId.IsNull()) {
2000 printf("submission failed. Reason:\n");
2001 gGrid->Stdout();
2002 gGrid->Stderr();
2003 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2004 return 0;
2005 }
2006 printf(" Job id: %s\n", jobId.Data());
2007 return atoi(jobId);
2008}
2009
2010//______________________________________________________________________________
b3e07543 2011Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2012{
b3e07543 2013// Merge given output files from basedir. Basedir can be an alien output directory
2014// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2015// files in a group (ignored for xml input). Merging can be done in stages:
2016// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2017// stage=1 : works with an xml of all root_archive.zip in the output directory
2018// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2019 TString outputFile = output;
0f389141 2020 TString command;
a2f5fc01 2021 TString outputChunk;
2022 TString previousChunk = "";
b3e07543 2023 TObjArray *listoffiles = new TObjArray();
2024// listoffiles->SetOwner();
a2f5fc01 2025 Int_t countChunk = 0;
2026 Int_t countZero = nmaxmerge;
0f389141 2027 Bool_t merged = kTRUE;
a2f5fc01 2028 Int_t index = outputFile.Index("@");
2029 if (index > 0) outputFile.Remove(index);
7c2cd90a 2030 TString inputFile = outputFile;
b3e07543 2031 TString sbasedir = basedir;
2032 if (sbasedir.Contains(".xml")) {
2033 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2034 nmaxmerge = 9999999;
2035 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2036 if (!coll) {
2037 ::Error("MergeOutput", "Input XML collection empty.");
2038 return kFALSE;
2039 }
2040 // Iterate grid collection
2041 while (coll->Next()) {
2042 TString fname = gSystem->DirName(coll->GetTURL());
2043 fname += "/";
2044 fname += inputFile;
2045 listoffiles->Add(new TNamed(fname.Data(),""));
2046 }
2047 } else {
2048 command = Form("find %s/ *%s", basedir, inputFile.Data());
2049 printf("command: %s\n", command.Data());
2050 TGridResult *res = gGrid->Command(command);
2051 if (!res) {
2052 ::Error("MergeOutput","No result for the find command\n");
2053 delete listoffiles;
2054 return kFALSE;
2055 }
2056 TIter nextmap(res);
2057 TMap *map = 0;
2058 while ((map=(TMap*)nextmap())) {
2059 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2060 if (!objs || !objs->GetString().Length()) {
2061 // Nothing found - skip this output
2062 delete res;
2063 delete listoffiles;
2064 return kFALSE;
2065 }
2066 listoffiles->Add(new TNamed(objs->GetName(),""));
2067 }
2068 delete res;
2069 }
2070 if (!listoffiles->GetEntries()) {
7c2cd90a 2071 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2072 delete listoffiles;
0f389141 2073 return kFALSE;
2074 }
2075
2076 TFileMerger *fm = 0;
b3e07543 2077 TIter next0(listoffiles);
2078 TObjArray *listoffilestmp = new TObjArray();
2079 listoffilestmp->SetOwner();
2080 TObject *nextfile;
2081 TString snextfile;
2082 // Keep only the files at upper level
2083 Int_t countChar = 0;
2084 while ((nextfile=next0())) {
2085 snextfile = nextfile->GetName();
2086 Int_t crtCount = snextfile.CountChar('/');
2087 if (nextfile == listoffiles->First()) countChar = crtCount;
2088 if (crtCount < countChar) countChar = crtCount;
2089 }
2090 next0.Reset();
2091 while ((nextfile=next0())) {
2092 snextfile = nextfile->GetName();
2093 Int_t crtCount = snextfile.CountChar('/');
2094 if (crtCount > countChar) {
2095 delete nextfile;
2096 continue;
2097 }
2098 listoffilestmp->Add(nextfile);
2099 }
2100 delete listoffiles;
2101 listoffiles = listoffilestmp; // Now contains 'good' files
2102 listoffiles->Print();
2103 TIter next(listoffiles);
7c2cd90a 2104 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2105 outputChunk = outputFile;
2106 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2107 // Check for existent temporary merge files
e1c22e21 2108 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2109 // Preserve old merging functionality for stage 0.
2110 if (stage==0) {
2111 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2112 while (1) {
2113 // Skip as many input files as in a chunk
2ea55496 2114 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2115 nextfile = next();
2116 if (!nextfile) {
2ea55496 2117 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2118 delete listoffiles;
2ea55496 2119 return kFALSE;
2120 }
b3e07543 2121 snextfile = nextfile->GetName();
7c2cd90a 2122 }
2123 outputChunk = outputFile;
2124 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2125 countChunk++;
2126 if (gSystem->AccessPathName(outputChunk)) continue;
2127 // Merged file with chunks up to <countChunk> found
2128 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2129 previousChunk = outputChunk;
2130 break;
2131 }
2132 }
2133 countZero = nmaxmerge;
2134
b3e07543 2135 while ((nextfile=next())) {
2136 snextfile = nextfile->GetName();
2ea55496 2137 // Loop 'find' results and get next LFN
7c2cd90a 2138 if (countZero == nmaxmerge) {
2139 // First file in chunk - create file merger and add previous chunk if any.
2140 fm = new TFileMerger(kFALSE);
2141 fm->SetFastMethod(kTRUE);
2142 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2143 outputChunk = outputFile;
2144 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2145 }
2146 // If last file found, put merged results in the output file
b3e07543 2147 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2148 // Add file to be merged and decrement chunk counter.
b3e07543 2149 fm->AddFile(snextfile);
7c2cd90a 2150 countZero--;
b3e07543 2151 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2152 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2153 // Nothing found - skip this output
2154 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2155 merged = kFALSE;
2156 break;
7c2cd90a 2157 }
2158 fm->OutputFile(outputChunk);
2159 // Merge the outputs, then go to next chunk
2160 if (!fm->Merge()) {
2161 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2162 merged = kFALSE;
2163 break;
7c2cd90a 2164 } else {
2165 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2166 gSystem->Unlink(previousChunk);
2167 }
b3e07543 2168 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2169 countChunk++;
2170 countZero = nmaxmerge;
2171 previousChunk = outputChunk;
0f389141 2172 }
0f389141 2173 }
b3e07543 2174 delete listoffiles;
2175 delete fm;
7c2cd90a 2176 return merged;
2177 }
2178 // Merging stage different than 0.
2179 // Move to the begining of the requested chunk.
7c2cd90a 2180 fm = new TFileMerger(kFALSE);
2181 fm->SetFastMethod(kTRUE);
b3e07543 2182 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2183 delete listoffiles;
7c2cd90a 2184 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2185 // Nothing found - skip this output
2186 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2187 delete fm;
2188 return kFALSE;
2189 }
b3e07543 2190 fm->OutputFile(outputFile);
7c2cd90a 2191 // Merge the outputs
2192 if (!fm->Merge()) {
2193 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2194 delete fm;
2195 return kFALSE;
2196 } else {
b3e07543 2197 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2198 }
7c2cd90a 2199 delete fm;
2200 return kTRUE;
0f389141 2201}
2202
c57f56b7 2203//______________________________________________________________________________
2204Bool_t AliAnalysisAlien::MergeOutputs()
2205{
2206// Merge analysis outputs existing in the AliEn space.
2207 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2208 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2209 if (!Connect()) {
2210 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2211 return kFALSE;
0f389141 2212 }
c9e8f7fd 2213 if (fMergeViaJDL) {
2214 if (!TestBit(AliAnalysisGrid::kMerge)) {
2215 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2216 return kFALSE;
2217 }
2218 if (fProductionMode) {
2219 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2220 return kFALSE;
2221 }
0f389141 2222 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2223 if (!SubmitMerging()) return kFALSE;
0f389141 2224 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2225 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2226 return kFALSE;
c57f56b7 2227 }
2228 // Get the output path
d2a409b2 2229 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2230 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2231 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2232 return kFALSE;
2233 }
2234 if (!fOutputFiles.Length()) {
2235 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2236 return kFALSE;
0f389141 2237 }
2238 // Check if fast read option was requested
e1c22e21 2239 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2240 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2241 if (fFastReadOption) {
d3339be3 2242 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2243 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2244 gEnv->SetValue("XNet.ConnectTimeout",50);
2245 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2246 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2247 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2248 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2249 }
e8b839ab 2250 // Make sure we change the temporary directory
2251 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2252 // Set temporary compilation directory to current one
2253 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2254 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2255 TIter next(list);
2256 TObjString *str;
a2f5fc01 2257 TString outputFile;
c57f56b7 2258 Bool_t merged = kTRUE;
2259 while((str=(TObjString*)next())) {
a2f5fc01 2260 outputFile = str->GetString();
2261 Int_t index = outputFile.Index("@");
2262 if (index > 0) outputFile.Remove(index);
2263 TString outputChunk = outputFile;
2264 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2265 // Skip already merged outputs
a2f5fc01 2266 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2267 if (fOverwriteMode) {
a2f5fc01 2268 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2269 gSystem->Unlink(outputFile);
2270 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2271 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2272 outputChunk.Data());
2273 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2274 }
2275 } else {
a2f5fc01 2276 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2277 continue;
2278 }
2279 } else {
a2f5fc01 2280 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2281 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2282 outputChunk.Data());
2283 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2284 }
2285 }
bb885a9e 2286 if (fMergeExcludes.Length() &&
a2f5fc01 2287 fMergeExcludes.Contains(outputFile.Data())) continue;
16a4353c 2288 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2289 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2290 if (!merged) {
2291 Error("MergeOutputs", "Terminate() will NOT be executed");
2292 return kFALSE;
ff07ec61 2293 }
2294 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2295 if (fileOpened) fileOpened->Close();
c57f56b7 2296 }
0f389141 2297 return kTRUE;
c57f56b7 2298}
2299
bb885a9e 2300//______________________________________________________________________________
2301void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2302{
2303// Use the output files connected to output containers from the analysis manager
2304// rather than the files defined by SetOutputFiles
2305 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2306 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2307 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2308}
2309
149d288c 2310//______________________________________________________________________________
2311void AliAnalysisAlien::SetOutputFiles(const char *list)
2312{
2313// Manually set the output files list.
2314// Removes duplicates. Not allowed if default outputs are not disabled.
2315 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2316 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2317 return;
2318 }
2319 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2320 fOutputFiles = "";
2321 TString slist = list;
2322 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2323 TObjArray *arr = slist.Tokenize(" ");
2324 TObjString *os;
2325 TIter next(arr);
2326 TString sout;
2327 while ((os=(TObjString*)next())) {
2328 sout = os->GetString();
2329 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2330 if (fOutputFiles.Contains(sout)) continue;
2331 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2332 fOutputFiles += sout;
2333 }
2334 delete arr;
f790bc1b 2335}
149d288c 2336
2337//______________________________________________________________________________
2338void AliAnalysisAlien::SetOutputArchive(const char *list)
2339{
2340// Manually set the output archive list. Free text - you are on your own...
2341// Not allowed if default outputs are not disabled.
2342 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2343 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2344 return;
2345 }
2346 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2347 fOutputArchive = list;
2348}
2349
2350//______________________________________________________________________________
2351void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2352{
2353// Setting a prefered output SE is not allowed anymore.
2354 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2355}
2356
5513444a 2357//______________________________________________________________________________
2358Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2359{
2360// Start remote grid analysis.
3bdcb562 2361 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2362 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2363 if (!mgr || !mgr->IsInitialized()) {
2364 Error("StartAnalysis", "You need an initialized analysis manager for this");
2365 return kFALSE;
2366 }
2367 // Are we in PROOF mode ?
2368 if (mgr->IsProofMode()) {
90d50a8c 2369 Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2370 if (fProofCluster.IsNull()) {
2371 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2372 return kFALSE;
2373 }
2374 if (fProofDataSet.IsNull() && !testMode) {
2375 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2376 return kFALSE;
2377 }
2378 // Set the needed environment
2379 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2380 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2381 if (fProofReset && !testMode) {
2382 if (fProofReset==1) {
2383 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2384 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2385 } else {
2386 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2387 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2388 }
2389 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2390 return kFALSE;
2391 }
2392 // Do we need to change the ROOT version ? The success of this cannot be checked.
2393 if (!fRootVersionForProof.IsNull() && !testMode) {
2394 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2395 fProofCluster.Data(), fRootVersionForProof.Data()));
2396 }
2397 // Connect to PROOF and check the status
2398 Long_t proof = 0;
27734f0e 2399 TString sworkers;
2400 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2401 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2402 if (!testMode) {
27734f0e 2403 if (!sworkers.IsNull())
2404 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2405 else
2406 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2407 } else {
2408 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2409 if (!proof) {
2410 Error("StartAnalysis", "Could not start PROOF in test mode");
2411 return kFALSE;
2412 }
2413 }
2414 if (!proof) {
2415 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2416 return kFALSE;
2417 }
27734f0e 2418 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2419 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
3bdcb562 2420 // Is dataset existing ?
2421 if (!testMode) {
2422 TString dataset = fProofDataSet;
2423 Int_t index = dataset.Index("#");
2424 if (index>=0) dataset.Remove(index);
830acc4c 2425// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
2426// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
2427// return kFALSE;
2428// }
2429// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 2430 }
2431 // Is ClearPackages() needed ?
2432 if (TestSpecialBit(kClearPackages)) {
2433 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
2434 gROOT->ProcessLine("gProof->ClearPackages();");
2435 }
2436 // Is a given aliroot mode requested ?
2437 TList optionsList;
1ac846e9 2438 TString parLibs;
830acc4c 2439 if (!fAliRootMode.IsNull()) {
3bdcb562 2440 TString alirootMode = fAliRootMode;
2441 if (alirootMode == "default") alirootMode = "";
2442 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
2443 optionsList.SetOwner();
2444 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
2445 // Check the additional libs to be loaded
2446 TString extraLibs;
1ac846e9 2447 Bool_t parMode = kFALSE;
3bdcb562 2448 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:ANALYSISalice";
2449 // Parse the extra libs for .so
2450 if (fAdditionalLibs.Length()) {
2451 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2452 TIter next(list);
2453 TObjString *str;
1ac846e9 2454 while((str=(TObjString*)next())) {
2455 if (str->GetString().Contains(".so")) {
2456 if (parMode) {
2457 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
2458 break;
2459 }
2460 TString stmp = str->GetName();
2461 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
2462 stmp.ReplaceAll(".so","");
2463 if (!extraLibs.IsNull()) extraLibs += ":";
2464 extraLibs += stmp;
2465 continue;
2466 }
2467 if (str->GetString().Contains(".par")) {
2468 // The first par file found in the list will not allow any further .so
2469 parMode = kTRUE;
2470 if (!parLibs.IsNull()) parLibs += ":";
2471 parLibs += str->GetName();
2472 continue;
2473 }
3bdcb562 2474 }
2475 if (list) delete list;
2476 }
2477 if (!extraLibs.IsNull()) optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
830acc4c 2478 // Check extra includes
2479 if (!fIncludePath.IsNull()) {
2480 TString includePath = fIncludePath;
2481 includePath.ReplaceAll(" ",":");
8ddad121 2482 includePath.ReplaceAll("$ALICE_ROOT","");
2483 includePath.ReplaceAll("${ALICE_ROOT}","");
2484 includePath.ReplaceAll("-I","");
830acc4c 2485 includePath.Strip(TString::kTrailing, ':');
2486 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
2487 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
2488 }
27734f0e 2489 // Check if connection to grid is requested
2490 if (TestSpecialBit(kProofConnectGrid))
2491 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 2492 // Enable AliRoot par
2493 if (testMode) {
2494 // Enable proof lite package
2495 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
2496 for (Int_t i=0; i<optionsList.GetSize(); i++) {
2497 TNamed *obj = (TNamed*)optionsList.At(i);
2498 printf("%s %s\n", obj->GetName(), obj->GetTitle());
2499 }
2500 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 2501 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 2502 Info("StartAnalysis", "AliRootProofLite enabled");
2503 } else {
2504 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
2505 return kFALSE;
2506 }
2507 } else {
8ddad121 2508 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
2509 fAliROOTVersion.Data(), &optionsList))) {
830acc4c 2510 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
2511 return kFALSE;
2512 }
27734f0e 2513 }
1ac846e9 2514 // Enable first par files from fAdditionalLibs
2515 if (!parLibs.IsNull()) {
2516 TObjArray *list = parLibs.Tokenize(":");
2517 TIter next(list);
2518 TObjString *package;
2519 while((package=(TObjString*)next())) {
40501bb7 2520 TString spkg = package->GetName();
2521 spkg.ReplaceAll(".par", "");
2522 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 2523 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 2524 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
2525 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 2526 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2527 return kFALSE;
2528 }
2529 } else {
2530 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2531 return kFALSE;
2532 }
2533 }
2534 if (list) delete list;
2535 }
3bdcb562 2536 } else {
2537 if (fAdditionalLibs.Contains(".so") && !testMode) {
2538 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
2539 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
2540 return kFALSE;
2541 }
2542 }
2543 // Enable par files if requested
2544 if (fPackages && fPackages->GetEntries()) {
2545 TIter next(fPackages);
2546 TObject *package;
2547 while ((package=next())) {
1ac846e9 2548 // Skip packages already enabled
2549 if (parLibs.Contains(package->GetName())) continue;
40501bb7 2550 TString spkg = package->GetName();
2551 spkg.ReplaceAll(".par", "");
2552 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
3bdcb562 2553 if (gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 2554 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 2555 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2556 return kFALSE;
2557 }
2558 } else {
2559 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2560 return kFALSE;
2561 }
2562 }
2563 }
2564 // Do we need to load analysis source files ?
2565 // NOTE: don't load on client since this is anyway done by the user to attach his task.
2566 if (fAnalysisSource.Length()) {
2567 TObjArray *list = fAnalysisSource.Tokenize(" ");
2568 TIter next(list);
2569 TObjString *str;
2570 while((str=(TObjString*)next())) {
2571 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
2572 }
2573 if (list) delete list;
2574 }
830acc4c 2575 if (testMode) {
2576 // Register dataset to proof lite.
2577 if (fFileForTestMode.IsNull()) {
2578 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2579 return kFALSE;
2580 }
2581 if (gSystem->AccessPathName(fFileForTestMode)) {
2582 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2583 return kFALSE;
2584 }
2585 TFileCollection *coll = new TFileCollection();
2586 coll->AddFromFile(fFileForTestMode);
8ddad121 2587 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 2588 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 2589 }
3bdcb562 2590 return kTRUE;
2591 }
c57f56b7 2592
43da816a 2593 // Check if output files have to be taken from the analysis manager
2594 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 2595 // Add output files and AOD files
2596 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 2597 // Add extra files registered to the analysis manager
f790bc1b 2598 TString extra = GetListOfFiles("ext");
2599 if (!extra.IsNull()) {
149d288c 2600 extra.ReplaceAll(".root", "*.root");
f790bc1b 2601 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 2602 fOutputFiles += extra;
84fcd93f 2603 }
149d288c 2604 // Compose the output archive.
b83f84f0 2605 fOutputArchive = "log_archive.zip:std*@disk=1 ";
2606 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 2607 }
f7b1cbc2 2608// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 2609 if (TestBit(AliAnalysisGrid::kOffline)) {
2610 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
2611 \n there nor any job run. You can revise the JDL and analysis \
2612 \n macro then run the same in \"submit\" mode.");
2613 } else if (TestBit(AliAnalysisGrid::kTest)) {
2614 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
2615 \n dataset.");
2616 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
2617 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
2618 \n space and job submitted.");
2619 } else if (TestBit(AliAnalysisGrid::kMerge)) {
2620 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 2621 if (fMergeViaJDL) CheckInputData();
5513444a 2622 return kTRUE;
c57f56b7 2623 } else {
2624 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
2625 }
2626
348be253 2627 Print();
c57f56b7 2628 if (!Connect()) {
2629 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 2630 return kFALSE;
16a4353c 2631 }
d3b18c4c 2632 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 2633 if (!CheckInputData()) {
2634 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 2635 return kFALSE;
c57f56b7 2636 }
d3339be3 2637 if (!CreateDataset(fDataPattern)) {
2638 TString serror;
2639 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
2640 if (fRunNumbers.Length()) serror = "run numbers";
2641 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
2642 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
2643 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
2644 return kFALSE;
2645 }
c57f56b7 2646 WriteAnalysisFile();
2647 WriteAnalysisMacro();
2648 WriteExecutable();
2649 WriteValidationScript();
0f389141 2650 if (fMergeViaJDL) {
2651 WriteMergingMacro();
2652 WriteMergeExecutable();
2653 WriteValidationScript(kTRUE);
2654 }
5513444a 2655 if (!CreateJDL()) return kFALSE;
2656 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 2657 if (testMode) {
c57f56b7 2658 // Locally testing the analysis
2659 Info("StartAnalysis", "\n_______________________________________________________________________ \
2660 \n Running analysis script in a daughter shell as on a worker node \
2661 \n_______________________________________________________________________");
149d288c 2662 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2663 TIter next(list);
2664 TObjString *str;
a2f5fc01 2665 TString outputFile;
c57f56b7 2666 while((str=(TObjString*)next())) {
a2f5fc01 2667 outputFile = str->GetString();
2668 Int_t index = outputFile.Index("@");
2669 if (index > 0) outputFile.Remove(index);
2670 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 2671 }
2672 delete list;
2673 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 2674 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 2675// gSystem->Exec("cat stdout");
5513444a 2676 return kFALSE;
c57f56b7 2677 }
5513444a 2678 // Check if submitting is managed by LPM manager
a3e84053 2679 if (fProductionMode) {
5513444a 2680 TString prodfile = fJDLName;
2681 prodfile.ReplaceAll(".jdl", ".prod");
2682 WriteProductionFile(prodfile);
2683 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
2684 return kFALSE;
2685 }
a8739e8a 2686 // Submit AliEn job(s)
d2a409b2 2687 gGrid->Cd(fGridOutputDir);
a8739e8a 2688 TGridResult *res;
c57f56b7 2689 TString jobID = "";
d2a409b2 2690 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 2691 // Submit a given xml or a set of runs
a8739e8a 2692 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 2693 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 2694 if (res) {
2695 const char *cjobId = res->GetKey(0,"jobId");
2696 if (!cjobId) {
a03be957 2697 gGrid->Stdout();
2698 gGrid->Stderr();
a8739e8a 2699 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 2700 return kFALSE;
a8739e8a 2701 } else {
2702 Info("StartAnalysis", "\n_______________________________________________________________________ \
2703 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
2704 \n_______________________________________________________________________",
2705 fJDLName.Data(), cjobId);
2706 jobID = cjobId;
2707 }
2708 delete res;
a03be957 2709 } else {
2710 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2711 return kFALSE;
a8739e8a 2712 }
2713 } else {
d2a409b2 2714 // Submit for a range of enumeration of runs.
a03be957 2715 if (!Submit()) return kFALSE;
c57f56b7 2716 }
a8739e8a 2717
c57f56b7 2718 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
2719 \n You may exit at any time and terminate the job later using the option <terminate> \
2720 \n ##################################################################################", jobID.Data());
bb885a9e 2721 gSystem->Exec("aliensh");
5513444a 2722 return kTRUE;
c57f56b7 2723}
2724
f790bc1b 2725//______________________________________________________________________________
2726const char *AliAnalysisAlien::GetListOfFiles(const char *type)
2727{
2728// Get a comma-separated list of output files of the requested type.
2729// Type can be (case unsensitive):
2730// aod - list of aod files (std, extensions and filters)
2731// out - list of output files connected to containers (but not aod's or extras)
2732// ext - list of extra files registered to the manager
2733// ter - list of files produced in terminate
2734 static TString files;
2735 files = "";
2736 TString stype = type;
2737 stype.ToLower();
2738 TString aodfiles, extra;
2739 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2740 if (!mgr) {
2741 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
2742 return files.Data();
2743 }
2744 if (mgr->GetOutputEventHandler()) {
2745 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
2746 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
2747 if (!extraaod.IsNull()) {
2748 aodfiles += ",";
2749 aodfiles += extraaod;
2750 }
2751 }
2752 if (stype.Contains("aod")) {
2753 files = aodfiles;
2754 if (stype == "aod") return files.Data();
2755 }
2756 // Add output files that are not in the list of AOD files
2757 TString outputfiles = "";
2758 TIter next(mgr->GetOutputs());
2759 AliAnalysisDataContainer *output;
2760 const char *filename = 0;
2761 while ((output=(AliAnalysisDataContainer*)next())) {
2762 filename = output->GetFileName();
2763 if (!(strcmp(filename, "default"))) continue;
2764 if (outputfiles.Contains(filename)) continue;
2765 if (aodfiles.Contains(filename)) continue;
2766 if (!outputfiles.IsNull()) outputfiles += ",";
2767 outputfiles += filename;
2768 }
2769 if (stype.Contains("out")) {
2770 if (!files.IsNull()) files += ",";
2771 files += outputfiles;
2772 if (stype == "out") return files.Data();
2773 }
2774 // Add extra files registered to the analysis manager
2775 TString sextra;
2776 extra = mgr->GetExtraFiles();
2777 if (!extra.IsNull()) {
2778 extra.Strip();
2779 extra.ReplaceAll(" ", ",");
2780 TObjArray *fextra = extra.Tokenize(",");
2781 TIter nextx(fextra);
2782 TObject *obj;
2783 while ((obj=nextx())) {
2784 if (aodfiles.Contains(obj->GetName())) continue;
2785 if (outputfiles.Contains(obj->GetName())) continue;
2786 if (sextra.Contains(obj->GetName())) continue;
2787 if (!sextra.IsNull()) sextra += ",";
2788 sextra += obj->GetName();
2789 }
2790 delete fextra;
2791 if (stype.Contains("ext")) {
2792 if (!files.IsNull()) files += ",";
2793 files += sextra;
2794 }
2795 }
2796 if (stype == "ext") return files.Data();
2797 TString termfiles;
2798 if (!fTerminateFiles.IsNull()) {
2799 fTerminateFiles.Strip();
2800 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 2801 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 2802 TIter nextx(fextra);
2803 TObject *obj;
2804 while ((obj=nextx())) {
2805 if (aodfiles.Contains(obj->GetName())) continue;
2806 if (outputfiles.Contains(obj->GetName())) continue;
2807 if (termfiles.Contains(obj->GetName())) continue;
2808 if (sextra.Contains(obj->GetName())) continue;
2809 if (!termfiles.IsNull()) termfiles += ",";
2810 termfiles += obj->GetName();
2811 }
2812 delete fextra;
2813 }
2814 if (stype.Contains("ter")) {
2815 if (!files.IsNull() && !termfiles.IsNull()) {
2816 files += ",";
2817 files += termfiles;
2818 }
2819 }
2820 return files.Data();
2821}
2822
d2a409b2 2823//______________________________________________________________________________
a03be957 2824Bool_t AliAnalysisAlien::Submit()
d2a409b2 2825{
2826// Submit all master jobs.
2827 Int_t nmasterjobs = fInputFiles->GetEntries();
2828 Long_t tshoot = gSystem->Now();
a03be957 2829 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 2830 while (fNsubmitted < nmasterjobs) {
2831 Long_t now = gSystem->Now();
2832 if ((now-tshoot)>30000) {
2833 tshoot = now;
a03be957 2834 if (!SubmitNext()) return kFALSE;
d2a409b2 2835 }
2836 }
a03be957 2837 return kTRUE;
d2a409b2 2838}
2839
0f389141 2840//______________________________________________________________________________
a03be957 2841Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 2842{
2843// Submit all merging jobs.
2844 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2845 gGrid->Cd(fGridOutputDir);
2846 TString mergeJDLName = fExecutable;
2847 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 2848 if (!fInputFiles) {
2849 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
2850 return kFALSE;
2851 }
0f389141 2852 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 2853 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 2854 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2855 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 2856 if (fOutputToRunNo) {
2857 // The output directory is the run number
2858 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
2859 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
2860 } else {
17b4e9a4 2861 if (!fRunNumbers.Length() && !fRunRange[0]) {
2862 // The output directory is the grid outdir
2863 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
2864 runOutDir = fGridOutputDir;
2865 } else {
2866 // The output directory is the master number in 3 digits format
2867 printf("### Submitting merging job for master <%03d>\n", i);
2868 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
2869 }
7c2cd90a 2870 }
2871 // Check now the number of merging stages.
2b222f7c 2872 TObjArray *list = fOutputFiles.Tokenize(",");
2873 TIter next(list);
2874 TObjString *str;
2875 TString outputFile;
2876 while((str=(TObjString*)next())) {
2877 outputFile = str->GetString();
2878 Int_t index = outputFile.Index("@");
2879 if (index > 0) outputFile.Remove(index);
2880 if (!fMergeExcludes.Contains(outputFile)) break;
2881 }
2882 delete list;
b3e07543 2883 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 2884 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 2885 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 2886 }
a03be957 2887 if (!ntosubmit) return kTRUE;
9f020952 2888 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
2889 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
2890 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
2891 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
2892 \n ################################################################################################################");
0f389141 2893 gSystem->Exec("aliensh");
a03be957 2894 return kTRUE;
0f389141 2895}
2896
d2a409b2 2897//______________________________________________________________________________
a03be957 2898Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 2899{
08d5b699 2900// Submit next bunch of master jobs if the queue is free. The first master job is
2901// submitted right away, while the next will not be unless the previous was split.
2902// The plugin will not submit new master jobs if there are more that 500 jobs in
2903// waiting phase.
d2a409b2 2904 static Bool_t iscalled = kFALSE;
2905 static Int_t firstmaster = 0;
2906 static Int_t lastmaster = 0;
2907 static Int_t npermaster = 0;
a03be957 2908 if (iscalled) return kTRUE;
d2a409b2 2909 iscalled = kTRUE;
2910 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
2911 Int_t ntosubmit = 0;
2912 TGridResult *res;
2913 TString jobID = "";
3bdcb562 2914 Int_t nmasterjobs = fInputFiles->GetEntries();
2915 if (!fNsubmitted) {
2916 ntosubmit = 1;
2917 if (!IsUseSubmitPolicy()) {
90d50a8c 2918 if (nmasterjobs>5)
3bdcb562 2919 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
2920 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
2921 ntosubmit = nmasterjobs;
2922 }
2923 } else {
d2a409b2 2924 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 2925 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 2926 // If last master not split, just return
a03be957 2927 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 2928 // No more than 100 waiting jobs
08d5b699 2929 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2930 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 2931 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 2932 if (!ntosubmit) ntosubmit = 1;
84fcd93f 2933 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 2934 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
2935 }
d2a409b2 2936 for (Int_t i=0; i<ntosubmit; i++) {
2937 // Submit for a range of enumeration of runs.
a03be957 2938 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2939 TString query;
cd11251e 2940 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
2941 runOutDir.ReplaceAll(".xml", "");
2942 if (fOutputToRunNo)
2943 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
2944 else
2945 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 2946 printf("********* %s\n",query.Data());
d2a409b2 2947 res = gGrid->Command(query);
2948 if (res) {
98ca124f 2949 TString cjobId1 = res->GetKey(0,"jobId");
2950 if (!cjobId1.Length()) {
d2a409b2 2951 iscalled = kFALSE;
a03be957 2952 gGrid->Stdout();
2953 gGrid->Stderr();
2954 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
2955 return kFALSE;
d2a409b2 2956 } else {
2957 Info("StartAnalysis", "\n_______________________________________________________________________ \
2958 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
2959 \n_______________________________________________________________________",
98ca124f 2960 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 2961 jobID += cjobId1;
2962 jobID += " ";
98ca124f 2963 lastmaster = cjobId1.Atoi();
d2a409b2 2964 if (!firstmaster) firstmaster = lastmaster;
2965 fNsubmitted++;
2966 }
2967 delete res;
a03be957 2968 } else {
2969 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2970 return kFALSE;
d2a409b2 2971 }
2972 }
2973 iscalled = kFALSE;
a03be957 2974 return kTRUE;
d2a409b2 2975}
2976
c57f56b7 2977//______________________________________________________________________________
2978void AliAnalysisAlien::WriteAnalysisFile()
2979{
f10e8481 2980// Write current analysis manager into the file <analysisFile>
2981 TString analysisFile = fExecutable;
2982 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 2983 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2984 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2985 if (!mgr || !mgr->IsInitialized()) {
2986 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
2987 return;
2988 }
2989 // Check analysis type
2990 TObject *handler;
2991 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
2992 handler = (TObject*)mgr->GetInputEventHandler();
2993 if (handler) {
65eb22f8 2994 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
2995 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
2996 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
2997 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
2998 } else {
2999 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3000 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3001 }
c57f56b7 3002 }
3003 TDirectory *cdir = gDirectory;
f10e8481 3004 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3005 if (file) {
e16a394c 3006 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3007 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3008 // Unless merging makes no sense
3009 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3010 mgr->Write();
3011 delete file;
fe2d7fc2 3012 // Enable termination for local jobs
3013 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3014 }
3015 if (cdir) cdir->cd();
f10e8481 3016 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3017 }
3018 Bool_t copy = kTRUE;
d3b18c4c 3019 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3020 if (copy) {
3021 CdWork();
3022 TString workdir = gGrid->GetHomeDirectory();
3023 workdir += fGridWorkingDir;
d3b18c4c 3024 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3025 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
3026 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 3027 }
3028}
3029
3030//______________________________________________________________________________
3031void AliAnalysisAlien::WriteAnalysisMacro()
3032{
3033// Write the analysis macro that will steer the analysis in grid mode.
3034 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3035 ofstream out;
3036 out.open(fAnalysisMacro.Data(), ios::out);
3037 if (!out.good()) {
3038 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3039 return;
3040 }
5b273635 3041 Bool_t hasSTEERBase = kFALSE;
3042 Bool_t hasESD = kFALSE;
3043 Bool_t hasAOD = kFALSE;
3044 Bool_t hasANALYSIS = kFALSE;
3045 Bool_t hasANALYSISalice = kFALSE;
3046 Bool_t hasCORRFW = kFALSE;
c57f56b7 3047 TString func = fAnalysisMacro;
3048 TString type = "ESD";
3049 TString comment = "// Analysis using ";
c85cfc0f 3050 if (IsUseMCchain()) {
3051 type = "MC";
3052 comment += "MC";
3053 } else {
3054 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3055 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3056 type = "AOD";
3057 comment += "AOD";
3058 }
3059 }
0df6ccf2 3060 if (type!="AOD" && fFriendChainName!="") {
3061 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3062 return;
3063 }
c57f56b7 3064 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3065 else comment += " data";
3066 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3067 func.ReplaceAll(".C", "");
3068 out << "void " << func.Data() << "()" << endl;
3069 out << "{" << endl;
3070 out << comment.Data() << endl;
3071 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3072 out << " TStopwatch timer;" << endl;
3073 out << " timer.Start();" << endl << endl;
13f0c00b 3074 // Change temp directory to current one
3075 out << "// Set temporary merging directory to current one" << endl;
3076 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3077 out << "// Set temporary compilation directory to current one" << endl;
3078 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3079 // Reset existing include path
3080 out << "// Reset existing include path and add current directory first in the search" << endl;
3081 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3082 if (!fExecutableCommand.Contains("aliroot")) {
3083 out << "// load base root libraries" << endl;
3084 out << " gSystem->Load(\"libTree\");" << endl;
3085 out << " gSystem->Load(\"libGeom\");" << endl;
3086 out << " gSystem->Load(\"libVMC\");" << endl;
3087 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3088 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3089 }
d5c6455a 3090 if (fAdditionalRootLibs.Length()) {
47a4137d 3091 // in principle libtree /lib geom libvmc etc. can go into this list, too
3092 out << "// Add aditional libraries" << endl;
3093 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3094 TIter next(list);
3095 TObjString *str;
3096 while((str=(TObjString*)next())) {
3097 if (str->GetString().Contains(".so"))
3098 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3099 }
47a4137d 3100 if (list) delete list;
d5c6455a 3101 }
57377eb5 3102 out << "// Load analysis framework libraries" << endl;
652561d5 3103 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3104 if (!fPackages) {
f790bc1b 3105 if (!fExecutableCommand.Contains("aliroot")) {
3106 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3107 out << " gSystem->Load(\"libESD\");" << endl;
3108 out << " gSystem->Load(\"libAOD\");" << endl;
3109 }
4e5c5506 3110 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 3111 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3112 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3113 } else {
4e5c5506 3114 TIter next(fPackages);
3115 TObject *obj;
57377eb5 3116 TString pkgname;
57377eb5 3117 while ((obj=next())) {
3118 pkgname = obj->GetName();
4478e6f1 3119 if (pkgname == "STEERBase" ||
3120 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3121 if (pkgname == "ESD" ||
3122 pkgname == "ESD.par") hasESD = kTRUE;
3123 if (pkgname == "AOD" ||
3124 pkgname == "AOD.par") hasAOD = kTRUE;
3125 if (pkgname == "ANALYSIS" ||
3126 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3127 if (pkgname == "ANALYSISalice" ||
3128 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3129 if (pkgname == "CORRFW" ||
3130 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3131 }
3132 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3133 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3134 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3135 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3136 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3137 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3138 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3139 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3140 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
57377eb5 3141 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3142 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3143 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3144 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3145 out << "// Compile other par packages" << endl;
3146 next.Reset();
fcc9bb6f 3147 while ((obj=next())) {
3148 pkgname = obj->GetName();
4478e6f1 3149 if (pkgname == "STEERBase" ||
3150 pkgname == "STEERBase.par" ||
3151 pkgname == "ESD" ||
3152 pkgname == "ESD.par" ||
3153 pkgname == "AOD" ||
3154 pkgname == "AOD.par" ||
3155 pkgname == "ANALYSIS" ||
3156 pkgname == "ANALYSIS.par" ||
3157 pkgname == "ANALYSISalice" ||
3158 pkgname == "ANALYSISalice.par" ||
3159 pkgname == "CORRFW" ||
3160 pkgname == "CORRFW.par") continue;
5b273635 3161 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3162 }
4e5c5506 3163 }
b3e07543 3164 out << "// include path" << endl;
3165 // Get the include path from the interpreter and remove entries pointing to AliRoot
3166 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3167 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3168 out << " TIter nextpath(listpaths);" << endl;
3169 out << " TObjString *pname;" << endl;
3170 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3171 out << " TString current = pname->GetName();" << endl;
3172 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3173 out << " gSystem->AddIncludePath(current);" << endl;
3174 out << " }" << endl;
3175 out << " if (listpaths) delete listpaths;" << endl;
3176 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3177 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3178 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3179 if (fAdditionalLibs.Length()) {
3180 out << "// Add aditional AliRoot libraries" << endl;
3181 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3182 TIter next(list);
3183 TObjString *str;
3184 while((str=(TObjString*)next())) {
3185 if (str->GetString().Contains(".so"))
3186 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3187 if (str->GetString().Contains(".par"))
3188 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3189 }
3190 if (list) delete list;
3191 }
3192 out << endl;
c57f56b7 3193 out << "// analysis source to be compiled at runtime (if any)" << endl;
3194 if (fAnalysisSource.Length()) {
3195 TObjArray *list = fAnalysisSource.Tokenize(" ");
3196 TIter next(list);
3197 TObjString *str;
3198 while((str=(TObjString*)next())) {
3199 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3200 }
3201 if (list) delete list;
3202 }
3203 out << endl;
b3e07543 3204// out << " printf(\"Currently load libraries:\\n\");" << endl;
3205// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3206 if (fFastReadOption) {
d3339be3 3207 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3208 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3209 out << "// fast xrootd reading enabled" << endl;
3210 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3211 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3212 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3213 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3214 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3215 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
3216 }
c57f56b7 3217 out << "// connect to AliEn and make the chain" << endl;
3218 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
c57f56b7 3219 out << "// read the analysis manager from file" << endl;
f10e8481 3220 TString analysisFile = fExecutable;
3221 analysisFile.ReplaceAll(".sh", ".root");
3222 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 3223 out << " if (!file) return;" << endl;
3224 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3225 out << " AliAnalysisManager *mgr = 0;" << endl;
3226 out << " TKey *key;" << endl;
3227 out << " while ((key=(TKey*)nextkey())) {" << endl;
3228 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3229 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3230 out << " };" << endl;
3231 out << " if (!mgr) {" << endl;
205b201f 3232 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file " << analysisFile <<"\");" << endl;
c57f56b7 3233 out << " return;" << endl;
3234 out << " }" << endl << endl;
3235 out << " mgr->PrintStatus();" << endl;
52b6a92b 3236 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3237 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3238 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3239 } else {
9a8288ff 3240 if (TestBit(AliAnalysisGrid::kTest))
3241 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3242 else
3243 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3244 }
3245 }
242accb2 3246 if (IsUsingTags()) {
3247 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
3248 } else {
3249 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
3250 }
c57f56b7 3251 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 3252 out << " timer.Stop();" << endl;
3253 out << " timer.Print();" << endl;
c57f56b7 3254 out << "}" << endl << endl;
3255 if (IsUsingTags()) {
3256 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
3257 out << "{" << endl;
3258 out << "// Create a chain using tags from the xml file." << endl;
3259 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
3260 out << " if (!coll) {" << endl;
3261 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3262 out << " return NULL;" << endl;
3263 out << " }" << endl;
3264 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
3265 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
3266 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
3267 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
3268 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
3269 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
3270 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
3271 out << " // Check if the cuts configuration file was provided" << endl;
3272 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
3273 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
3274 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3275 out << " }" << endl;
0df6ccf2 3276 if (fFriendChainName=="") {
3277 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3278 } else {
3279 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
3280 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
3281 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
3282 }
c57f56b7 3283 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
3284 out << " chain->ls();" << endl;
3285 out << " return chain;" << endl;
fcc9bb6f 3286 out << "}" << endl << endl;
c57f56b7 3287 if (gSystem->AccessPathName("ConfigureCuts.C")) {
3288 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
3289 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
3290 msg += " AliLHCTagCuts *lhcCuts,\n";
3291 msg += " AliDetectorTagCuts *detCuts,\n";
3292 msg += " AliEventTagCuts *evCuts)";
23329835 3293 Info("WriteAnalysisMacro", "%s", msg.Data());
c57f56b7 3294 }
0df6ccf2 3295 }
3296 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 3297 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3298 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3299 out << "{" << endl;
3300 out << "// Create a chain using url's from xml file" << endl;
242accb2 3301 out << " TString filename;" << endl;
3302 out << " Int_t run = 0;" << endl;
c85cfc0f 3303 if (IsUseMCchain()) {
3304 out << " TString treename = \"TE\";" << endl;
3305 } else {
3306 out << " TString treename = type;" << endl;
3307 out << " treename.ToLower();" << endl;
3308 out << " treename += \"Tree\";" << endl;
3309 }
e02fee64 3310 out << " printf(\"***************************************\\n\");" << endl;
3311 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3312 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3313 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3314 out << " if (!coll) {" << endl;
3315 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3316 out << " return NULL;" << endl;
3317 out << " }" << endl;
242accb2 3318 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3319 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3320 if(fFriendChainName!="") {
3321 out << " TChain *chainFriend = new TChain(treename);" << endl;
3322 }
c57f56b7 3323 out << " coll->Reset();" << endl;
0df6ccf2 3324 out << " while (coll->Next()) {" << endl;
242accb2 3325 out << " filename = coll->GetTURL("");" << endl;
3326 out << " if (mgr) {" << endl;
3327 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3328 out << " if (nrun && nrun != run) {" << endl;
3329 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3330 out << " mgr->SetRunFromPath(nrun);" << endl;
3331 out << " run = nrun;" << endl;
3332 out << " }" << endl;
3333 out << " }" << endl;
3334 out << " chain->Add(filename);" << endl;
0df6ccf2 3335 if(fFriendChainName!="") {
3336 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3337 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3338 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3339 out << " chainFriend->Add(fileFriend.Data());" << endl;
3340 }
3341 out << " }" << endl;
c57f56b7 3342 out << " if (!chain->GetNtrees()) {" << endl;
3343 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3344 out << " return NULL;" << endl;
3345 out << " }" << endl;
0df6ccf2 3346 if(fFriendChainName!="") {
3347 out << " chain->AddFriend(chainFriend);" << endl;
3348 }
c57f56b7 3349 out << " return chain;" << endl;
fcc9bb6f 3350 out << "}" << endl << endl;
c57f56b7 3351 }
5b273635 3352 if (hasANALYSISalice) {
3353 out <<"//________________________________________________________________________________" << endl;
3354 out << "Bool_t SetupPar(const char *package) {" << endl;
3355 out << "// Compile the package and set it up." << endl;
3356 out << " TString pkgdir = package;" << endl;
3357 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3358 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3359 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3360 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3361 out << " // Check for BUILD.sh and execute" << endl;
3362 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3363 out << " printf(\"*******************************\\n\");" << endl;
3364 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3365 out << " printf(\"*******************************\\n\");" << endl;
3366 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3367 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3368 out << " gSystem->ChangeDirectory(cdir);" << endl;
3369 out << " return kFALSE;" << endl;
3370 out << " }" << endl;
3371 out << " } else {" << endl;
3372 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3373 out << " gSystem->ChangeDirectory(cdir);" << endl;
3374 out << " return kFALSE;" << endl;
3375 out << " }" << endl;
3376 out << " // Check for SETUP.C and execute" << endl;
3377 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3378 out << " printf(\"*******************************\\n\");" << endl;
3379 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3380 out << " printf(\"*******************************\\n\");" << endl;
3381 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3382 out << " } else {" << endl;
3383 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3384 out << " gSystem->ChangeDirectory(cdir);" << endl;
3385 out << " return kFALSE;" << endl;
3386 out << " }" << endl;
3387 out << " // Restore original workdir" << endl;
3388 out << " gSystem->ChangeDirectory(cdir);" << endl;
3389 out << " return kTRUE;" << endl;
3390 out << "}" << endl;
3391 }
c57f56b7 3392 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3393 }
3394 Bool_t copy = kTRUE;
d3b18c4c 3395 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3396 if (copy) {
3397 CdWork();
3398 TString workdir = gGrid->GetHomeDirectory();
3399 workdir += fGridWorkingDir;
3400 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
3401 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
3402 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
3403 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
3404 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
3405 }
3406 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
3407 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3408 }
3409}
3410
0f389141 3411//______________________________________________________________________________
3412void AliAnalysisAlien::WriteMergingMacro()
3413{
3414// Write a macro to merge the outputs per master job.
3415 if (!fMergeViaJDL) return;
3416 if (!fOutputFiles.Length()) {
3417 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3418 return;
3419 }
3420 TString mergingMacro = fExecutable;
3421 mergingMacro.ReplaceAll(".sh","_merge.C");
3422 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
3423 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3424 ofstream out;
3425 out.open(mergingMacro.Data(), ios::out);
3426 if (!out.good()) {
3427 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3428 return;
3429 }
5b273635 3430 Bool_t hasSTEERBase = kFALSE;
3431 Bool_t hasESD = kFALSE;
3432 Bool_t hasAOD = kFALSE;
3433 Bool_t hasANALYSIS = kFALSE;
3434 Bool_t hasANALYSISalice = kFALSE;
3435 Bool_t hasCORRFW = kFALSE;
0f389141 3436 TString func = mergingMacro;
3437 TString comment;
3438 func.ReplaceAll(".C", "");
f47d5cb4 3439 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 3440 out << "{" << endl;
3441 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
3442 out << " TStopwatch timer;" << endl;
3443 out << " timer.Start();" << endl << endl;
b3e07543 3444 // Reset existing include path
3445 out << "// Reset existing include path and add current directory first in the search" << endl;
3446 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 3447 if (!fExecutableCommand.Contains("aliroot")) {
3448 out << "// load base root libraries" << endl;
3449 out << " gSystem->Load(\"libTree\");" << endl;
3450 out << " gSystem->Load(\"libGeom\");" << endl;
3451 out << " gSystem->Load(\"libVMC\");" << endl;
3452 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3453 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3454 }
0f389141 3455 if (fAdditionalRootLibs.Length()) {
3456 // in principle libtree /lib geom libvmc etc. can go into this list, too
3457 out << "// Add aditional libraries" << endl;
3458 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3459 TIter next(list);
3460 TObjString *str;
3461 while((str=(TObjString*)next())) {
3462 if (str->GetString().Contains(".so"))
3463 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3464 }
3465 if (list) delete list;
3466 }
0f389141 3467 out << "// Load analysis framework libraries" << endl;
3468 if (!fPackages) {
7c2cd90a 3469 if (!fExecutableCommand.Contains("aliroot")) {
3470 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3471 out << " gSystem->Load(\"libESD\");" << endl;
3472 out << " gSystem->Load(\"libAOD\");" << endl;
3473 }
0f389141 3474 out << " gSystem->Load(\"libANALYSIS\");" << endl;
3475 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3476 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3477 } else {
3478 TIter next(fPackages);
3479 TObject *obj;
3480 TString pkgname;
5b273635 3481 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 3482 while ((obj=next())) {
3483 pkgname = obj->GetName();
3484 if (pkgname == "STEERBase" ||
3485 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3486 if (pkgname == "ESD" ||
3487 pkgname == "ESD.par") hasESD = kTRUE;
3488 if (pkgname == "AOD" ||
3489 pkgname == "AOD.par") hasAOD = kTRUE;
3490 if (pkgname == "ANALYSIS" ||
3491 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3492 if (pkgname == "ANALYSISalice" ||
3493 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3494 if (pkgname == "CORRFW" ||
3495 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
3496 }
5b273635 3497 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 3498 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3499 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 3500 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3501 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 3502 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3503 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0f389141 3504 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3505 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
0f389141 3506 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3507 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 3508 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3509 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 3510 out << "// Compile other par packages" << endl;
3511 next.Reset();
3512 while ((obj=next())) {
3513 pkgname = obj->GetName();
3514 if (pkgname == "STEERBase" ||
3515 pkgname == "STEERBase.par" ||
3516 pkgname == "ESD" ||
3517 pkgname == "ESD.par" ||
3518 pkgname == "AOD" ||
3519 pkgname == "AOD.par" ||
3520 pkgname == "ANALYSIS" ||
3521 pkgname == "ANALYSIS.par" ||
3522 pkgname == "ANALYSISalice" ||
3523 pkgname == "ANALYSISalice.par" ||
3524 pkgname == "CORRFW" ||
3525 pkgname == "CORRFW.par") continue;
5b273635 3526 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 3527 }
3528 }
b3e07543 3529 out << "// include path" << endl;
3530 // Get the include path from the interpreter and remove entries pointing to AliRoot
3531 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3532 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3533 out << " TIter nextpath(listpaths);" << endl;
3534 out << " TObjString *pname;" << endl;
3535 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3536 out << " TString current = pname->GetName();" << endl;
3537 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3538 out << " gSystem->AddIncludePath(current);" << endl;
3539 out << " }" << endl;
3540 out << " if (listpaths) delete listpaths;" << endl;
3541 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3542 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3543 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 3544 if (fAdditionalLibs.Length()) {
3545 out << "// Add aditional AliRoot libraries" << endl;
3546 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3547 TIter next(list);
3548 TObjString *str;
3549 while((str=(TObjString*)next())) {
3550 if (str->GetString().Contains(".so"))
3551 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3552 }
3553 if (list) delete list;
3554 }
3555 out << endl;
3556 out << "// Analysis source to be compiled at runtime (if any)" << endl;
3557 if (fAnalysisSource.Length()) {
3558 TObjArray *list = fAnalysisSource.Tokenize(" ");
3559 TIter next(list);
3560 TObjString *str;
3561 while((str=(TObjString*)next())) {
3562 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3563 }
3564 if (list) delete list;
3565 }
149d288c 3566 out << endl;
3567
0f389141 3568 if (fFastReadOption) {
3569 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
3570 out << "// fast xrootd reading enabled" << endl;
3571 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3572 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3573 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3574 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3575 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3576 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 3577 }
3578 // Change temp directory to current one
3579 out << "// Set temporary merging directory to current one" << endl;
3580 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3581 out << "// Set temporary compilation directory to current one" << endl;
3582 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
0f389141 3583 out << "// Connect to AliEn" << endl;
3584 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 3585 out << " TString outputDir = dir;" << endl;
f790bc1b 3586 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
0f389141 3587 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
58268c13 3588 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 3589 out << " TIter *iter = new TIter(list);" << endl;
3590 out << " TObjString *str;" << endl;
a2f5fc01 3591 out << " TString outputFile;" << endl;
0f389141 3592 out << " Bool_t merged = kTRUE;" << endl;
3593 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 3594 out << " outputFile = str->GetString();" << endl;
7c2cd90a 3595 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 3596 out << " Int_t index = outputFile.Index(\"@\");" << endl;
3597 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 3598 out << " // Skip already merged outputs" << endl;
a2f5fc01 3599 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
3600 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 3601 out << " continue;" << endl;
3602 out << " }" << endl;
a2f5fc01 3603 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 3604 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 3605 out << " if (!merged) {" << endl;
a2f5fc01 3606 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 3607 out << " return;" << endl;
0f389141 3608 out << " }" << endl;
3609 out << " }" << endl;
7c2cd90a 3610 out << " // all outputs merged, validate" << endl;
3611 out << " ofstream out;" << endl;
3612 out << " out.open(\"outputs_valid\", ios::out);" << endl;
3613 out << " out.close();" << endl;
3614 out << " // read the analysis manager from file" << endl;
f866cba5 3615 TString analysisFile = fExecutable;
3616 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 3617 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
f866cba5 3618 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
7c2cd90a 3619 out << " if (!file) return;" << endl;
f866cba5 3620 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3621 out << " AliAnalysisManager *mgr = 0;" << endl;
3622 out << " TKey *key;" << endl;
3623 out << " while ((key=(TKey*)nextkey())) {" << endl;
3624 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3625 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3626 out << " };" << endl;
3627 out << " if (!mgr) {" << endl;
3628 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
3629 out << " return;" << endl;
3630 out << " }" << endl << endl;
5c4250fc 3631 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 3632 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 3633 out << " mgr->PrintStatus();" << endl;
3634 if (AliAnalysisManager::GetAnalysisManager()) {
3635 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
3636 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
3637 } else {
9a8288ff 3638 if (TestBit(AliAnalysisGrid::kTest))
3639 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3640 else
3641 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 3642 }
3643 }
9a8288ff 3644 out << " TTree *tree = NULL;" << endl;
3645 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 3646 out << "}" << endl << endl;
5b273635 3647 if (hasANALYSISalice) {
3648 out <<"//________________________________________________________________________________" << endl;
3649 out << "Bool_t SetupPar(const char *package) {" << endl;
3650 out << "// Compile the package and set it up." << endl;
3651 out << " TString pkgdir = package;" << endl;
3652 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3653 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3654 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3655 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3656 out << " // Check for BUILD.sh and execute" << endl;
3657 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3658 out << " printf(\"*******************************\\n\");" << endl;
3659 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3660 out << " printf(\"*******************************\\n\");" << endl;
3661 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3662 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3663 out << " gSystem->ChangeDirectory(cdir);" << endl;
3664 out << " return kFALSE;" << endl;
3665 out << " }" << endl;
3666 out << " } else {" << endl;
3667 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3668 out << " gSystem->ChangeDirectory(cdir);" << endl;
3669 out << " return kFALSE;" << endl;
3670 out << " }" << endl;
3671 out << " // Check for SETUP.C and execute" << endl;
3672 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3673 out << " printf(\"*******************************\\n\");" << endl;
3674 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3675 out << " printf(\"*******************************\\n\");" << endl;
3676 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3677 out << " } else {" << endl;
3678 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3679 out << " gSystem->ChangeDirectory(cdir);" << endl;
3680 out << " return kFALSE;" << endl;
3681 out << " }" << endl;
3682 out << " // Restore original workdir" << endl;
3683 out << " gSystem->ChangeDirectory(cdir);" << endl;
3684 out << " return kTRUE;" << endl;
3685 out << "}" << endl;
3686 }
0f389141 3687 }
3688 Bool_t copy = kTRUE;
d3b18c4c 3689 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3690 if (copy) {
3691 CdWork();
3692 TString workdir = gGrid->GetHomeDirectory();
3693 workdir += fGridWorkingDir;
3694 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
3695 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
3696 TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
3697 }
3698}
3699
3700//______________________________________________________________________________
3701Bool_t AliAnalysisAlien::SetupPar(const char *package)
3702{
205b201f 3703// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 3704// Note that for loading the compiled library. The current directory should have precedence in
3705// LD_LIBRARY_PATH
3706 TString pkgdir = package;
3707 pkgdir.ReplaceAll(".par","");
b50a41c3 3708 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 3709 TString cdir = gSystem->WorkingDirectory();
3710 gSystem->ChangeDirectory(pkgdir);
3711 // Check for BUILD.sh and execute
3712 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
3713 printf("**************************************************\n");
3714 printf("*** Building PAR archive %s\n", package);
3715 printf("**************************************************\n");
3716 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
3717 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
3718 gSystem->ChangeDirectory(cdir);
3719 return kFALSE;
3720 }
3721 } else {
3722 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
3723 gSystem->ChangeDirectory(cdir);
3724 return kFALSE;
3725 }
3726 // Check for SETUP.C and execute
3727 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
3728 printf("**************************************************\n");
3729 printf("*** Setup PAR archive %s\n", package);
3730 printf("**************************************************\n");
3731 gROOT->Macro("PROOF-INF/SETUP.C");
3732 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
3733 } else {
3734 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
3735 gSystem->ChangeDirectory(cdir);
3736 return kFALSE;
3737 }
3738 // Restore original workdir
3739 gSystem->ChangeDirectory(cdir);
3740 return kTRUE;
3741}
3742
c57f56b7 3743//______________________________________________________________________________
3744void AliAnalysisAlien::WriteExecutable()
3745{
3746// Generate the alien executable script.
3747 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3748 ofstream out;
3749 out.open(fExecutable.Data(), ios::out);
3750 if (out.bad()) {
5513444a 3751 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 3752 return;
3753 }
3754 out << "#!/bin/bash" << endl;
2a4d5166 3755 // Make sure we can properly compile par files
3756 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 3757 out << "echo \"=========================================\"" << endl;
3758 out << "echo \"############## PATH : ##############\"" << endl;
3759 out << "echo $PATH" << endl;
3760 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3761 out << "echo $LD_LIBRARY_PATH" << endl;
3762 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3763 out << "echo $ROOTSYS" << endl;
3764 out << "echo \"############## which root : ##############\"" << endl;
3765 out << "which root" << endl;
3766 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3767 out << "echo $ALICE_ROOT" << endl;
3768 out << "echo \"############## which aliroot : ##############\"" << endl;
3769 out << "which aliroot" << endl;
9c5ddadc 3770 out << "echo \"############## system limits : ##############\"" << endl;
3771 out << "ulimit -a" << endl;
3772 out << "echo \"############## memory : ##############\"" << endl;
3773 out << "free -m" << endl;
c57f56b7 3774 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 3775 out << fExecutableCommand << " ";
631c0b05 3776 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 3777 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
3778 out << "echo \"############## memory after: ##############\"" << endl;
3779 out << "free -m" << endl;
c57f56b7 3780 }
3781 Bool_t copy = kTRUE;
d3b18c4c 3782 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3783 if (copy) {
3784 CdWork();
3785 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 3786 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3787 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 3788 workdir += fGridWorkingDir;
3789 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
3790 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3791 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
c57f56b7 3792 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
3793 }
3794}
3795
0f389141 3796//______________________________________________________________________________
3797void AliAnalysisAlien::WriteMergeExecutable()
3798{
3799// Generate the alien executable script for the merging job.
3800 if (!fMergeViaJDL) return;
3801 TString mergeExec = fExecutable;
3802 mergeExec.ReplaceAll(".sh", "_merge.sh");
3803 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3804 ofstream out;
3805 out.open(mergeExec.Data(), ios::out);
3806 if (out.bad()) {
3807 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
3808 return;
3809 }
3810 out << "#!/bin/bash" << endl;
2a4d5166 3811 // Make sure we can properly compile par files
3812 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 3813 out << "echo \"=========================================\"" << endl;
3814 out << "echo \"############## PATH : ##############\"" << endl;
3815 out << "echo $PATH" << endl;
3816 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3817 out << "echo $LD_LIBRARY_PATH" << endl;
3818 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3819 out << "echo $ROOTSYS" << endl;
3820 out << "echo \"############## which root : ##############\"" << endl;
3821 out << "which root" << endl;
3822 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3823 out << "echo $ALICE_ROOT" << endl;
3824 out << "echo \"############## which aliroot : ##############\"" << endl;
3825 out << "which aliroot" << endl;
3826 out << "echo \"############## system limits : ##############\"" << endl;
3827 out << "ulimit -a" << endl;
3828 out << "echo \"############## memory : ##############\"" << endl;
3829 out << "free -m" << endl;
3830 out << "echo \"=========================================\"" << endl << endl;
0f389141 3831 TString mergeMacro = fExecutable;
3832 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 3833 if (IsOneStageMerging())
3834 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
3835 else
f47d5cb4 3836 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 3837 out << fExecutableCommand << " " << "$ARG" << endl;
3838 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
3839 out << "echo \"############## memory after: ##############\"" << endl;
3840 out << "free -m" << endl;
0f389141 3841 }
3842 Bool_t copy = kTRUE;
d3b18c4c 3843 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3844 if (copy) {
3845 CdWork();
3846 TString workdir = gGrid->GetHomeDirectory();
3847 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3848 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 3849 workdir += fGridWorkingDir;
3850 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
3851 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3852 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
0f389141 3853 TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
3854 }
3855}
3856
c57f56b7 3857//______________________________________________________________________________
5513444a 3858void AliAnalysisAlien::WriteProductionFile(const char *filename) const
3859{
3860// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 3861// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 3862// Next lines: full_path_to_dataset XXX (XXX is a string)
3863// To submit, one has to: submit jdl XXX for all lines
3864 ofstream out;
3865 out.open(filename, ios::out);
3866 if (out.bad()) {
3867 Error("WriteProductionFile", "Bad file name: %s", filename);
3868 return;
3869 }
d3b18c4c 3870 TString workdir;
3871 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
3872 workdir = gGrid->GetHomeDirectory();
5513444a 3873 workdir += fGridWorkingDir;
f5e8c702 3874 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 3875 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 3876 out << locjdl << " " << njobspermaster << endl;
5513444a 3877 Int_t nmasterjobs = fInputFiles->GetEntries();
3878 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 3879 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3880 runOutDir.ReplaceAll(".xml", "");
3881 if (fOutputToRunNo)
3882 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
3883 else
3884 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 3885 }
d3b18c4c 3886 if (gGrid) {
3887 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
3888 if (FileExists(filename)) gGrid->Rm(filename);
3889 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
3890 }
5513444a 3891}
3892
3893//______________________________________________________________________________
0f389141 3894void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 3895{
3896// Generate the alien validation script.
3897 // Generate the validation script
3898 TObjString *os;
d3b18c4c 3899 if (fValidationScript.IsNull()) {
3900 fValidationScript = fExecutable;
3901 fValidationScript.ReplaceAll(".sh", "_validation.sh");
3902 }
3903 TString validationScript = fValidationScript;
3904 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 3905 if (!Connect()) {
3906 Error("WriteValidationScript", "Alien connection required");
3907 return;
3908 }
d3b18c4c 3909 if (!fTerminateFiles.IsNull()) {
3910 fTerminateFiles.Strip();
3911 fTerminateFiles.ReplaceAll(" ",",");
3912 }
a2f5fc01 3913 TString outStream = "";
3914 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 3915 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3916 ofstream out;
0d5d317c 3917 out.open(validationScript, ios::out);
c57f56b7 3918 out << "#!/bin/bash" << endl;
3919 out << "##################################################" << endl;
3920 out << "validateout=`dirname $0`" << endl;
3921 out << "validatetime=`date`" << endl;
3922 out << "validated=\"0\";" << endl;
3923 out << "error=0" << endl;
3924 out << "if [ -z $validateout ]" << endl;
3925 out << "then" << endl;
3926 out << " validateout=\".\"" << endl;
3927 out << "fi" << endl << endl;
3928 out << "cd $validateout;" << endl;
3929 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 3930 out << "echo \"*******************************************************\"" << outStream << endl;
3931 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 3932 out << "" << endl;
a2f5fc01 3933 out << "echo \"* Time: $validatetime \"" << outStream << endl;
3934 out << "echo \"* Dir: $validateout\"" << outStream << endl;
3935 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
3936 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3937 out << "ls -la ./" << outStream << endl;
3938 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 3939 out << "##################################################" << endl;
ebec370a 3940 out << "" << endl;
3941
3942 out << "if [ ! -f stderr ] ; then" << endl;
3943 out << " error=1" << endl;
a2f5fc01 3944 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
3945 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3946 out << "fi" << endl;
3947
b34c9f51 3948 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
3949 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
3950 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
3951 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
3952 out << "" << endl;
3953
ebec370a 3954 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
3955 out << " error=1" << endl;
a2f5fc01 3956 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
3957 out << " echo \"$parArch\" " << outStream << endl;
3958 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3959 out << "fi" << endl;
3960
3961 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
3962 out << " error=1" << endl;
a2f5fc01 3963 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
3964 out << " echo \"$segViol\" " << outStream << endl;
3965 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3966 out << "fi" << endl;
3967
3968 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
3969 out << " error=1" << endl;
a2f5fc01 3970 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
3971 out << " echo \"$segFault\" " << outStream << endl;
3972 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3973 out << "fi" << endl;
3974
b34c9f51 3975 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
3976 out << " error=1" << endl;
a2f5fc01 3977 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
3978 out << " echo \"$glibcErr\" " << outStream << endl;
3979 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 3980 out << "fi" << endl;
3981
ebec370a 3982 // Part dedicated to the specific analyses running into the train
3983
d3b18c4c 3984 TString outputFiles = fOutputFiles;
3985 if (merge && !fTerminateFiles.IsNull()) {
3986 outputFiles += ",";
3987 outputFiles += fTerminateFiles;
3988 }
3989 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 3990 TIter next1(arr);
a2f5fc01 3991 TString outputFile;
bbd88db8 3992 while (!merge && (os=(TObjString*)next1())) {
3993 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 3994 outputFile = os->GetString();
3995 Int_t index = outputFile.Index("@");
3996 if (index > 0) outputFile.Remove(index);
bbd88db8 3997 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 3998 if (outputFile.Contains("*")) continue;
3999 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4000 out << " error=1" << endl;
7c2cd90a 4001 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4002 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4003 out << "fi" << endl;
4004 }
4005 delete arr;
7c2cd90a 4006 out << "if ! [ -f outputs_valid ] ; then" << endl;
4007 out << " error=1" << endl;
4008 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4009 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4010 out << "fi" << endl;
923e2ca5 4011
c57f56b7 4012 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4013 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4014 if (!IsKeepLogs()) {
4015 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4016 outStream = "";
149d288c 4017 out << " rm -f std*" << endl;
4018 }
c57f56b7 4019 out << "fi" << endl;
4020
a2f5fc01 4021 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4022 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4023 out << "cd -" << endl;
4024 out << "exit $error" << endl;
4025 }
4026 Bool_t copy = kTRUE;
d3b18c4c 4027 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4028 if (copy) {
4029 CdWork();
4030 TString workdir = gGrid->GetHomeDirectory();
4031 workdir += fGridWorkingDir;
d3b18c4c 4032 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4033 if (FileExists(validationScript)) gGrid->Rm(validationScript);
4034 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 4035 }
4036}