]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
fix in CopyLocalDataSet
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
51ClassImp(AliAnalysisAlien)
6c395669 52#if 0
53;
54#endif
c57f56b7 55
6c395669 56namespace {
57 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
58 {
59 TString sl(Form("file:%s", loc));
60 TString sr(Form("alien://%s", rem));
61 Bool_t ret = TFile::Cp(sl, sr);
62 if (!ret) {
63 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
64 }
65 return ret;
66 }
67}
68
c57f56b7 69//______________________________________________________________________________
70AliAnalysisAlien::AliAnalysisAlien()
71 :AliAnalysisGrid(),
72 fGridJDL(NULL),
0f389141 73 fMergingJDL(NULL),
c57f56b7 74 fPrice(0),
75 fTTL(0),
76 fSplitMaxInputFileNumber(0),
77 fMaxInitFailed(0),
78 fMasterResubmitThreshold(0),
bb885a9e 79 fNtestFiles(0),
319593fb 80 fNrunsPerMaster(0),
16a4353c 81 fMaxMergeFiles(0),
b3e07543 82 fMaxMergeStages(0),
d2a409b2 83 fNsubmitted(0),
a3e84053 84 fProductionMode(0),
cd11251e 85 fOutputToRunNo(0),
0f389141 86 fMergeViaJDL(0),
87 fFastReadOption(0),
e1c22e21 88 fOverwriteMode(1),
149d288c 89 fNreplicas(2),
3bdcb562 90 fNproofWorkers(0),
27734f0e 91 fNproofWorkersPerSlave(0),
3bdcb562 92 fProofReset(0),
c57f56b7 93 fRunNumbers(),
94 fExecutable(),
0a1c1f7f 95 fExecutableCommand(),
c57f56b7 96 fArguments(),
631c0b05 97 fExecutableArgs(),
c57f56b7 98 fAnalysisMacro(),
99 fAnalysisSource(),
d3b18c4c 100 fValidationScript(),
d5c6455a 101 fAdditionalRootLibs(),
c57f56b7 102 fAdditionalLibs(),
103 fSplitMode(),
104 fAPIVersion(),
105 fROOTVersion(),
106 fAliROOTVersion(),
648174cf 107 fExternalPackages(),
c57f56b7 108 fUser(),
109 fGridWorkingDir(),
110 fGridDataDir(),
111 fDataPattern(),
112 fGridOutputDir(),
113 fOutputArchive(),
114 fOutputFiles(),
115 fInputFormat(),
e7c71df0 116 fDatasetName(),
c57f56b7 117 fJDLName(),
d3b18c4c 118 fTerminateFiles(),
bb885a9e 119 fMergeExcludes(),
37d21c01 120 fRegisterExcludes(),
f965131e 121 fIncludePath(),
bb885a9e 122 fCloseSE(),
0df6ccf2 123 fFriendChainName(),
c6cb3634 124 fJobTag(),
648174cf 125 fOutputSingle(),
5fce53f4 126 fRunPrefix(),
3bdcb562 127 fProofCluster(),
128 fProofDataSet(),
129 fFileForTestMode(),
130 fRootVersionForProof(),
131 fAliRootMode(),
f47d5cb4 132 fMergeDirName(),
4e5c5506 133 fInputFiles(0),
28ce6c9c 134 fPackages(0),
77f34eae 135 fModules(0),
28ce6c9c 136 fProofParam()
c57f56b7 137{
138// Dummy ctor.
139 SetDefaults();
140}
141
142//______________________________________________________________________________
143AliAnalysisAlien::AliAnalysisAlien(const char *name)
144 :AliAnalysisGrid(name),
145 fGridJDL(NULL),
0f389141 146 fMergingJDL(NULL),
c57f56b7 147 fPrice(0),
148 fTTL(0),
149 fSplitMaxInputFileNumber(0),
150 fMaxInitFailed(0),
151 fMasterResubmitThreshold(0),
bb885a9e 152 fNtestFiles(0),
319593fb 153 fNrunsPerMaster(0),
16a4353c 154 fMaxMergeFiles(0),
b3e07543 155 fMaxMergeStages(0),
d2a409b2 156 fNsubmitted(0),
a3e84053 157 fProductionMode(0),
cd11251e 158 fOutputToRunNo(0),
0f389141 159 fMergeViaJDL(0),
160 fFastReadOption(0),
e1c22e21 161 fOverwriteMode(1),
149d288c 162 fNreplicas(2),
3bdcb562 163 fNproofWorkers(0),
27734f0e 164 fNproofWorkersPerSlave(0),
3bdcb562 165 fProofReset(0),
c57f56b7 166 fRunNumbers(),
167 fExecutable(),
0a1c1f7f 168 fExecutableCommand(),
c57f56b7 169 fArguments(),
631c0b05 170 fExecutableArgs(),
c57f56b7 171 fAnalysisMacro(),
172 fAnalysisSource(),
d3b18c4c 173 fValidationScript(),
d5c6455a 174 fAdditionalRootLibs(),
c57f56b7 175 fAdditionalLibs(),
176 fSplitMode(),
177 fAPIVersion(),
178 fROOTVersion(),
179 fAliROOTVersion(),
648174cf 180 fExternalPackages(),
c57f56b7 181 fUser(),
182 fGridWorkingDir(),
183 fGridDataDir(),
184 fDataPattern(),
185 fGridOutputDir(),
186 fOutputArchive(),
187 fOutputFiles(),
188 fInputFormat(),
e7c71df0 189 fDatasetName(),
c57f56b7 190 fJDLName(),
d3b18c4c 191 fTerminateFiles(),
bb885a9e 192 fMergeExcludes(),
37d21c01 193 fRegisterExcludes(),
f965131e 194 fIncludePath(),
bb885a9e 195 fCloseSE(),
0df6ccf2 196 fFriendChainName(),
c6cb3634 197 fJobTag(),
648174cf 198 fOutputSingle(),
5fce53f4 199 fRunPrefix(),
3bdcb562 200 fProofCluster(),
201 fProofDataSet(),
202 fFileForTestMode(),
203 fRootVersionForProof(),
204 fAliRootMode(),
f47d5cb4 205 fMergeDirName(),
4e5c5506 206 fInputFiles(0),
28ce6c9c 207 fPackages(0),
77f34eae 208 fModules(0),
28ce6c9c 209 fProofParam()
c57f56b7 210{
211// Default ctor.
212 SetDefaults();
213}
214
215//______________________________________________________________________________
216AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
217 :AliAnalysisGrid(other),
218 fGridJDL(NULL),
0f389141 219 fMergingJDL(NULL),
c57f56b7 220 fPrice(other.fPrice),
221 fTTL(other.fTTL),
222 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
223 fMaxInitFailed(other.fMaxInitFailed),
224 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 225 fNtestFiles(other.fNtestFiles),
319593fb 226 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 227 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 228 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 229 fNsubmitted(other.fNsubmitted),
a3e84053 230 fProductionMode(other.fProductionMode),
cd11251e 231 fOutputToRunNo(other.fOutputToRunNo),
0f389141 232 fMergeViaJDL(other.fMergeViaJDL),
233 fFastReadOption(other.fFastReadOption),
234 fOverwriteMode(other.fOverwriteMode),
149d288c 235 fNreplicas(other.fNreplicas),
3bdcb562 236 fNproofWorkers(other.fNproofWorkers),
27734f0e 237 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 238 fProofReset(other.fProofReset),
c57f56b7 239 fRunNumbers(other.fRunNumbers),
240 fExecutable(other.fExecutable),
0a1c1f7f 241 fExecutableCommand(other.fExecutableCommand),
c57f56b7 242 fArguments(other.fArguments),
631c0b05 243 fExecutableArgs(other.fExecutableArgs),
c57f56b7 244 fAnalysisMacro(other.fAnalysisMacro),
245 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 246 fValidationScript(other.fValidationScript),
d5c6455a 247 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 248 fAdditionalLibs(other.fAdditionalLibs),
249 fSplitMode(other.fSplitMode),
250 fAPIVersion(other.fAPIVersion),
251 fROOTVersion(other.fROOTVersion),
252 fAliROOTVersion(other.fAliROOTVersion),
648174cf 253 fExternalPackages(other.fExternalPackages),
c57f56b7 254 fUser(other.fUser),
255 fGridWorkingDir(other.fGridWorkingDir),
256 fGridDataDir(other.fGridDataDir),
257 fDataPattern(other.fDataPattern),
258 fGridOutputDir(other.fGridOutputDir),
259 fOutputArchive(other.fOutputArchive),
260 fOutputFiles(other.fOutputFiles),
261 fInputFormat(other.fInputFormat),
e7c71df0 262 fDatasetName(other.fDatasetName),
c57f56b7 263 fJDLName(other.fJDLName),
d3b18c4c 264 fTerminateFiles(other.fTerminateFiles),
bb885a9e 265 fMergeExcludes(other.fMergeExcludes),
37d21c01 266 fRegisterExcludes(other.fRegisterExcludes),
f965131e 267 fIncludePath(other.fIncludePath),
bb885a9e 268 fCloseSE(other.fCloseSE),
0df6ccf2 269 fFriendChainName(other.fFriendChainName),
c6cb3634 270 fJobTag(other.fJobTag),
648174cf 271 fOutputSingle(other.fOutputSingle),
5fce53f4 272 fRunPrefix(other.fRunPrefix),
3bdcb562 273 fProofCluster(other.fProofCluster),
274 fProofDataSet(other.fProofDataSet),
275 fFileForTestMode(other.fFileForTestMode),
276 fRootVersionForProof(other.fRootVersionForProof),
277 fAliRootMode(other.fAliRootMode),
f47d5cb4 278 fMergeDirName(other.fMergeDirName),
4e5c5506 279 fInputFiles(0),
28ce6c9c 280 fPackages(0),
77f34eae 281 fModules(0),
28ce6c9c 282 fProofParam()
c57f56b7 283{
284// Copy ctor.
285 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 286 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 287 fRunRange[0] = other.fRunRange[0];
288 fRunRange[1] = other.fRunRange[1];
c57f56b7 289 if (other.fInputFiles) {
290 fInputFiles = new TObjArray();
291 TIter next(other.fInputFiles);
292 TObject *obj;
293 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
294 fInputFiles->SetOwner();
295 }
4e5c5506 296 if (other.fPackages) {
297 fPackages = new TObjArray();
298 TIter next(other.fPackages);
299 TObject *obj;
300 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
301 fPackages->SetOwner();
302 }
77f34eae 303 if (other.fModules) {
304 fModules = new TObjArray();
305 fModules->SetOwner();
306 TIter next(other.fModules);
307 AliAnalysisTaskCfg *mod, *crt;
308 while ((crt=(AliAnalysisTaskCfg*)next())) {
309 mod = new AliAnalysisTaskCfg(*crt);
310 fModules->Add(mod);
311 }
312 }
c57f56b7 313}
314
315//______________________________________________________________________________
316AliAnalysisAlien::~AliAnalysisAlien()
317{
318// Destructor.
77f34eae 319 delete fGridJDL;
320 delete fMergingJDL;
321 delete fInputFiles;
322 delete fPackages;
323 delete fModules;
28ce6c9c 324 fProofParam.DeleteAll();
c57f56b7 325}
326
327//______________________________________________________________________________
328AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
329{
330// Assignment.
331 if (this != &other) {
332 AliAnalysisGrid::operator=(other);
333 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 334 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 335 fPrice = other.fPrice;
336 fTTL = other.fTTL;
337 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
338 fMaxInitFailed = other.fMaxInitFailed;
339 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 340 fNtestFiles = other.fNtestFiles;
a3e84053 341 fNrunsPerMaster = other.fNrunsPerMaster;
342 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 343 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 344 fNsubmitted = other.fNsubmitted;
345 fProductionMode = other.fProductionMode;
cd11251e 346 fOutputToRunNo = other.fOutputToRunNo;
0f389141 347 fMergeViaJDL = other.fMergeViaJDL;
348 fFastReadOption = other.fFastReadOption;
349 fOverwriteMode = other.fOverwriteMode;
149d288c 350 fNreplicas = other.fNreplicas;
3bdcb562 351 fNproofWorkers = other.fNproofWorkers;
27734f0e 352 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 353 fProofReset = other.fProofReset;
c57f56b7 354 fRunNumbers = other.fRunNumbers;
355 fExecutable = other.fExecutable;
0a1c1f7f 356 fExecutableCommand = other.fExecutableCommand;
c57f56b7 357 fArguments = other.fArguments;
631c0b05 358 fExecutableArgs = other.fExecutableArgs;
c57f56b7 359 fAnalysisMacro = other.fAnalysisMacro;
360 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 361 fValidationScript = other.fValidationScript;
d5c6455a 362 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 363 fAdditionalLibs = other.fAdditionalLibs;
364 fSplitMode = other.fSplitMode;
365 fAPIVersion = other.fAPIVersion;
366 fROOTVersion = other.fROOTVersion;
367 fAliROOTVersion = other.fAliROOTVersion;
648174cf 368 fExternalPackages = other.fExternalPackages;
c57f56b7 369 fUser = other.fUser;
370 fGridWorkingDir = other.fGridWorkingDir;
371 fGridDataDir = other.fGridDataDir;
372 fDataPattern = other.fDataPattern;
373 fGridOutputDir = other.fGridOutputDir;
374 fOutputArchive = other.fOutputArchive;
375 fOutputFiles = other.fOutputFiles;
376 fInputFormat = other.fInputFormat;
e7c71df0 377 fDatasetName = other.fDatasetName;
c57f56b7 378 fJDLName = other.fJDLName;
d3b18c4c 379 fTerminateFiles = other.fTerminateFiles;
bb885a9e 380 fMergeExcludes = other.fMergeExcludes;
37d21c01 381 fRegisterExcludes = other.fRegisterExcludes;
f965131e 382 fIncludePath = other.fIncludePath;
bb885a9e 383 fCloseSE = other.fCloseSE;
0df6ccf2 384 fFriendChainName = other.fFriendChainName;
c6cb3634 385 fJobTag = other.fJobTag;
648174cf 386 fOutputSingle = other.fOutputSingle;
5fce53f4 387 fRunPrefix = other.fRunPrefix;
3bdcb562 388 fProofCluster = other.fProofCluster;
389 fProofDataSet = other.fProofDataSet;
390 fFileForTestMode = other.fFileForTestMode;
391 fRootVersionForProof = other.fRootVersionForProof;
392 fAliRootMode = other.fAliRootMode;
f47d5cb4 393 fMergeDirName = other.fMergeDirName;
c57f56b7 394 if (other.fInputFiles) {
395 fInputFiles = new TObjArray();
396 TIter next(other.fInputFiles);
397 TObject *obj;
398 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
399 fInputFiles->SetOwner();
400 }
4e5c5506 401 if (other.fPackages) {
402 fPackages = new TObjArray();
403 TIter next(other.fPackages);
404 TObject *obj;
405 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
406 fPackages->SetOwner();
407 }
77f34eae 408 if (other.fModules) {
409 fModules = new TObjArray();
410 fModules->SetOwner();
411 TIter next(other.fModules);
412 AliAnalysisTaskCfg *mod, *crt;
413 while ((crt=(AliAnalysisTaskCfg*)next())) {
414 mod = new AliAnalysisTaskCfg(*crt);
415 fModules->Add(mod);
416 }
417 }
c57f56b7 418 }
419 return *this;
420}
421
37d21c01 422//______________________________________________________________________________
423void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
424{
425// Add a single additional library to be loaded. Extension must be present.
426 TString lib(name);
427 if (!lib.Contains(".")) {
428 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
429 return;
430 }
431 if (fAdditionalLibs.Contains(name)) {
432 Warning("AddAdditionalLibrary", "Library %s already added.", name);
433 return;
434 }
435 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
436 fAdditionalLibs += lib;
437}
438
77f34eae 439//______________________________________________________________________________
440void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
441{
442// Adding a module. Checks if already existing. Becomes owned by this.
443 if (!module) return;
444 if (GetModule(module->GetName())) {
445 Error("AddModule", "A module having the same name %s already added", module->GetName());
446 return;
447 }
448 if (!fModules) {
449 fModules = new TObjArray();
450 fModules->SetOwner();
451 }
452 fModules->Add(module);
453}
454
455//______________________________________________________________________________
456void AliAnalysisAlien::AddModules(TObjArray *list)
457{
458// Adding a list of modules. Checks if already existing. Becomes owned by this.
459 TIter next(list);
460 AliAnalysisTaskCfg *module;
461 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
462}
463
464//______________________________________________________________________________
465Bool_t AliAnalysisAlien::CheckDependencies()
466{
467// Check if all dependencies are satisfied. Reorder modules if needed.
468 Int_t nmodules = GetNmodules();
469 if (!nmodules) {
470 Warning("CheckDependencies", "No modules added yet to check their dependencies");
471 return kTRUE;
472 }
473 AliAnalysisTaskCfg *mod = 0;
474 AliAnalysisTaskCfg *dep = 0;
475 TString depname;
476 Int_t i, j, k;
477 for (i=0; i<nmodules; i++) {
478 mod = (AliAnalysisTaskCfg*) fModules->At(i);
479 Int_t ndeps = mod->GetNdeps();
480 Int_t istart = i;
481 for (j=0; j<ndeps; j++) {
482 depname = mod->GetDependency(j);
483 dep = GetModule(depname);
484 if (!dep) {
485 Error("CheckDependencies","Dependency %s not added for module %s",
486 depname.Data(), mod->GetName());
487 return kFALSE;
488 }
489 if (dep->NeedsDependency(mod->GetName())) {
490 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
491 mod->GetName(), dep->GetName());
492 return kFALSE;
493 }
494 Int_t idep = fModules->IndexOf(dep);
495 // The dependency task must come first
496 if (idep>i) {
497 // Remove at idep and move all objects below up one slot
498 // down to index i included.
499 fModules->RemoveAt(idep);
3e40fd4c 500 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 501 fModules->AddAt(dep, i++);
502 }
503 //Redo from istart if dependencies were inserted
504 if (i>istart) i=istart-1;
505 }
506 }
507 return kTRUE;
508}
509
3e40fd4c 510//______________________________________________________________________________
511AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
512{
513// Create the analysis manager and optionally execute the macro in filename.
514 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
515 if (mgr) return mgr;
516 mgr = new AliAnalysisManager(name);
517 mgr->SetGridHandler((AliAnalysisGrid*)this);
518 if (strlen(filename)) {
519 TString line = gSystem->ExpandPathName(filename);
520 line.Prepend(".x ");
521 gROOT->ProcessLine(line.Data());
522 }
523 return mgr;
524}
525
77f34eae 526//______________________________________________________________________________
527Int_t AliAnalysisAlien::GetNmodules() const
528{
529// Get number of modules.
530 if (!fModules) return 0;
531 return fModules->GetEntries();
532}
533
534//______________________________________________________________________________
535AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
536{
537// Get a module by name.
538 if (!fModules) return 0;
539 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
540}
541
542//______________________________________________________________________________
543Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
544{
545// Load a given module.
546 if (mod->IsLoaded()) return kTRUE;
547 Int_t ndeps = mod->GetNdeps();
548 TString depname;
549 for (Int_t j=0; j<ndeps; j++) {
550 depname = mod->GetDependency(j);
551 AliAnalysisTaskCfg *dep = GetModule(depname);
552 if (!dep) {
553 Error("LoadModule","Dependency %s not existing for module %s",
554 depname.Data(), mod->GetName());
555 return kFALSE;
556 }
557 if (!LoadModule(dep)) {
558 Error("LoadModule","Dependency %s for module %s could not be loaded",
559 depname.Data(), mod->GetName());
560 return kFALSE;
561 }
562 }
563 // Load libraries for the module
564 if (!mod->CheckLoadLibraries()) {
565 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
566 return kFALSE;
567 }
568 // Execute the macro
569 if (mod->ExecuteMacro()<0) {
570 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
571 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
572 return kFALSE;
573 }
574 // Configure dependencies
575 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
576 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
577 mod->GetConfigMacro()->GetTitle(), mod->GetName());
578 return kFALSE;
579 }
5e8c7c22 580 // Adjust extra libraries
581 Int_t nlibs = mod->GetNlibs();
582 TString lib;
583 for (Int_t i=0; i<nlibs; i++) {
584 lib = mod->GetLibrary(i);
585 if (fAdditionalLibs.Contains(lib)) continue;
586 lib = Form("lib%s.so", lib.Data());
587 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
588 fAdditionalLibs += lib;
589 }
77f34eae 590 return kTRUE;
591}
592
139fe6ba 593//______________________________________________________________________________
594Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
595{
596// Generate the full train.
597 fAdditionalLibs = "";
598 if (!LoadModules()) return kFALSE;
599 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
600 if (!mgr->InitAnalysis()) return kFALSE;
601 mgr->PrintStatus();
602 Int_t productionMode = fProductionMode;
603 SetProductionMode();
604 TString macro = fAnalysisMacro;
605 TString executable = fExecutable;
606 TString validation = fValidationScript;
607 TString execCommand = fExecutableCommand;
608 SetAnalysisMacro(Form("%s.C", name));
609 SetExecutable(Form("%s.sh", name));
610 SetExecutableCommand("aliroot -b -q ");
611 SetValidationScript(Form("%s_validation.sh", name));
612 StartAnalysis();
613 SetProductionMode(productionMode);
614 fAnalysisMacro = macro;
615 fExecutable = executable;
616 fExecutableCommand = execCommand;
617 fValidationScript = validation;
618 return kTRUE;
619}
620
77f34eae 621//______________________________________________________________________________
21ca8e59 622Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 623{
624// Generate test macros for a single module or for the full train.
5e8c7c22 625 fAdditionalLibs = "";
77f34eae 626 if (strlen(modname)) {
21ca8e59 627 if (!CheckDependencies()) return kFALSE;
628 AliAnalysisTaskCfg *mod = GetModule(modname);
629 if (!mod) {
630 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
631 return kFALSE;
632 }
633 if (!LoadModule(mod)) return kFALSE;
634 } else if (!LoadModules()) return kFALSE;
635 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
636 if (!mgr->InitAnalysis()) return kFALSE;
637 mgr->PrintStatus();
638 SetLocalTest(kTRUE);
639 Int_t productionMode = fProductionMode;
640 SetProductionMode();
641 TString macro = fAnalysisMacro;
642 TString executable = fExecutable;
643 TString validation = fValidationScript;
644 TString execCommand = fExecutableCommand;
645 SetAnalysisMacro(Form("%s.C", name));
646 SetExecutable(Form("%s.sh", name));
647 SetExecutableCommand("aliroot -b -q ");
648 SetValidationScript(Form("%s_validation.sh", name));
649 WriteAnalysisFile();
650 WriteAnalysisMacro();
651 WriteExecutable();
652 WriteValidationScript();
653 SetLocalTest(kFALSE);
654 SetProductionMode(productionMode);
655 fAnalysisMacro = macro;
656 fExecutable = executable;
657 fExecutableCommand = execCommand;
658 fValidationScript = validation;
77f34eae 659 return kTRUE;
660}
661
662//______________________________________________________________________________
663Bool_t AliAnalysisAlien::LoadModules()
664{
665// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 666 fAdditionalLibs = "";
21ca8e59 667 Int_t nmodules = GetNmodules();
668 if (!nmodules) {
669 Warning("LoadModules", "No module to be loaded");
670 return kTRUE;
671 }
3e40fd4c 672 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
673 if (!mgr) {
674 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
675 return kFALSE;
676 }
77f34eae 677 if (!CheckDependencies()) return kFALSE;
21ca8e59 678 nmodules = GetNmodules();
77f34eae 679 AliAnalysisTaskCfg *mod;
21ca8e59 680 for (Int_t imod=0; imod<nmodules; imod++) {
681 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 682 if (!LoadModule(mod)) return kFALSE;
683 }
77f34eae 684 return kTRUE;
685}
686
74013535 687//______________________________________________________________________________
688void AliAnalysisAlien::SetRunPrefix(const char *prefix)
689{
690// Set the run number format. Can be a prefix or a format like "%09d"
691 fRunPrefix = prefix;
692 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
693}
694
f965131e 695//______________________________________________________________________________
696void AliAnalysisAlien::AddIncludePath(const char *path)
697{
698// Add include path in the remote analysis macro.
699 TString p(path);
700 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
701 else fIncludePath += Form("-I%s ", path);
702}
703
c57f56b7 704//______________________________________________________________________________
705void AliAnalysisAlien::AddRunNumber(Int_t run)
706{
707// Add a run number to the list of runs to be processed.
708 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 709 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 710}
711
98690895 712//______________________________________________________________________________
713void AliAnalysisAlien::AddRunList(const char* runList)
714{
715// Add several runs into the list of runs; they are expected to be separated by a blank character.
716 TString sList = runList;
717 TObjArray *list = sList.Tokenize(" ");
718 Int_t n = list->GetEntries();
719 for (Int_t i = 0; i < n; i++) {
720 TObjString *os = (TObjString*)list->At(i);
721 AddRunNumber(os->GetString().Atoi());
722 }
723 delete list;
724}
725
ee75cfc3 726//______________________________________________________________________________
727void AliAnalysisAlien::AddRunNumber(const char* run)
728{
729// Add a run number to the list of runs to be processed.
7aa5d539 730 TString runs = run;
731 TObjString *os;
732 TObjArray *arr = runs.Tokenize(" ");
733 TIter next(arr);
734 TString prefix;
735 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
736 while ((os=(TObjString*)next())){
737 if (fRunNumbers.Length()) fRunNumbers += " ";
738 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
739 }
740 delete arr;
ee75cfc3 741}
742
c57f56b7 743//______________________________________________________________________________
744void AliAnalysisAlien::AddDataFile(const char *lfn)
745{
746// Adds a data file to the input to be analysed. The file should be a valid LFN
747// or point to an existing file in the alien workdir.
748 if (!fInputFiles) fInputFiles = new TObjArray();
749 fInputFiles->Add(new TObjString(lfn));
750}
648174cf 751
752//______________________________________________________________________________
753void AliAnalysisAlien::AddExternalPackage(const char *package)
754{
755// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
756 if (fExternalPackages) fExternalPackages += " ";
757 fExternalPackages += package;
758}
759
c57f56b7 760//______________________________________________________________________________
761Bool_t AliAnalysisAlien::Connect()
762{
763// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
764 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 765 if (fProductionMode) return kTRUE;
c57f56b7 766 if (!gGrid) {
767 Info("Connect", "Trying to connect to AliEn ...");
768 TGrid::Connect("alien://");
769 }
770 if (!gGrid || !gGrid->IsConnected()) {
771 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
772 return kFALSE;
773 }
774 fUser = gGrid->GetUser();
775 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
776 return kTRUE;
777}
778
779//______________________________________________________________________________
780void AliAnalysisAlien::CdWork()
781{
782// Check validity of alien workspace. Create directory if possible.
783 if (!Connect()) {
784 Error("CdWork", "Alien connection required");
785 return;
786 }
787 TString homedir = gGrid->GetHomeDirectory();
788 TString workdir = homedir + fGridWorkingDir;
923e2ca5 789 if (DirectoryExists(workdir)) {
790 gGrid->Cd(workdir);
791 return;
792 }
793 // Work directory not existing - create it
794 gGrid->Cd(homedir);
b93f8109 795 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 796 gGrid->Cd(fGridWorkingDir);
d3b18c4c 797 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 798 } else {
d3b18c4c 799 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 800 workdir.Data(), homedir.Data());
801 fGridWorkingDir = "";
802 }
c57f56b7 803}
804
348be253 805//______________________________________________________________________________
806Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
807{
808// Check if file copying is possible.
d3b18c4c 809 if (fProductionMode) return kTRUE;
348be253 810 if (!Connect()) {
811 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
812 return kFALSE;
813 }
d3339be3 814 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
815 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 816 // Check if alien_CLOSE_SE is defined
817 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
818 if (!closeSE.IsNull()) {
819 Info("CheckFileCopy", "Your current close storage is pointing to: \
820 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
821 } else {
822 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
823 }
824 // Check if grid directory exists.
825 if (!DirectoryExists(alienpath)) {
826 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
827 return kFALSE;
828 }
d3339be3 829 TFile f("plugin_test_copy", "RECREATE");
348be253 830 // User may not have write permissions to current directory
831 if (f.IsZombie()) {
832 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
833 gSystem->WorkingDirectory());
834 return kFALSE;
835 }
836 f.Close();
d3339be3 837 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
838 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
839 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 840 \n# 1. Make sure you have write permissions there. If this is the case: \
841 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
842 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
843 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
844 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
845 gSystem->Unlink(f.GetName());
846 return kFALSE;
847 }
848 gSystem->Unlink(f.GetName());
d3339be3 849 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
850 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 851 return kTRUE;
852}
853
c57f56b7 854//______________________________________________________________________________
855Bool_t AliAnalysisAlien::CheckInputData()
856{
857// Check validity of input data. If necessary, create xml files.
d3b18c4c 858 if (fProductionMode) return kTRUE;
a8739e8a 859 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 860 if (!fGridDataDir.Length()) {
861 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
862 return kFALSE;
863 }
9f020952 864 if (fMergeViaJDL) {
865 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
866 return kFALSE;
867 }
d2a409b2 868 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 869 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
870 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 871 return kTRUE;
c57f56b7 872 }
873 // Process declared files
a2f5fc01 874 Bool_t isCollection = kFALSE;
875 Bool_t isXml = kFALSE;
876 Bool_t useTags = kFALSE;
c57f56b7 877 Bool_t checked = kFALSE;
d3b18c4c 878 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 879 TString file;
880 TString workdir = gGrid->GetHomeDirectory();
881 workdir += fGridWorkingDir;
882 if (fInputFiles) {
883 TObjString *objstr;
884 TIter next(fInputFiles);
885 while ((objstr=(TObjString*)next())) {
886 file = workdir;
887 file += "/";
888 file += objstr->GetString();
889 // Store full lfn path
890 if (FileExists(file)) objstr->SetString(file);
891 else {
892 file = objstr->GetName();
893 if (!FileExists(objstr->GetName())) {
894 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
895 objstr->GetName(), workdir.Data());
896 return kFALSE;
897 }
898 }
899 Bool_t iscoll, isxml, usetags;
900 CheckDataType(file, iscoll, isxml, usetags);
901 if (!checked) {
902 checked = kTRUE;
a2f5fc01 903 isCollection = iscoll;
904 isXml = isxml;
905 useTags = usetags;
906 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 907 } else {
a2f5fc01 908 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 909 Error("CheckInputData", "Some conflict was found in the types of inputs");
910 return kFALSE;
911 }
912 }
913 }
914 }
915 // Process requested run numbers
a8739e8a 916 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 917 // Check validity of alien data directory
918 if (!fGridDataDir.Length()) {
919 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
920 return kFALSE;
921 }
923e2ca5 922 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 923 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
924 return kFALSE;
925 }
a2f5fc01 926 if (isCollection) {
c57f56b7 927 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
928 return kFALSE;
929 }
930
a2f5fc01 931 if (checked && !isXml) {
c57f56b7 932 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
933 return kFALSE;
934 }
935 // Check validity of run number(s)
936 TObjArray *arr;
937 TObjString *os;
74013535 938 TString format;
319593fb 939 Int_t nruns = 0;
904f9f5f 940 TString schunk, schunk2;
c57f56b7 941 TString path;
942 if (!checked) {
943 checked = kTRUE;
a2f5fc01 944 useTags = fDataPattern.Contains("tag");
945 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 946 }
a2f5fc01 947 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 948 Error("CheckInputData", "Cannot mix input files using/not using tags");
949 return kFALSE;
950 }
951 if (fRunNumbers.Length()) {
a8739e8a 952 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 953 arr = fRunNumbers.Tokenize(" ");
954 TIter next(arr);
955 while ((os=(TObjString*)next())) {
956 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 957 if (!DirectoryExists(path)) {
958 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 959 continue;
c57f56b7 960 }
961 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
962 TString msg = "\n##### file: ";
963 msg += path;
964 msg += " type: xml_collection;";
a2f5fc01 965 if (useTags) msg += " using_tags: Yes";
c57f56b7 966 else msg += " using_tags: No";
23329835 967 Info("CheckDataType", "%s", msg.Data());
319593fb 968 if (fNrunsPerMaster<2) {
d2a409b2 969 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 970 } else {
971 nruns++;
972 if (((nruns-1)%fNrunsPerMaster) == 0) {
973 schunk = os->GetString();
974 }
975 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
976 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 977 AddDataFile(schunk);
319593fb 978 }
c57f56b7 979 }
980 delete arr;
a8739e8a 981 } else {
982 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
983 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 984 format = Form("%%s/%s ", fRunPrefix.Data());
985 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 986 if (!DirectoryExists(path)) {
a8739e8a 987 continue;
988 }
74013535 989 format = Form("%%s/%s.xml", fRunPrefix.Data());
990 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 991 TString msg = "\n##### file: ";
992 msg += path;
993 msg += " type: xml_collection;";
a2f5fc01 994 if (useTags) msg += " using_tags: Yes";
a8739e8a 995 else msg += " using_tags: No";
23329835 996 Info("CheckDataType", "%s", msg.Data());
319593fb 997 if (fNrunsPerMaster<2) {
74013535 998 format = Form("%s.xml", fRunPrefix.Data());
999 AddDataFile(Form(format.Data(),irun));
319593fb 1000 } else {
1001 nruns++;
1002 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1003 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1004 }
74013535 1005 format = Form("_%s.xml", fRunPrefix.Data());
1006 schunk2 = Form(format.Data(), irun);
319593fb 1007 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1008 schunk += schunk2;
d2a409b2 1009 AddDataFile(schunk);
319593fb 1010 }
a8739e8a 1011 }
904f9f5f 1012 if (!fInputFiles) {
1013 schunk += schunk2;
1014 AddDataFile(schunk);
1015 }
c57f56b7 1016 }
1017 return kTRUE;
1018}
1019
21ca8e59 1020//______________________________________________________________________________
43b807f6 1021Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1022{
1023// Copy data from the given grid directory according a pattern and make a local
1024// dataset.
1025 if (!Connect()) {
1026 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
1027 return kFALSE;
1028 }
1029 if (!DirectoryExists(griddir)) {
1030 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
1031 return kFALSE;
1032 }
1033 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1034 printf("Running command: %s\n", command.Data());
1035 TGridResult *res = gGrid->Command(command);
1036 Int_t nfound = res->GetEntries();
1037 if (!nfound) {
1038 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
1039 return kFALSE;
1040 }
1041 printf("... found %d files. Copying locally ...\n", nfound);
1042 // Copy files locally
1043 ofstream out;
1044 out.open(output, ios::out);
1045 TMap *map;
1046 TString turl, dirname, filename, temp;
1047 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1048 gSystem->MakeDirectory(outputdir);
1049 gSystem->ChangeDirectory(outputdir);
21ca8e59 1050 for (Int_t i=0; i<nfound; i++) {
1051 map = (TMap*)res->At(i);
1052 turl = map->GetValue("turl")->GetName();
1053 filename = gSystem->BaseName(turl.Data());
1054 dirname = gSystem->DirName(turl.Data());
1055 dirname = gSystem->BaseName(dirname.Data());
1056 gSystem->MakeDirectory(dirname);
43b807f6 1057
1058 TString source(turl);
1059 TString targetFileName(filename);
1060
1061 if (strlen(archivefile) > 0) {
1062 // TODO here the archive in which the file resides should be determined
1063 // however whereis returns only a guid, and guid2lfn does not work
1064 // Therefore we use the one provided as argument for now
1065 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1066 targetFileName = archivefile;
1067 }
1068 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
1069 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1070 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
21ca8e59 1071 }
1072 }
1073 gSystem->ChangeDirectory(cdir);
1074 delete res;
1075 return kTRUE;
1076}
1077
c57f56b7 1078//______________________________________________________________________________
1079Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1080{
1081// Create dataset for the grid data directory + run number.
3af861ff 1082 const Int_t gMaxEntries = 15000;
d3b18c4c 1083 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1084 if (!Connect()) {
1085 Error("CreateDataset", "Cannot create dataset with no grid connection");
1086 return kFALSE;
1087 }
1088
1089 // Cd workspace
d3b18c4c 1090 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1091 TString workdir = gGrid->GetHomeDirectory();
1092 workdir += fGridWorkingDir;
1093
1094 // Compose the 'find' command arguments
74013535 1095 TString format;
c57f56b7 1096 TString command;
1097 TString options = "-x collection ";
bb885a9e 1098 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1099 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1100 TString conditions = "";
3af861ff 1101 Int_t nstart = 0;
1102 Int_t ncount = 0;
1103 Int_t stage = 0;
c57f56b7 1104 TString file;
1105 TString path;
319593fb 1106 Int_t nruns = 0;
904f9f5f 1107 TString schunk, schunk2;
ab254fd1 1108 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1109 if (!fRunNumbers.Length() && !fRunRange[0]) {
1110 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1111 // Make a single data collection from data directory.
1112 path = fGridDataDir;
923e2ca5 1113 if (!DirectoryExists(path)) {
d2a409b2 1114 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1115 return kFALSE;
1116 }
923e2ca5 1117// CdWork();
d2a409b2 1118 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1119 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1120 while (1) {
1121 ncount = 0;
1122 stage++;
1123 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1124 command = "find ";
1125 command += Form("%s -o %d ",options.Data(), nstart);
1126 command += path;
1127 command += " ";
1128 command += pattern;
1129 command += conditions;
1130 printf("command: %s\n", command.Data());
1131 TGridResult *res = gGrid->Command(command);
1132 if (res) delete res;
1133 // Write standard output to file
1134 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1135 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1136 Bool_t nullFile = kFALSE;
1137 if (!hasGrep) {
1138 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1139 } else {
1140 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1141 if (nullFile) {
1142 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1143 gSystem->Exec("rm -f __tmp*");
1144 return kFALSE;
1145 }
1146 TString line;
1147 ifstream in;
1148 in.open("__tmp__");
1149 in >> line;
1150 in.close();
1151 gSystem->Exec("rm -f __tmp__");
1152 ncount = line.Atoi();
1153 }
1154 }
1155 if (ncount == gMaxEntries) {
1156 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1157 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1158 if (!cbase) cbase = cadd;
1159 else {
1160 cbase->Add(cadd);
1161 delete cadd;
defd7a3a 1162 }
3af861ff 1163 nstart += ncount;
1164 } else {
1165 if (cbase) {
1166 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1167 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1168 cbase->Add(cadd);
1169 delete cadd;
1170 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1171 delete cbase; cbase = 0;
1172 } else {
1173 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1174 }
1175 gSystem->Exec("rm -f __tmp*");
1176 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1177 break;
1178 }
0f389141 1179 }
1180 Bool_t fileExists = FileExists(file);
1181 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1182 // Copy xml file to alien space
0f389141 1183 if (fileExists) gGrid->Rm(file);
d2a409b2 1184 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1185 if (!FileExists(file)) {
1186 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1187 return kFALSE;
1188 }
1189 // Update list of files to be processed.
1190 }
1191 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1192 return kTRUE;
1193 }
c57f56b7 1194 // Several runs
a2f5fc01 1195 Bool_t nullResult = kTRUE;
a8739e8a 1196 if (fRunNumbers.Length()) {
1197 TObjArray *arr = fRunNumbers.Tokenize(" ");
1198 TObjString *os;
1199 TIter next(arr);
1200 while ((os=(TObjString*)next())) {
3af861ff 1201 nstart = 0;
1202 stage = 0;
dc030a24 1203 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1204 if (!DirectoryExists(path)) continue;
1205// CdWork();
a8739e8a 1206 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1207 else file = Form("%s.xml", os->GetString().Data());
319593fb 1208 // If local collection file does not exist, create it via 'find' command.
3af861ff 1209 while (1) {
1210 ncount = 0;
1211 stage++;
1212 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1213 command = "find ";
1214 command += Form("%s -o %d ",options.Data(), nstart);
1215 command += path;
1216 command += pattern;
1217 command += conditions;
1218 TGridResult *res = gGrid->Command(command);
1219 if (res) delete res;
1220 // Write standard output to file
1221 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1222 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1223 Bool_t nullFile = kFALSE;
1224 if (!hasGrep) {
1225 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1226 } else {
1227 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1228 if (nullFile) {
1229 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1230 gSystem->Exec("rm -f __tmp*");
1231 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1232 break;
3af861ff 1233 }
1234 TString line;
1235 ifstream in;
1236 in.open("__tmp__");
1237 in >> line;
1238 in.close();
1239 gSystem->Exec("rm -f __tmp__");
1240 ncount = line.Atoi();
1241 }
1242 nullResult = kFALSE;
1243 }
1244 if (ncount == gMaxEntries) {
1245 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1246 if (fNrunsPerMaster > 1) {
1247 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1248 file.Data(),gMaxEntries);
1249 return kFALSE;
1250 }
1251 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1252 if (!cbase) cbase = cadd;
1253 else {
1254 cbase->Add(cadd);
1255 delete cadd;
defd7a3a 1256 }
3af861ff 1257 nstart += ncount;
1258 } else {
c89a84bc 1259 if (cbase && fNrunsPerMaster<2) {
3af861ff 1260 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1261 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1262 cbase->Add(cadd);
1263 delete cadd;
1264 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1265 delete cbase; cbase = 0;
1266 } else {
1267 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1268 }
1269 gSystem->Exec("rm -f __tmp*");
1270 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1271 break;
d3339be3 1272 }
3af861ff 1273 }
a8739e8a 1274 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1275 // Check if there is one run per master job.
1276 if (fNrunsPerMaster<2) {
1277 if (FileExists(file)) {
0f389141 1278 if (fOverwriteMode) gGrid->Rm(file);
1279 else {
1280 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1281 continue;
1282 }
319593fb 1283 }
1284 // Copy xml file to alien space
1285 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1286 if (!FileExists(file)) {
1287 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1288 delete arr;
1289 return kFALSE;
1290 }
1291 } else {
1292 nruns++;
1293 if (((nruns-1)%fNrunsPerMaster) == 0) {
1294 schunk = os->GetString();
1295 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1296 } else {
1297 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1298 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1299 cbase->Add(cadd);
1300 delete cadd;
1301 }
1302 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1303 continue;
1304 }
1305 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1306 if (FileExists(schunk)) {
1307 if (fOverwriteMode) gGrid->Rm(file);
1308 else {
1309 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1310 continue;
1311 }
319593fb 1312 }
84fcd93f 1313 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1314 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1315 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1316 if (!FileExists(schunk)) {
1317 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1318 delete arr;
1319 return kFALSE;
1320 }
d3339be3 1321 }
a8739e8a 1322 }
1323 delete arr;
a2f5fc01 1324 if (nullResult) {
d3339be3 1325 Error("CreateDataset", "No valid dataset corresponding to the query!");
1326 return kFALSE;
1327 }
a8739e8a 1328 } else {
1329 // Process a full run range.
1330 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1331 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1332 nstart = 0;
1333 stage = 0;
74013535 1334 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1335 if (!DirectoryExists(path)) continue;
1336// CdWork();
74013535 1337 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1338 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1339 else file = Form(format.Data(), irun);
0f389141 1340 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1341 if (fOverwriteMode) gGrid->Rm(file);
1342 else {
1343 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1344 continue;
1345 }
a8739e8a 1346 }
319593fb 1347 // If local collection file does not exist, create it via 'find' command.
3af861ff 1348 while (1) {
1349 ncount = 0;
1350 stage++;
1351 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1352 command = "find ";
1353 command += Form("%s -o %d ",options.Data(), nstart);
1354 command += path;
1355 command += pattern;
1356 command += conditions;
1357 TGridResult *res = gGrid->Command(command);
1358 if (res) delete res;
1359 // Write standard output to file
1360 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1361 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1362 Bool_t nullFile = kFALSE;
1363 if (!hasGrep) {
1364 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1365 } else {
1366 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1367 if (nullFile) {
1368 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1369 gSystem->Exec("rm -f __tmp*");
1604c9ed 1370 break;
3af861ff 1371 }
1372 TString line;
1373 ifstream in;
1374 in.open("__tmp__");
1375 in >> line;
1376 in.close();
1377 gSystem->Exec("rm -f __tmp__");
1378 ncount = line.Atoi();
1379 }
1380 nullResult = kFALSE;
1381 }
1382 if (ncount == gMaxEntries) {
1383 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1384 if (fNrunsPerMaster > 1) {
1385 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1386 file.Data(),gMaxEntries);
1387 return kFALSE;
1388 }
1389 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1390 if (!cbase) cbase = cadd;
1391 else {
1392 cbase->Add(cadd);
1393 delete cadd;
defd7a3a 1394 }
3af861ff 1395 nstart += ncount;
1396 } else {
c89a84bc 1397 if (cbase && fNrunsPerMaster<2) {
3af861ff 1398 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1399 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1400 cbase->Add(cadd);
1401 delete cadd;
1402 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1403 delete cbase; cbase = 0;
1404 } else {
1405 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1406 }
1407 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1408 break;
d3339be3 1409 }
319593fb 1410 }
a8739e8a 1411 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1412 // Check if there is one run per master job.
1413 if (fNrunsPerMaster<2) {
1414 if (FileExists(file)) {
0f389141 1415 if (fOverwriteMode) gGrid->Rm(file);
1416 else {
1417 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1418 continue;
1419 }
319593fb 1420 }
1421 // Copy xml file to alien space
1422 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1423 if (!FileExists(file)) {
1424 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1425 return kFALSE;
1426 }
1427 } else {
1428 nruns++;
95e5b448 1429 // Check if the collection for the chunk exist locally.
1430 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1431 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1432 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1433 else continue;
1434 }
84fcd93f 1435 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1436 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1437 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1438 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1439 } else {
1440 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1441 cbase->Add(cadd);
1442 delete cadd;
1443 }
74013535 1444 format = Form("%%s_%s.xml", fRunPrefix.Data());
1445 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1446 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1447 continue;
1448 }
904f9f5f 1449 schunk = schunk2;
319593fb 1450 if (FileExists(schunk)) {
0f389141 1451 if (fOverwriteMode) gGrid->Rm(schunk);
1452 else {
1453 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1454 continue;
1455 }
319593fb 1456 }
84fcd93f 1457 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1458 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1459 if (FileExists(schunk)) {
0f389141 1460 if (fOverwriteMode) gGrid->Rm(schunk);
1461 else {
1462 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1463 continue;
1464 }
95e5b448 1465 }
319593fb 1466 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1467 if (!FileExists(schunk)) {
1468 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1469 return kFALSE;
1470 }
1471 }
c57f56b7 1472 }
a2f5fc01 1473 if (nullResult) {
d3339be3 1474 Error("CreateDataset", "No valid dataset corresponding to the query!");
1475 return kFALSE;
1476 }
a8739e8a 1477 }
c57f56b7 1478 return kTRUE;
1479}
1480
1481//______________________________________________________________________________
1482Bool_t AliAnalysisAlien::CreateJDL()
1483{
1484// Generate a JDL file according to current settings. The name of the file is
1485// specified by fJDLName.
1486 Bool_t error = kFALSE;
1487 TObjArray *arr = 0;
1488 Bool_t copy = kTRUE;
d3b18c4c 1489 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1490 Bool_t generate = kTRUE;
1491 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1492 if (!Connect()) {
1493 Error("CreateJDL", "Alien connection required");
1494 return kFALSE;
1495 }
1496 // Check validity of alien workspace
d3b18c4c 1497 TString workdir;
1498 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1499 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1500 workdir += fGridWorkingDir;
1501 if (generate) {
1502 TObjString *os;
1503 if (!fInputFiles) {
1504 Error("CreateJDL()", "Define some input files for your analysis.");
1505 error = kTRUE;
1506 }
1507 // Compose list of input files
1508 // Check if output files were defined
1509 if (!fOutputFiles.Length()) {
1510 Error("CreateJDL", "You must define at least one output file");
1511 error = kTRUE;
1512 }
1513 // Check if an output directory was defined and valid
1514 if (!fGridOutputDir.Length()) {
1515 Error("CreateJDL", "You must define AliEn output directory");
1516 error = kTRUE;
1517 } else {
d3b18c4c 1518 if (!fProductionMode) {
1519 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1520 if (!DirectoryExists(fGridOutputDir)) {
1521 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1522 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1523 } else {
1524 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1525 // error = kTRUE;
1526 }
67a3b254 1527 } else {
1528 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1529 }
1530 gGrid->Cd(workdir);
1531 }
c57f56b7 1532 }
1533 // Exit if any error up to now
1534 if (error) return kFALSE;
1535 // Set JDL fields
0f389141 1536 if (!fUser.IsNull()) {
1537 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1538 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1539 }
1540 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1541 TString mergeExec = fExecutable;
1542 mergeExec.ReplaceAll(".sh", "_merge.sh");
1543 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1544 mergeExec.ReplaceAll(".sh", ".C");
1545 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1546 if (!fArguments.IsNull())
1547 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1548 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1549 else {
1550 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1551 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1552 }
b3e07543 1553
1f0d1ca2 1554 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1555 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1556 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1557 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1558
0f389141 1559 if (fMaxInitFailed > 0) {
c57f56b7 1560 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1561 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1562 }
1563 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1564 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1565 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1566 }
1567 if (!IsOneStageMerging()) {
f47d5cb4 1568 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1569 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1570 }
1571 if (fSplitMode.Length()) {
c57f56b7 1572 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1573 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1574 }
1575 fMergingJDL->SetValue("Split", "\"se\"");
1576 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1577 if (!fAliROOTVersion.IsNull()) {
1578 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1579 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1580 }
1581 if (!fROOTVersion.IsNull()) {
c57f56b7 1582 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1583 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1584 }
1585 if (!fAPIVersion.IsNull()) {
c57f56b7 1586 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1587 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1588 }
648174cf 1589 if (!fExternalPackages.IsNull()) {
1590 arr = fExternalPackages.Tokenize(" ");
1591 TIter next(arr);
1592 while ((os=(TObjString*)next())) {
1593 TString pkgname = os->GetString();
1594 Int_t index = pkgname.Index("::");
1595 TString pkgversion = pkgname(index+2, pkgname.Length());
1596 pkgname.Remove(index);
1597 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1598 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1599 }
1600 delete arr;
1601 }
0f389141 1602 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1603 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1604 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1605 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1606 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1607 TString analysisFile = fExecutable;
1608 analysisFile.ReplaceAll(".sh", ".root");
1609 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1610 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1611 if (fAdditionalLibs.Length()) {
1612 arr = fAdditionalLibs.Tokenize(" ");
1613 TIter next(arr);
1614 while ((os=(TObjString*)next())) {
1615 if (os->GetString().Contains(".so")) continue;
1616 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1617 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1618 }
1619 delete arr;
1620 }
4e5c5506 1621 if (fPackages) {
1622 TIter next(fPackages);
1623 TObject *obj;
0f389141 1624 while ((obj=next())) {
4e5c5506 1625 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1626 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1627 }
4e5c5506 1628 }
c57f56b7 1629 if (fOutputArchive.Length()) {
1630 arr = fOutputArchive.Tokenize(" ");
1631 TIter next(arr);
0f389141 1632 Bool_t first = kTRUE;
1633 const char *comment = "Files to be archived";
1634 const char *comment1 = comment;
1635 while ((os=(TObjString*)next())) {
1636 if (!first) comment = NULL;
1637 if (!os->GetString().Contains("@") && fCloseSE.Length())
1638 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1639 else
1640 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1641 first = kFALSE;
1642 }
c57f56b7 1643 delete arr;
f790bc1b 1644 // Output archive for the merging jdl
1645 TString outputArchive;
1646 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1647 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1648 // Add normal output files, extra files + terminate files
1649 TString files = GetListOfFiles("outextter");
37d21c01 1650 // Do not register files in fRegisterExcludes
1651 if (!fRegisterExcludes.IsNull()) {
1652 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1653 TIter next1(arr);
1654 while ((os=(TObjString*)next1())) {
1655 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1656 files.ReplaceAll(os->GetString(),"");
1657 }
66d71516 1658 delete arr;
0f389141 1659 }
f790bc1b 1660 files.ReplaceAll(".root", "*.root");
b83f84f0 1661 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1662 } else {
95f84779 1663 TString files = fOutputArchive;
1664 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1665 outputArchive = files;
f790bc1b 1666 }
0f389141 1667 arr = outputArchive.Tokenize(" ");
705adb3e 1668 TIter next2(arr);
0f389141 1669 comment = comment1;
1670 first = kTRUE;
705adb3e 1671 while ((os=(TObjString*)next2())) {
0f389141 1672 if (!first) comment = NULL;
0cdf65a8 1673 TString currentfile = os->GetString();
0cdf65a8 1674 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1675 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1676 else
0cdf65a8 1677 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1678 first = kFALSE;
1679 }
1680 delete arr;
c57f56b7 1681 }
149d288c 1682 arr = fOutputFiles.Tokenize(",");
c57f56b7 1683 TIter next(arr);
0f389141 1684 Bool_t first = kTRUE;
f790bc1b 1685 const char *comment = "Files to be saved";
43da816a 1686 while ((os=(TObjString*)next())) {
e1eaf596 1687 // Ignore ouputs in jdl that are also in outputarchive
1688 TString sout = os->GetString();
f790bc1b 1689 sout.ReplaceAll("*", "");
1690 sout.ReplaceAll(".root", "");
e1eaf596 1691 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1692 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1693 // Ignore fRegisterExcludes
1694 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1695 if (!first) comment = NULL;
43da816a 1696 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1697 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1698 else
0f389141 1699 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1700 first = kFALSE;
f790bc1b 1701 if (fMergeExcludes.Contains(sout)) continue;
1702 if (!os->GetString().Contains("@") && fCloseSE.Length())
1703 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1704 else
1705 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1706 }
c57f56b7 1707 delete arr;
0f389141 1708 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1709 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1710 TString validationScript = fValidationScript;
0f389141 1711 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1712 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1713 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1714 if (fMasterResubmitThreshold) {
1715 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1716 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1717 }
d2a409b2 1718 // Write a jdl with 2 input parameters: collection name and output dir name.
1719 WriteJDL(copy);
c57f56b7 1720 }
1721 // Copy jdl to grid workspace
a8739e8a 1722 if (copy) {
b5e4aaa7 1723 // Check if an output directory was defined and valid
1724 if (!fGridOutputDir.Length()) {
1725 Error("CreateJDL", "You must define AliEn output directory");
1726 return kFALSE;
1727 } else {
1728 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1729 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1730 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1731 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1732 } else {
1733 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1734 return kFALSE;
1735 }
1736 }
1737 gGrid->Cd(workdir);
1738 }
648174cf 1739 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1740 TString mergeJDLName = fExecutable;
1741 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1742 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1743 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1744 if (fProductionMode) {
648174cf 1745 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1746 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1747 }
648174cf 1748 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1749 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1750 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1751 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1752 Fatal("","Terminating");
1753// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1754 if (fMergeViaJDL) {
1755 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1756// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1757 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1758 Fatal("","Terminating");
0f389141 1759 }
648174cf 1760 }
c57f56b7 1761 if (fAdditionalLibs.Length()) {
e7c71df0 1762 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1763 TObjString *os;
1764 TIter next(arr);
1765 while ((os=(TObjString*)next())) {
c57f56b7 1766 if (os->GetString().Contains(".so")) continue;
4e5c5506 1767 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1768 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1769// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1770 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1771 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1772 Fatal("","Terminating");
c57f56b7 1773 }
1774 delete arr;
1775 }
4e5c5506 1776 if (fPackages) {
1777 TIter next(fPackages);
1778 TObject *obj;
1779 while ((obj=next())) {
fdbbc7be 1780 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1781 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1782// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1783 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1784 Form("%s/%s", workdir.Data(), obj->GetName())))
1785 Fatal("","Terminating");
4e5c5506 1786 }
1787 }
c57f56b7 1788 }
1789 return kTRUE;
1790}
1791
a8739e8a 1792//______________________________________________________________________________
d2a409b2 1793Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1794{
1795// Writes one or more JDL's corresponding to findex. If findex is negative,
1796// all run numbers are considered in one go (jdl). For non-negative indices
1797// they correspond to the indices in the array fInputFiles.
1798 if (!fInputFiles) return kFALSE;
d3b18c4c 1799 TObject *os;
1800 TString workdir;
1801 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1802 workdir += fGridWorkingDir;
f47d5cb4 1803 TString stageName = "$2";
1804 if (fProductionMode) stageName = "$4";
1805 if (!fMergeDirName.IsNull()) {
1806 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1807 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1808 } else {
1809 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1810 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1811 }
d3b18c4c 1812 if (fProductionMode) {
a8739e8a 1813 TIter next(fInputFiles);
b3e07543 1814 while ((os=next())) {
d3b18c4c 1815 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1816 }
56754bf6 1817 if (!fOutputToRunNo)
1818 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1819 else
1820 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1821 } else {
1822 if (!fRunNumbers.Length() && !fRunRange[0]) {
1823 // One jdl with no parameters in case input data is specified by name.
1824 TIter next(fInputFiles);
1825 while ((os=next()))
1826 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1827 if (!fOutputSingle.IsNull())
1828 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1829 else {
1830 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1831 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1832 }
1833 } else {
1834 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1835 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1836 if (!fOutputSingle.IsNull()) {
1837 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1838 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1839 } else {
1840 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1841 }
1842 }
a8739e8a 1843 }
1844
a8739e8a 1845 // Generate the JDL as a string
1846 TString sjdl = fGridJDL->Generate();
0f389141 1847 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1848 // Final merge jdl
1849 if (!fMergeDirName.IsNull()) {
1850 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1851 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1852 } else {
1853 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1854 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1855 }
b3e07543 1856 TString sjdl2 = fMergingJDL->Generate();
1857 Int_t index, index1;
a8739e8a 1858 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1859 sjdl.ReplaceAll("(member", "\n (member");
1860 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1861 sjdl.ReplaceAll("{", "{\n ");
1862 sjdl.ReplaceAll("};", "\n};");
1863 sjdl.ReplaceAll("{\n \n", "{\n");
1864 sjdl.ReplaceAll("\n\n", "\n");
1865 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1866 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1867 sjdl1.ReplaceAll("(member", "\n (member");
1868 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1869 sjdl1.ReplaceAll("{", "{\n ");
1870 sjdl1.ReplaceAll("};", "\n};");
1871 sjdl1.ReplaceAll("{\n \n", "{\n");
1872 sjdl1.ReplaceAll("\n\n", "\n");
1873 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1874 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1875 sjdl2.ReplaceAll("(member", "\n (member");
1876 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1877 sjdl2.ReplaceAll("{", "{\n ");
1878 sjdl2.ReplaceAll("};", "\n};");
1879 sjdl2.ReplaceAll("{\n \n", "{\n");
1880 sjdl2.ReplaceAll("\n\n", "\n");
1881 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1882 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1883 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1884 index = sjdl.Index("JDLVariables");
1885 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1886 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1887 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1888 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1889 index = fJobTag.Index(":");
1890 if (index < 0) index = fJobTag.Length();
1891 TString jobTag = fJobTag;
f47d5cb4 1892 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1893 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1894 if (fProductionMode) {
1895 sjdl1.Prepend("# Generated merging jdl (production mode) \
1896 \n# $1 = full alien path to output directory to be merged \
1897 \n# $2 = train number \
1898 \n# $3 = production (like LHC10b) \
1899 \n# $4 = merging stage \
1900 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1901 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1902 sjdl2.Prepend("# Generated merging jdl \
1903 \n# $1 = full alien path to output directory to be merged \
1904 \n# $2 = train number \
1905 \n# $3 = production (like LHC10b) \
1906 \n# $4 = merging stage \
1907 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1908 } else {
1909 sjdl1.Prepend("# Generated merging jdl \
1910 \n# $1 = full alien path to output directory to be merged \
1911 \n# $2 = merging stage \
03335324 1912 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1913 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1914 sjdl2.Prepend("# Generated merging jdl \
1915 \n# $1 = full alien path to output directory to be merged \
1916 \n# $2 = merging stage \
03335324 1917 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1918 }
0f389141 1919 index = sjdl1.Index("JDLVariables");
1920 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1921 index = sjdl2.Index("JDLVariables");
1922 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1923 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1924 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1925 index = sjdl2.Index("Split =");
1926 if (index>=0) {
1927 index1 = sjdl2.Index("\n", index);
1928 sjdl2.Remove(index, index1-index+1);
1929 }
1930 index = sjdl2.Index("SplitMaxInputFileNumber");
1931 if (index>=0) {
1932 index1 = sjdl2.Index("\n", index);
1933 sjdl2.Remove(index, index1-index+1);
1934 }
1935 index = sjdl2.Index("InputDataCollection");
1936 if (index>=0) {
1937 index1 = sjdl2.Index(";", index);
1938 sjdl2.Remove(index, index1-index+1);
1939 }
1940 index = sjdl2.Index("InputDataListFormat");
1941 if (index>=0) {
1942 index1 = sjdl2.Index("\n", index);
1943 sjdl2.Remove(index, index1-index+1);
1944 }
1945 index = sjdl2.Index("InputDataList");
1946 if (index>=0) {
1947 index1 = sjdl2.Index("\n", index);
1948 sjdl2.Remove(index, index1-index+1);
1949 }
f47d5cb4 1950 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1951 // Write jdl to file
a8739e8a 1952 ofstream out;
d2a409b2 1953 out.open(fJDLName.Data(), ios::out);
a8739e8a 1954 if (out.bad()) {
d3b18c4c 1955 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1956 return kFALSE;
1957 }
1958 out << sjdl << endl;
b3e07543 1959 out.close();
0f389141 1960 TString mergeJDLName = fExecutable;
1961 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1962 if (fMergeViaJDL) {
1963 ofstream out1;
1964 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1965 if (out1.bad()) {
d3b18c4c 1966 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1967 return kFALSE;
1968 }
1969 out1 << sjdl1 << endl;
b3e07543 1970 out1.close();
1971 ofstream out2;
1972 TString finalJDL = mergeJDLName;
1973 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1974 out2.open(finalJDL.Data(), ios::out);
1975 if (out2.bad()) {
1976 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1977 return kFALSE;
1978 }
1979 out2 << sjdl2 << endl;
1980 out2.close();
0f389141 1981 }
a8739e8a 1982
1983 // Copy jdl to grid workspace
1984 if (!copy) {
d3b18c4c 1985 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1986 } else {
d2a409b2 1987 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1988 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1989 TString finalJDL = mergeJDLName;
1990 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1991 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1992 if (fProductionMode) {
b5fe9cba 1993 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1994 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1995 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1996 }
d2a409b2 1997 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1998 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 1999 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2000 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2001// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2002 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2003 Fatal("","Terminating");
0f389141 2004 if (fMergeViaJDL) {
b3e07543 2005 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2006// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2007// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2008 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2009 Fatal("","Terminating");
2010 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2011 Fatal("","Terminating");
0f389141 2012 }
a8739e8a 2013 }
2014 return kTRUE;
2015}
2016
c57f56b7 2017//______________________________________________________________________________
5513444a 2018Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2019{
2020// Returns true if file exists.
5513444a 2021 if (!gGrid) return kFALSE;
2ea55496 2022 TString slfn = lfn;
2023 slfn.ReplaceAll("alien://","");
2024 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2025 if (!res) return kFALSE;
2026 TMap *map = dynamic_cast<TMap*>(res->At(0));
2027 if (!map) {
2028 delete res;
2029 return kFALSE;
2030 }
2031 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2032 if (!objs || !objs->GetString().Length()) {
2033 delete res;
2034 return kFALSE;
2035 }
2036 delete res;
2037 return kTRUE;
2038}
2039
923e2ca5 2040//______________________________________________________________________________
2041Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2042{
2043// Returns true if directory exists. Can be also a path.
2044 if (!gGrid) return kFALSE;
2045 // Check if dirname is a path
2046 TString dirstripped = dirname;
2047 dirstripped = dirstripped.Strip();
2048 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2049 TString dir = gSystem->BaseName(dirstripped);
2050 dir += "/";
2051 TString path = gSystem->DirName(dirstripped);
2052 TGridResult *res = gGrid->Ls(path, "-F");
2053 if (!res) return kFALSE;
2054 TIter next(res);
2055 TMap *map;
2056 TObject *obj;
2057 while ((map=dynamic_cast<TMap*>(next()))) {
2058 obj = map->GetValue("name");
2059 if (!obj) break;
2060 if (dir == obj->GetName()) {
2061 delete res;
2062 return kTRUE;
2063 }
2064 }
2065 delete res;
2066 return kFALSE;
2067}
2068
c57f56b7 2069//______________________________________________________________________________
a2f5fc01 2070void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2071{
2072// Check input data type.
a2f5fc01 2073 isCollection = kFALSE;
2074 isXml = kFALSE;
2075 useTags = kFALSE;
c57f56b7 2076 if (!gGrid) {
2077 Error("CheckDataType", "No connection to grid");
2078 return;
2079 }
a2f5fc01 2080 isCollection = IsCollection(lfn);
c57f56b7 2081 TString msg = "\n##### file: ";
2082 msg += lfn;
a2f5fc01 2083 if (isCollection) {
c57f56b7 2084 msg += " type: raw_collection;";
2085 // special treatment for collections
a2f5fc01 2086 isXml = kFALSE;
c57f56b7 2087 // check for tag files in the collection
2088 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2089 if (!res) {
2090 msg += " using_tags: No (unknown)";
23329835 2091 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2092 return;
2093 }
2094 const char* typeStr = res->GetKey(0, "origLFN");
2095 if (!typeStr || !strlen(typeStr)) {
2096 msg += " using_tags: No (unknown)";
23329835 2097 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2098 return;
2099 }
2100 TString file = typeStr;
a2f5fc01 2101 useTags = file.Contains(".tag");
2102 if (useTags) msg += " using_tags: Yes";
c57f56b7 2103 else msg += " using_tags: No";
23329835 2104 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2105 return;
2106 }
2107 TString slfn(lfn);
2108 slfn.ToLower();
a2f5fc01 2109 isXml = slfn.Contains(".xml");
2110 if (isXml) {
c57f56b7 2111 // Open xml collection and check if there are tag files inside
2112 msg += " type: xml_collection;";
2113 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2114 if (!coll) {
2115 msg += " using_tags: No (unknown)";
23329835 2116 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2117 return;
2118 }
2119 TMap *map = coll->Next();
2120 if (!map) {
2121 msg += " using_tags: No (unknown)";
23329835 2122 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2123 return;
2124 }
2125 map = (TMap*)map->GetValue("");
2126 TString file;
2127 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2128 useTags = file.Contains(".tag");
c57f56b7 2129 delete coll;
a2f5fc01 2130 if (useTags) msg += " using_tags: Yes";
c57f56b7 2131 else msg += " using_tags: No";
23329835 2132 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2133 return;
2134 }
a2f5fc01 2135 useTags = slfn.Contains(".tag");
c57f56b7 2136 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2137 else msg += " type: unknown file;";
a2f5fc01 2138 if (useTags) msg += " using_tags: Yes";
c57f56b7 2139 else msg += " using_tags: No";
23329835 2140 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2141}
2142
4e5c5506 2143//______________________________________________________________________________
2144void AliAnalysisAlien::EnablePackage(const char *package)
2145{
2146// Enables a par file supposed to exist in the current directory.
2147 TString pkg(package);
2148 pkg.ReplaceAll(".par", "");
2149 pkg += ".par";
2150 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2151 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2152 return;
2153 }
2154 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2155 Info("EnablePackage", "AliEn plugin will use .par packages");
2156 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2157 if (!fPackages) {
2158 fPackages = new TObjArray();
2159 fPackages->SetOwner();
2160 }
2161 fPackages->Add(new TObjString(pkg));
2162}
2163
3bdcb562 2164//______________________________________________________________________________
2165TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2166{
2167// Make a tree from files having the location specified in fFileForTestMode.
2168// Inspired from JF's CreateESDChain.
2169 if (fFileForTestMode.IsNull()) {
2170 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2171 return NULL;
2172 }
2173 if (gSystem->AccessPathName(fFileForTestMode)) {
2174 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2175 return NULL;
2176 }
2177 // Open the file
2178 ifstream in;
2179 in.open(fFileForTestMode);
2180 Int_t count = 0;
2181 // Read the input list of files and add them to the chain
2182 TString line;
2183 TChain *chain = new TChain(treeName);
2184 while (in.good())
2185 {
2186 in >> line;
2187 if (line.IsNull()) continue;
2188 if (count++ == fNtestFiles) break;
2189 TString esdFile(line);
2190 TFile *file = TFile::Open(esdFile);
2191 if (file) {
2192 if (!file->IsZombie()) chain->Add(esdFile);
2193 file->Close();
2194 } else {
2195 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2196 }
2197 }
2198 in.close();
2199 if (!chain->GetListOfFiles()->GetEntries()) {
2200 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2201 delete chain;
2202 return NULL;
2203 }
2204// chain->ls();
2205 return chain;
2206}
2207
d2a409b2 2208//______________________________________________________________________________
2209const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2210{
2211// Get job status for all jobs with jobid>jobidstart.
2212 static char mstatus[20];
2213 mstatus[0] = '\0';
2214 nrunning = 0;
2215 nwaiting = 0;
2216 nerror = 0;
2217 ndone = 0;
2218 TGridJobStatusList *list = gGrid->Ps("");
2219 if (!list) return mstatus;
2220 Int_t nentries = list->GetSize();
2221 TGridJobStatus *status;
2222 Int_t pid;
2223 for (Int_t ijob=0; ijob<nentries; ijob++) {
2224 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2225 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2226 if (pid<jobidstart) continue;
2227 if (pid == lastid) {
8ddad121 2228 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2229 }
2230 switch (status->GetStatus()) {
2231 case TGridJobStatus::kWAITING:
2232 nwaiting++; break;
2233 case TGridJobStatus::kRUNNING:
2234 nrunning++; break;
2235 case TGridJobStatus::kABORTED:
2236 case TGridJobStatus::kFAIL:
2237 case TGridJobStatus::kUNKNOWN:
2238 nerror++; break;
2239 case TGridJobStatus::kDONE:
2240 ndone++;
2241 }
2242 }
2243 list->Delete();
2244 delete list;
2245 return mstatus;
2246}
2247
c57f56b7 2248//______________________________________________________________________________
2249Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2250{
2251// Returns true if file is a collection. Functionality duplicated from
2252// TAlien::Type() because we don't want to directly depend on TAlien.
2253 if (!gGrid) {
2254 Error("IsCollection", "No connection to grid");
2255 return kFALSE;
2256 }
2257 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2258 if (!res) return kFALSE;
2259 const char* typeStr = res->GetKey(0, "type");
2260 if (!typeStr || !strlen(typeStr)) return kFALSE;
2261 if (!strcmp(typeStr, "collection")) return kTRUE;
2262 delete res;
2263 return kFALSE;
2264}
2265
fe2d7fc2 2266//______________________________________________________________________________
2267Bool_t AliAnalysisAlien::IsSingleOutput() const
2268{
2269// Check if single-ouput option is on.
2270 return (!fOutputSingle.IsNull());
2271}
2272
16a4353c 2273//______________________________________________________________________________
2274void AliAnalysisAlien::Print(Option_t *) const
2275{
2276// Print current plugin settings.
84fcd93f 2277 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2278 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2279 if (mgr && mgr->IsProofMode()) {
2280 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2281 if (TestBit(AliAnalysisGrid::kTest))
2282 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2283 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2284 if (!fProofDataSet.IsNull())
2285 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2286 if (fProofReset==1)
2287 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2288 if (fProofReset>1)
2289 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2290 if (!fRootVersionForProof.IsNull())
2291 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2292 else
2293 printf("= ROOT version requested________________________ default\n");
2294 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2295 if (!fAliRootMode.IsNull())
2296 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2297 if (fNproofWorkers)
2298 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2299 if (fNproofWorkersPerSlave)
2300 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2301 if (TestSpecialBit(kClearPackages))
2302 printf("= ClearPackages requested...\n");
2303 if (fIncludePath.Data())
2304 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2305 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2306 if (fPackages && fPackages->GetEntries()) {
2307 TIter next(fPackages);
2308 TObject *obj;
2309 TString list;
2310 while ((obj=next())) list += obj->GetName();
2311 printf("= Par files to be used: ________________________ %s\n", list.Data());
2312 }
2313 if (TestSpecialBit(kProofConnectGrid))
2314 printf("= Requested PROOF connection to grid\n");
2315 return;
2316 }
e1c22e21 2317 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2318 if (fOverwriteMode) {
2319 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2320 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2321 }
348be253 2322 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2323 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2324 printf("= Production mode:______________________________ %d\n", fProductionMode);
2325 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2326 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2327 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2328 if (fUser.Length())
84fcd93f 2329 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2330 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2331 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2332 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2333 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2334 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2335 if (fRunNumbers.Length())
84fcd93f 2336 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2337 if (fRunRange[0])
74013535 2338 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2339 if (!fRunRange[0] && !fRunNumbers.Length()) {
2340 TIter next(fInputFiles);
2341 TObject *obj;
2342 TString list;
2343 while ((obj=next())) list += obj->GetName();
84fcd93f 2344 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2345 }
2346 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2347 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2348 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2349 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2350 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2351 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2352 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2353 printf("=====================================================================\n");
2354 printf("= Job price: ___________________________________ %d\n", fPrice);
2355 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2356 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2357 if (fMaxInitFailed>0)
84fcd93f 2358 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2359 if (fMasterResubmitThreshold>0)
84fcd93f 2360 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2361 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2362 if (fNrunsPerMaster>0)
84fcd93f 2363 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2364 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2365 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2366 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2367 if (fArguments.Length())
84fcd93f 2368 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2369 if (fExecutableArgs.Length())
2370 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2371 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2372 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2373 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2374 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2375 if (fDatasetName)
84fcd93f 2376 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2377 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2378 if (fIncludePath.Data())
84fcd93f 2379 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2380 if (fCloseSE.Length())
84fcd93f 2381 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2382 if (fFriendChainName.Length())
84fcd93f 2383 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2384 if (fPackages && fPackages->GetEntries()) {
16a4353c 2385 TIter next(fPackages);
2386 TObject *obj;
2387 TString list;
2388 while ((obj=next())) list += obj->GetName();
84fcd93f 2389 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2390 }
2391}
2392
c57f56b7 2393//______________________________________________________________________________
2394void AliAnalysisAlien::SetDefaults()
2395{
2396// Set default values for everything. What cannot be filled will be left empty.
2397 if (fGridJDL) delete fGridJDL;
2398 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2399 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2400 fPrice = 1;
2401 fTTL = 30000;
2402 fSplitMaxInputFileNumber = 100;
2403 fMaxInitFailed = 0;
2404 fMasterResubmitThreshold = 0;
bb885a9e 2405 fNtestFiles = 10;
149d288c 2406 fNreplicas = 2;
a8739e8a 2407 fRunRange[0] = 0;
2408 fRunRange[1] = 0;
8e3bd06a 2409 fRunPrefix = "%d";
319593fb 2410 fNrunsPerMaster = 1;
16a4353c 2411 fMaxMergeFiles = 100;
c57f56b7 2412 fRunNumbers = "";
2413 fExecutable = "analysis.sh";
0a1c1f7f 2414 fExecutableCommand = "root -b -q";
c57f56b7 2415 fArguments = "";
631c0b05 2416 fExecutableArgs = "";
c57f56b7 2417 fAnalysisMacro = "myAnalysis.C";
2418 fAnalysisSource = "";
2419 fAdditionalLibs = "";
2420 fSplitMode = "se";
2421 fAPIVersion = "";
2422 fROOTVersion = "";
2423 fAliROOTVersion = "";
2424 fUser = ""; // Your alien user name
2425 fGridWorkingDir = "";
2426 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2427 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2428 fFriendChainName = "";
c57f56b7 2429 fGridOutputDir = "output";
b83f84f0 2430 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2431 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2432 fInputFormat = "xml-single";
2433 fJDLName = "analysis.jdl";
c6cb3634 2434 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2435 fMergeExcludes = "";
0f389141 2436 fMergeViaJDL = 0;
348be253 2437 SetUseCopy(kTRUE);
2438 SetCheckCopy(kTRUE);
149d288c 2439 SetDefaultOutputs(kTRUE);
e1c22e21 2440 fOverwriteMode = 1;
c57f56b7 2441}
2442
0f389141 2443//______________________________________________________________________________
b3e07543 2444Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2445{
b3e07543 2446// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2447 // First check if the result is already in the output directory.
2448 if (FileExists(Form("%s/%s",aliendir,filename))) {
2449 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2450 return kFALSE;
2451 }
b3e07543 2452 // Now check the last stage done.
2453 Int_t stage = 0;
2454 while (1) {
2455 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2456 stage++;
2457 }
2458 // Next stage of merging
2459 stage++;
2460 TString pattern = "*root_archive.zip";
2461 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2462 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2463 if (res) delete res;
2464 // Write standard output to file
2465 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2466 // Count the number of files inside
2467 ifstream ifile;
2468 ifile.open(Form("Stage_%d.xml",stage));
2469 if (!ifile.good()) {
2470 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2471 return kFALSE;
b3e07543 2472 }
2473 TString line;
2474 Int_t nfiles = 0;
2475 while (!ifile.eof()) {
2476 ifile >> line;
2477 if (line.Contains("/event")) nfiles++;
7c2cd90a 2478 }
b3e07543 2479 ifile.close();
2480 if (!nfiles) {
2481 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2482 return kFALSE;
2483 } else {
2484 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2485 }
b3e07543 2486 // Copy the file in the output directory
2487 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2488// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2489 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2490 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2491 // Check if this is the last stage to be done.
2492 Bool_t laststage = (nfiles<nperchunk);
2493 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2494 if (laststage) {
2495 printf("### Submiting final merging stage %d\n", stage);
2496 TString finalJDL = jdl;
2497 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2498 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2499 Int_t jobId = SubmitSingleJob(query);
2500 if (!jobId) return kFALSE;
2501 } else {
2502 printf("### Submiting merging stage %d\n", stage);
03335324 2503 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2504 Int_t jobId = SubmitSingleJob(query);
2505 if (!jobId) return kFALSE;
7c2cd90a 2506 }
b3e07543 2507 return kTRUE;
2508}
7c2cd90a 2509
105631b5 2510//______________________________________________________________________________
2511AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2512{
2513// Loat the analysis manager from a file.
2514 TFile *file = TFile::Open(fname);
2515 if (!file) {
2516 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2517 return 0;
2518 }
2519 TIter nextkey(file->GetListOfKeys());
2520 AliAnalysisManager *mgr = 0;
2521 TKey *key;
2522 while ((key=(TKey*)nextkey())) {
2523 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2524 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2525 }
2526 if (!mgr)
2527 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2528 return mgr;
2529}
2530
7c2cd90a 2531//______________________________________________________________________________
2532Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2533{
2534// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2535 if (!gGrid) return 0;
2536 printf("=> %s ------> ",query);
2537 TGridResult *res = gGrid->Command(query);
2538 if (!res) return 0;
2539 TString jobId = res->GetKey(0,"jobId");
2540 delete res;
2541 if (jobId.IsNull()) {
2542 printf("submission failed. Reason:\n");
2543 gGrid->Stdout();
2544 gGrid->Stderr();
2545 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2546 return 0;
2547 }
2548 printf(" Job id: %s\n", jobId.Data());
2549 return atoi(jobId);
2550}
2551
2552//______________________________________________________________________________
b3e07543 2553Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2554{
b3e07543 2555// Merge given output files from basedir. Basedir can be an alien output directory
2556// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2557// files in a group (ignored for xml input). Merging can be done in stages:
2558// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2559// stage=1 : works with an xml of all root_archive.zip in the output directory
2560// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2561 TString outputFile = output;
0f389141 2562 TString command;
a2f5fc01 2563 TString outputChunk;
2564 TString previousChunk = "";
b3e07543 2565 TObjArray *listoffiles = new TObjArray();
2566// listoffiles->SetOwner();
a2f5fc01 2567 Int_t countChunk = 0;
2568 Int_t countZero = nmaxmerge;
0f389141 2569 Bool_t merged = kTRUE;
a2f5fc01 2570 Int_t index = outputFile.Index("@");
2571 if (index > 0) outputFile.Remove(index);
7c2cd90a 2572 TString inputFile = outputFile;
b3e07543 2573 TString sbasedir = basedir;
2574 if (sbasedir.Contains(".xml")) {
2575 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2576 nmaxmerge = 9999999;
2577 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2578 if (!coll) {
2579 ::Error("MergeOutput", "Input XML collection empty.");
2580 return kFALSE;
2581 }
2582 // Iterate grid collection
2583 while (coll->Next()) {
2584 TString fname = gSystem->DirName(coll->GetTURL());
2585 fname += "/";
2586 fname += inputFile;
2587 listoffiles->Add(new TNamed(fname.Data(),""));
2588 }
2589 } else {
2590 command = Form("find %s/ *%s", basedir, inputFile.Data());
2591 printf("command: %s\n", command.Data());
2592 TGridResult *res = gGrid->Command(command);
2593 if (!res) {
2594 ::Error("MergeOutput","No result for the find command\n");
2595 delete listoffiles;
2596 return kFALSE;
2597 }
2598 TIter nextmap(res);
2599 TMap *map = 0;
2600 while ((map=(TMap*)nextmap())) {
2601 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2602 if (!objs || !objs->GetString().Length()) {
2603 // Nothing found - skip this output
2604 delete res;
2605 delete listoffiles;
2606 return kFALSE;
2607 }
2608 listoffiles->Add(new TNamed(objs->GetName(),""));
2609 }
2610 delete res;
2611 }
2612 if (!listoffiles->GetEntries()) {
7c2cd90a 2613 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2614 delete listoffiles;
0f389141 2615 return kFALSE;
2616 }
2617
2618 TFileMerger *fm = 0;
b3e07543 2619 TIter next0(listoffiles);
2620 TObjArray *listoffilestmp = new TObjArray();
2621 listoffilestmp->SetOwner();
2622 TObject *nextfile;
2623 TString snextfile;
2624 // Keep only the files at upper level
2625 Int_t countChar = 0;
2626 while ((nextfile=next0())) {
2627 snextfile = nextfile->GetName();
2628 Int_t crtCount = snextfile.CountChar('/');
2629 if (nextfile == listoffiles->First()) countChar = crtCount;
2630 if (crtCount < countChar) countChar = crtCount;
2631 }
2632 next0.Reset();
2633 while ((nextfile=next0())) {
2634 snextfile = nextfile->GetName();
2635 Int_t crtCount = snextfile.CountChar('/');
2636 if (crtCount > countChar) {
2637 delete nextfile;
2638 continue;
2639 }
2640 listoffilestmp->Add(nextfile);
2641 }
2642 delete listoffiles;
2643 listoffiles = listoffilestmp; // Now contains 'good' files
2644 listoffiles->Print();
2645 TIter next(listoffiles);
7c2cd90a 2646 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2647 outputChunk = outputFile;
2648 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2649 // Check for existent temporary merge files
e1c22e21 2650 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2651 // Preserve old merging functionality for stage 0.
2652 if (stage==0) {
2653 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2654 while (1) {
2655 // Skip as many input files as in a chunk
2ea55496 2656 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2657 nextfile = next();
2658 if (!nextfile) {
2ea55496 2659 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2660 delete listoffiles;
2ea55496 2661 return kFALSE;
2662 }
b3e07543 2663 snextfile = nextfile->GetName();
7c2cd90a 2664 }
2665 outputChunk = outputFile;
2666 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2667 countChunk++;
2668 if (gSystem->AccessPathName(outputChunk)) continue;
2669 // Merged file with chunks up to <countChunk> found
2670 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2671 previousChunk = outputChunk;
2672 break;
2673 }
2674 }
2675 countZero = nmaxmerge;
2676
b3e07543 2677 while ((nextfile=next())) {
2678 snextfile = nextfile->GetName();
2ea55496 2679 // Loop 'find' results and get next LFN
7c2cd90a 2680 if (countZero == nmaxmerge) {
2681 // First file in chunk - create file merger and add previous chunk if any.
2b895f34 2682 fm = new TFileMerger(kTRUE);
7c2cd90a 2683 fm->SetFastMethod(kTRUE);
2684 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2685 outputChunk = outputFile;
2686 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2687 }
2688 // If last file found, put merged results in the output file
b3e07543 2689 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2690 // Add file to be merged and decrement chunk counter.
b3e07543 2691 fm->AddFile(snextfile);
7c2cd90a 2692 countZero--;
b3e07543 2693 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2694 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2695 // Nothing found - skip this output
2696 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2697 merged = kFALSE;
2698 break;
7c2cd90a 2699 }
2700 fm->OutputFile(outputChunk);
2701 // Merge the outputs, then go to next chunk
2702 if (!fm->Merge()) {
2703 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2704 merged = kFALSE;
2705 break;
7c2cd90a 2706 } else {
2707 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2708 gSystem->Unlink(previousChunk);
2709 }
b3e07543 2710 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2711 countChunk++;
2712 countZero = nmaxmerge;
2713 previousChunk = outputChunk;
0f389141 2714 }
0f389141 2715 }
b3e07543 2716 delete listoffiles;
2717 delete fm;
7c2cd90a 2718 return merged;
2719 }
2720 // Merging stage different than 0.
2721 // Move to the begining of the requested chunk.
2b895f34 2722 fm = new TFileMerger(kTRUE);
7c2cd90a 2723 fm->SetFastMethod(kTRUE);
b3e07543 2724 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2725 delete listoffiles;
7c2cd90a 2726 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2727 // Nothing found - skip this output
2728 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2729 delete fm;
2730 return kFALSE;
2731 }
b3e07543 2732 fm->OutputFile(outputFile);
7c2cd90a 2733 // Merge the outputs
2734 if (!fm->Merge()) {
2735 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2736 delete fm;
2737 return kFALSE;
2738 } else {
b3e07543 2739 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2740 }
7c2cd90a 2741 delete fm;
2742 return kTRUE;
0f389141 2743}
2744
c57f56b7 2745//______________________________________________________________________________
2746Bool_t AliAnalysisAlien::MergeOutputs()
2747{
2748// Merge analysis outputs existing in the AliEn space.
2749 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2750 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2751 if (!Connect()) {
2752 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2753 return kFALSE;
0f389141 2754 }
c9e8f7fd 2755 if (fMergeViaJDL) {
2756 if (!TestBit(AliAnalysisGrid::kMerge)) {
2757 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2758 return kFALSE;
2759 }
2760 if (fProductionMode) {
2761 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2762 return kFALSE;
2763 }
0f389141 2764 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2765 if (!SubmitMerging()) return kFALSE;
0f389141 2766 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2767 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2768 return kFALSE;
c57f56b7 2769 }
2770 // Get the output path
dc030a24 2771 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2772 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2773 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2774 return kFALSE;
2775 }
2776 if (!fOutputFiles.Length()) {
2777 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2778 return kFALSE;
0f389141 2779 }
2780 // Check if fast read option was requested
e1c22e21 2781 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2782 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2783 if (fFastReadOption) {
d3339be3 2784 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2785 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2786 gEnv->SetValue("XNet.ConnectTimeout",50);
2787 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2788 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2789 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2790 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2791 }
e8b839ab 2792 // Make sure we change the temporary directory
2793 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2794 // Set temporary compilation directory to current one
2795 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2796 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2797 TIter next(list);
2798 TObjString *str;
a2f5fc01 2799 TString outputFile;
c57f56b7 2800 Bool_t merged = kTRUE;
2801 while((str=(TObjString*)next())) {
a2f5fc01 2802 outputFile = str->GetString();
2803 Int_t index = outputFile.Index("@");
2804 if (index > 0) outputFile.Remove(index);
2805 TString outputChunk = outputFile;
2806 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2807 // Skip already merged outputs
a2f5fc01 2808 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2809 if (fOverwriteMode) {
a2f5fc01 2810 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2811 gSystem->Unlink(outputFile);
2812 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2813 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2814 outputChunk.Data());
2815 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2816 }
2817 } else {
a2f5fc01 2818 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2819 continue;
2820 }
2821 } else {
a2f5fc01 2822 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2823 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2824 outputChunk.Data());
2825 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2826 }
2827 }
37d21c01 2828 if (fMergeExcludes.Contains(outputFile.Data()) ||
2829 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 2830 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2831 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2832 if (!merged) {
2833 Error("MergeOutputs", "Terminate() will NOT be executed");
2834 return kFALSE;
ff07ec61 2835 }
2836 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2837 if (fileOpened) fileOpened->Close();
c57f56b7 2838 }
0f389141 2839 return kTRUE;
c57f56b7 2840}
2841
bb885a9e 2842//______________________________________________________________________________
2843void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2844{
2845// Use the output files connected to output containers from the analysis manager
2846// rather than the files defined by SetOutputFiles
2847 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2848 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2849 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2850}
2851
149d288c 2852//______________________________________________________________________________
2853void AliAnalysisAlien::SetOutputFiles(const char *list)
2854{
2855// Manually set the output files list.
2856// Removes duplicates. Not allowed if default outputs are not disabled.
2857 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2858 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2859 return;
2860 }
2861 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2862 fOutputFiles = "";
2863 TString slist = list;
2864 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2865 TObjArray *arr = slist.Tokenize(" ");
2866 TObjString *os;
2867 TIter next(arr);
2868 TString sout;
2869 while ((os=(TObjString*)next())) {
2870 sout = os->GetString();
2871 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2872 if (fOutputFiles.Contains(sout)) continue;
2873 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2874 fOutputFiles += sout;
2875 }
2876 delete arr;
f790bc1b 2877}
149d288c 2878
2879//______________________________________________________________________________
2880void AliAnalysisAlien::SetOutputArchive(const char *list)
2881{
2882// Manually set the output archive list. Free text - you are on your own...
2883// Not allowed if default outputs are not disabled.
2884 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2885 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2886 return;
2887 }
2888 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2889 fOutputArchive = list;
2890}
2891
2892//______________________________________________________________________________
2893void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2894{
2895// Setting a prefered output SE is not allowed anymore.
2896 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2897}
2898
28ce6c9c 2899//______________________________________________________________________________
2900void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
2901{
2902// Set some PROOF special parameter.
2903 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2904 if (pair) {
2905 TObject *old = pair->Key();
2906 TObject *val = pair->Value();
2907 fProofParam.Remove(old);
2908 delete old;
2909 delete val;
2910 }
2911 fProofParam.Add(new TObjString(pname), new TObjString(value));
2912}
2913
2914//______________________________________________________________________________
2915const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
2916{
2917// Returns a special PROOF parameter.
2918 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2919 if (!pair) return 0;
2920 return pair->Value()->GetName();
2921}
2922
5513444a 2923//______________________________________________________________________________
2924Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2925{
2926// Start remote grid analysis.
3bdcb562 2927 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2928 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2929 if (!mgr || !mgr->IsInitialized()) {
2930 Error("StartAnalysis", "You need an initialized analysis manager for this");
2931 return kFALSE;
2932 }
2933 // Are we in PROOF mode ?
2934 if (mgr->IsProofMode()) {
baa34407 2935 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
2936 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2937 if (fProofCluster.IsNull()) {
2938 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2939 return kFALSE;
2940 }
2941 if (fProofDataSet.IsNull() && !testMode) {
2942 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2943 return kFALSE;
2944 }
2945 // Set the needed environment
2946 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2947 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2948 if (fProofReset && !testMode) {
2949 if (fProofReset==1) {
2950 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2951 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2952 } else {
2953 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2954 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2955 }
2956 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2957 return kFALSE;
2958 }
baa34407 2959
2960 if (!testMode) {
2961 // Check if there is an old active session
2962 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
2963 if (nsessions) {
2964 Error("StartAnalysis","You have to reset your old session first\n");
2965 return kFALSE;
2966 }
53911ca6 2967 }
3bdcb562 2968 // Do we need to change the ROOT version ? The success of this cannot be checked.
2969 if (!fRootVersionForProof.IsNull() && !testMode) {
2970 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2971 fProofCluster.Data(), fRootVersionForProof.Data()));
2972 }
2973 // Connect to PROOF and check the status
2974 Long_t proof = 0;
27734f0e 2975 TString sworkers;
2976 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2977 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2978 if (!testMode) {
27734f0e 2979 if (!sworkers.IsNull())
2980 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2981 else
2982 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2983 } else {
2984 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2985 if (!proof) {
2986 Error("StartAnalysis", "Could not start PROOF in test mode");
2987 return kFALSE;
2988 }
2989 }
2990 if (!proof) {
2991 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2992 return kFALSE;
2993 }
27734f0e 2994 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2995 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 2996 // Set proof special parameters if any
2997 TIter nextpp(&fProofParam);
2998 TObject *proofparam;
2999 while ((proofparam=nextpp())) {
3000 TString svalue = GetProofParameter(proofparam->GetName());
3001 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3002 }
3bdcb562 3003 // Is dataset existing ?
3004 if (!testMode) {
3005 TString dataset = fProofDataSet;
3006 Int_t index = dataset.Index("#");
3007 if (index>=0) dataset.Remove(index);
830acc4c 3008// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3009// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3010// return kFALSE;
3011// }
3012// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3013 }
3014 // Is ClearPackages() needed ?
3015 if (TestSpecialBit(kClearPackages)) {
3016 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3017 gROOT->ProcessLine("gProof->ClearPackages();");
3018 }
3019 // Is a given aliroot mode requested ?
3020 TList optionsList;
1ac846e9 3021 TString parLibs;
830acc4c 3022 if (!fAliRootMode.IsNull()) {
3bdcb562 3023 TString alirootMode = fAliRootMode;
3024 if (alirootMode == "default") alirootMode = "";
3025 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3026 optionsList.SetOwner();
3027 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3028 // Check the additional libs to be loaded
3029 TString extraLibs;
1ac846e9 3030 Bool_t parMode = kFALSE;
dc1ffbcb 3031 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3032 // Parse the extra libs for .so
3033 if (fAdditionalLibs.Length()) {
3034 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3035 TIter next(list);
3036 TObjString *str;
1ac846e9 3037 while((str=(TObjString*)next())) {
3038 if (str->GetString().Contains(".so")) {
3039 if (parMode) {
3040 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3041 break;
3042 }
3043 TString stmp = str->GetName();
3044 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3045 stmp.ReplaceAll(".so","");
3046 if (!extraLibs.IsNull()) extraLibs += ":";
3047 extraLibs += stmp;
3048 continue;
3049 }
3050 if (str->GetString().Contains(".par")) {
3051 // The first par file found in the list will not allow any further .so
3052 parMode = kTRUE;
3053 if (!parLibs.IsNull()) parLibs += ":";
3054 parLibs += str->GetName();
3055 continue;
3056 }
3bdcb562 3057 }
3058 if (list) delete list;
3059 }
dc030a24 3060 if (!extraLibs.IsNull()) {
3061 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3062 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3063 }
830acc4c 3064 // Check extra includes
3065 if (!fIncludePath.IsNull()) {
3066 TString includePath = fIncludePath;
3067 includePath.ReplaceAll(" ",":");
dc030a24 3068 includePath.ReplaceAll("$ALICE_ROOT/","");
3069 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3070 includePath.ReplaceAll("-I","");
dc030a24 3071 includePath.Remove(TString::kTrailing, ':');
830acc4c 3072 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3073 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3074 }
27734f0e 3075 // Check if connection to grid is requested
3076 if (TestSpecialBit(kProofConnectGrid))
3077 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3078 // Enable AliRoot par
3079 if (testMode) {
3080 // Enable proof lite package
3081 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3082 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3083 TNamed *obj = (TNamed*)optionsList.At(i);
3084 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3085 }
3086 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3087 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3088 Info("StartAnalysis", "AliRootProofLite enabled");
3089 } else {
3090 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3091 return kFALSE;
3092 }
3093 } else {
dc030a24 3094 if ( ! fAliROOTVersion.IsNull() ) {
3095 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3096 fAliROOTVersion.Data(), &optionsList))) {
3097 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3098 return kFALSE;
3099 }
3100 }
27734f0e 3101 }
1ac846e9 3102 // Enable first par files from fAdditionalLibs
3103 if (!parLibs.IsNull()) {
3104 TObjArray *list = parLibs.Tokenize(":");
3105 TIter next(list);
3106 TObjString *package;
3107 while((package=(TObjString*)next())) {
40501bb7 3108 TString spkg = package->GetName();
3109 spkg.ReplaceAll(".par", "");
3110 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3111 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3112 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3113 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3114 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3115 return kFALSE;
3116 }
3117 } else {
3118 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3119 return kFALSE;
3120 }
3121 }
3122 if (list) delete list;
3123 }
3bdcb562 3124 } else {
3125 if (fAdditionalLibs.Contains(".so") && !testMode) {
3126 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3127 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3128 return kFALSE;
3129 }
3130 }
3131 // Enable par files if requested
3132 if (fPackages && fPackages->GetEntries()) {
3133 TIter next(fPackages);
3134 TObject *package;
3135 while ((package=next())) {
1ac846e9 3136 // Skip packages already enabled
3137 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3138 TString spkg = package->GetName();
3139 spkg.ReplaceAll(".par", "");
3140 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3141 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3142 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3143 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3144 return kFALSE;
3145 }
3146 } else {
3147 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3148 return kFALSE;
3149 }
3150 }
3151 }
3152 // Do we need to load analysis source files ?
3153 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3154 if (fAnalysisSource.Length()) {
3155 TObjArray *list = fAnalysisSource.Tokenize(" ");
3156 TIter next(list);
3157 TObjString *str;
3158 while((str=(TObjString*)next())) {
3159 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3160 }
3bdcb562 3161 if (list) delete list;
3162 }
830acc4c 3163 if (testMode) {
3164 // Register dataset to proof lite.
3165 if (fFileForTestMode.IsNull()) {
3166 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3167 return kFALSE;
3168 }
3169 if (gSystem->AccessPathName(fFileForTestMode)) {
3170 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3171 return kFALSE;
3172 }
3173 TFileCollection *coll = new TFileCollection();
3174 coll->AddFromFile(fFileForTestMode);
8ddad121 3175 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3176 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3177 }
3bdcb562 3178 return kTRUE;
3179 }
c57f56b7 3180
43da816a 3181 // Check if output files have to be taken from the analysis manager
3182 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3183 // Add output files and AOD files
3184 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3185 // Add extra files registered to the analysis manager
f790bc1b 3186 TString extra = GetListOfFiles("ext");
3187 if (!extra.IsNull()) {
149d288c 3188 extra.ReplaceAll(".root", "*.root");
f790bc1b 3189 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3190 fOutputFiles += extra;
84fcd93f 3191 }
149d288c 3192 // Compose the output archive.
b83f84f0 3193 fOutputArchive = "log_archive.zip:std*@disk=1 ";
3194 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3195 }
f7b1cbc2 3196// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3197 if (TestBit(AliAnalysisGrid::kOffline)) {
3198 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3199 \n there nor any job run. You can revise the JDL and analysis \
3200 \n macro then run the same in \"submit\" mode.");
3201 } else if (TestBit(AliAnalysisGrid::kTest)) {
3202 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3203 \n dataset.");
3204 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3205 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3206 \n space and job submitted.");
3207 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3208 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3209 if (fMergeViaJDL) CheckInputData();
5513444a 3210 return kTRUE;
c57f56b7 3211 } else {
3212 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3213 }
3214
348be253 3215 Print();
c57f56b7 3216 if (!Connect()) {
3217 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3218 return kFALSE;
16a4353c 3219 }
d3b18c4c 3220 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3221 if (!CheckInputData()) {
3222 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3223 return kFALSE;
c57f56b7 3224 }
d3339be3 3225 if (!CreateDataset(fDataPattern)) {
3226 TString serror;
3227 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3228 if (fRunNumbers.Length()) serror = "run numbers";
3229 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3230 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3231 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3232 return kFALSE;
3233 }
21ca8e59 3234 WriteAnalysisFile();
c57f56b7 3235 WriteAnalysisMacro();
3236 WriteExecutable();
3237 WriteValidationScript();
0f389141 3238 if (fMergeViaJDL) {
3239 WriteMergingMacro();
3240 WriteMergeExecutable();
3241 WriteValidationScript(kTRUE);
3242 }
5513444a 3243 if (!CreateJDL()) return kFALSE;
3244 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3245 if (testMode) {
c57f56b7 3246 // Locally testing the analysis
3247 Info("StartAnalysis", "\n_______________________________________________________________________ \
3248 \n Running analysis script in a daughter shell as on a worker node \
3249 \n_______________________________________________________________________");
149d288c 3250 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3251 TIter next(list);
3252 TObjString *str;
a2f5fc01 3253 TString outputFile;
c57f56b7 3254 while((str=(TObjString*)next())) {
a2f5fc01 3255 outputFile = str->GetString();
3256 Int_t index = outputFile.Index("@");
3257 if (index > 0) outputFile.Remove(index);
3258 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3259 }
3260 delete list;
3261 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3262 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3263// gSystem->Exec("cat stdout");
5513444a 3264 return kFALSE;
c57f56b7 3265 }
5513444a 3266 // Check if submitting is managed by LPM manager
a3e84053 3267 if (fProductionMode) {
5513444a 3268 TString prodfile = fJDLName;
3269 prodfile.ReplaceAll(".jdl", ".prod");
3270 WriteProductionFile(prodfile);
3271 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3272 return kFALSE;
3273 }
a8739e8a 3274 // Submit AliEn job(s)
d2a409b2 3275 gGrid->Cd(fGridOutputDir);
a8739e8a 3276 TGridResult *res;
c57f56b7 3277 TString jobID = "";
d2a409b2 3278 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3279 // Submit a given xml or a set of runs
a8739e8a 3280 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3281 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3282 if (res) {
3283 const char *cjobId = res->GetKey(0,"jobId");
3284 if (!cjobId) {
a03be957 3285 gGrid->Stdout();
3286 gGrid->Stderr();
a8739e8a 3287 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3288 return kFALSE;
a8739e8a 3289 } else {
3290 Info("StartAnalysis", "\n_______________________________________________________________________ \
3291 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3292 \n_______________________________________________________________________",
3293 fJDLName.Data(), cjobId);
3294 jobID = cjobId;
3295 }
3296 delete res;
a03be957 3297 } else {
3298 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3299 return kFALSE;
a8739e8a 3300 }
3301 } else {
d2a409b2 3302 // Submit for a range of enumeration of runs.
a03be957 3303 if (!Submit()) return kFALSE;
c57f56b7 3304 }
a8739e8a 3305
c57f56b7 3306 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3307 \n You may exit at any time and terminate the job later using the option <terminate> \
3308 \n ##################################################################################", jobID.Data());
bb885a9e 3309 gSystem->Exec("aliensh");
5513444a 3310 return kTRUE;
c57f56b7 3311}
3312
f790bc1b 3313//______________________________________________________________________________
3314const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3315{
3316// Get a comma-separated list of output files of the requested type.
3317// Type can be (case unsensitive):
3318// aod - list of aod files (std, extensions and filters)
3319// out - list of output files connected to containers (but not aod's or extras)
3320// ext - list of extra files registered to the manager
3321// ter - list of files produced in terminate
3322 static TString files;
3323 files = "";
3324 TString stype = type;
3325 stype.ToLower();
3326 TString aodfiles, extra;
3327 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3328 if (!mgr) {
3329 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3330 return files.Data();
3331 }
3332 if (mgr->GetOutputEventHandler()) {
3333 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3334 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3335 if (!extraaod.IsNull()) {
3336 aodfiles += ",";
3337 aodfiles += extraaod;
3338 }
3339 }
3340 if (stype.Contains("aod")) {
3341 files = aodfiles;
3342 if (stype == "aod") return files.Data();
3343 }
3344 // Add output files that are not in the list of AOD files
3345 TString outputfiles = "";
3346 TIter next(mgr->GetOutputs());
3347 AliAnalysisDataContainer *output;
3348 const char *filename = 0;
3349 while ((output=(AliAnalysisDataContainer*)next())) {
3350 filename = output->GetFileName();
3351 if (!(strcmp(filename, "default"))) continue;
3352 if (outputfiles.Contains(filename)) continue;
3353 if (aodfiles.Contains(filename)) continue;
3354 if (!outputfiles.IsNull()) outputfiles += ",";
3355 outputfiles += filename;
3356 }
3357 if (stype.Contains("out")) {
3358 if (!files.IsNull()) files += ",";
3359 files += outputfiles;
3360 if (stype == "out") return files.Data();
3361 }
3362 // Add extra files registered to the analysis manager
3363 TString sextra;
3364 extra = mgr->GetExtraFiles();
3365 if (!extra.IsNull()) {
3366 extra.Strip();
3367 extra.ReplaceAll(" ", ",");
3368 TObjArray *fextra = extra.Tokenize(",");
3369 TIter nextx(fextra);
3370 TObject *obj;
3371 while ((obj=nextx())) {
3372 if (aodfiles.Contains(obj->GetName())) continue;
3373 if (outputfiles.Contains(obj->GetName())) continue;
3374 if (sextra.Contains(obj->GetName())) continue;
3375 if (!sextra.IsNull()) sextra += ",";
3376 sextra += obj->GetName();
3377 }
3378 delete fextra;
3379 if (stype.Contains("ext")) {
3380 if (!files.IsNull()) files += ",";
3381 files += sextra;
3382 }
3383 }
3384 if (stype == "ext") return files.Data();
3385 TString termfiles;
3386 if (!fTerminateFiles.IsNull()) {
3387 fTerminateFiles.Strip();
3388 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3389 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3390 TIter nextx(fextra);
3391 TObject *obj;
3392 while ((obj=nextx())) {
3393 if (aodfiles.Contains(obj->GetName())) continue;
3394 if (outputfiles.Contains(obj->GetName())) continue;
3395 if (termfiles.Contains(obj->GetName())) continue;
3396 if (sextra.Contains(obj->GetName())) continue;
3397 if (!termfiles.IsNull()) termfiles += ",";
3398 termfiles += obj->GetName();
3399 }
3400 delete fextra;
3401 }
3402 if (stype.Contains("ter")) {
3403 if (!files.IsNull() && !termfiles.IsNull()) {
3404 files += ",";
3405 files += termfiles;
3406 }
3407 }
3408 return files.Data();
3409}
3410
d2a409b2 3411//______________________________________________________________________________
a03be957 3412Bool_t AliAnalysisAlien::Submit()
d2a409b2 3413{
3414// Submit all master jobs.
3415 Int_t nmasterjobs = fInputFiles->GetEntries();
3416 Long_t tshoot = gSystem->Now();
a03be957 3417 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3418 while (fNsubmitted < nmasterjobs) {
3419 Long_t now = gSystem->Now();
3420 if ((now-tshoot)>30000) {
3421 tshoot = now;
a03be957 3422 if (!SubmitNext()) return kFALSE;
d2a409b2 3423 }
3424 }
a03be957 3425 return kTRUE;
d2a409b2 3426}
3427
0f389141 3428//______________________________________________________________________________
a03be957 3429Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3430{
3431// Submit all merging jobs.
dc030a24 3432 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3433 gGrid->Cd(fGridOutputDir);
3434 TString mergeJDLName = fExecutable;
3435 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3436 if (!fInputFiles) {
3437 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3438 return kFALSE;
3439 }
0f389141 3440 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3441 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3442 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3443 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3444 if (fOutputToRunNo) {
3445 // The output directory is the run number
3446 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3447 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3448 } else {
17b4e9a4 3449 if (!fRunNumbers.Length() && !fRunRange[0]) {
3450 // The output directory is the grid outdir
3451 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3452 runOutDir = fGridOutputDir;
3453 } else {
3454 // The output directory is the master number in 3 digits format
3455 printf("### Submitting merging job for master <%03d>\n", i);
3456 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3457 }
7c2cd90a 3458 }
3459 // Check now the number of merging stages.
2b222f7c 3460 TObjArray *list = fOutputFiles.Tokenize(",");
3461 TIter next(list);
3462 TObjString *str;
3463 TString outputFile;
3464 while((str=(TObjString*)next())) {
3465 outputFile = str->GetString();
3466 Int_t index = outputFile.Index("@");
3467 if (index > 0) outputFile.Remove(index);
37d21c01 3468 if (!fMergeExcludes.Contains(outputFile) &&
3469 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3470 }
3471 delete list;
b3e07543 3472 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3473 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3474 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3475 }
a03be957 3476 if (!ntosubmit) return kTRUE;
9f020952 3477 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3478 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3479 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3480 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3481 \n ################################################################################################################");
0f389141 3482 gSystem->Exec("aliensh");
a03be957 3483 return kTRUE;
0f389141 3484}
3485
d2a409b2 3486//______________________________________________________________________________
a03be957 3487Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3488{
08d5b699 3489// Submit next bunch of master jobs if the queue is free. The first master job is
3490// submitted right away, while the next will not be unless the previous was split.
3491// The plugin will not submit new master jobs if there are more that 500 jobs in
3492// waiting phase.
d2a409b2 3493 static Bool_t iscalled = kFALSE;
3494 static Int_t firstmaster = 0;
3495 static Int_t lastmaster = 0;
3496 static Int_t npermaster = 0;
a03be957 3497 if (iscalled) return kTRUE;
d2a409b2 3498 iscalled = kTRUE;
3499 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3500 Int_t ntosubmit = 0;
3501 TGridResult *res;
3502 TString jobID = "";
3bdcb562 3503 Int_t nmasterjobs = fInputFiles->GetEntries();
3504 if (!fNsubmitted) {
3505 ntosubmit = 1;
3506 if (!IsUseSubmitPolicy()) {
90d50a8c 3507 if (nmasterjobs>5)
3bdcb562 3508 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3509 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3510 ntosubmit = nmasterjobs;
3511 }
3512 } else {
d2a409b2 3513 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3514 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3515 // If last master not split, just return
a03be957 3516 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3517 // No more than 100 waiting jobs
08d5b699 3518 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3519 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3520 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3521 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3522 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3523 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3524 }
d2a409b2 3525 for (Int_t i=0; i<ntosubmit; i++) {
3526 // Submit for a range of enumeration of runs.
a03be957 3527 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3528 TString query;
cd11251e 3529 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3530 runOutDir.ReplaceAll(".xml", "");
3531 if (fOutputToRunNo)
3532 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3533 else
3534 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3535 printf("********* %s\n",query.Data());
d2a409b2 3536 res = gGrid->Command(query);
3537 if (res) {
98ca124f 3538 TString cjobId1 = res->GetKey(0,"jobId");
3539 if (!cjobId1.Length()) {
d2a409b2 3540 iscalled = kFALSE;
a03be957 3541 gGrid->Stdout();
3542 gGrid->Stderr();
3543 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3544 return kFALSE;
d2a409b2 3545 } else {
3546 Info("StartAnalysis", "\n_______________________________________________________________________ \
3547 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3548 \n_______________________________________________________________________",
98ca124f 3549 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 3550 jobID += cjobId1;
3551 jobID += " ";
98ca124f 3552 lastmaster = cjobId1.Atoi();
d2a409b2 3553 if (!firstmaster) firstmaster = lastmaster;
3554 fNsubmitted++;
3555 }
3556 delete res;
a03be957 3557 } else {
3558 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3559 return kFALSE;
d2a409b2 3560 }
3561 }
3562 iscalled = kFALSE;
a03be957 3563 return kTRUE;
d2a409b2 3564}
3565
c57f56b7 3566//______________________________________________________________________________
3567void AliAnalysisAlien::WriteAnalysisFile()
3568{
f10e8481 3569// Write current analysis manager into the file <analysisFile>
3570 TString analysisFile = fExecutable;
3571 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3572 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3573 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3574 if (!mgr || !mgr->IsInitialized()) {
3575 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3576 return;
3577 }
3578 // Check analysis type
3579 TObject *handler;
3580 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3581 handler = (TObject*)mgr->GetInputEventHandler();
3582 if (handler) {
65eb22f8 3583 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3584 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3585 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3586 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3587 } else {
3588 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3589 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3590 }
c57f56b7 3591 }
3592 TDirectory *cdir = gDirectory;
f10e8481 3593 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3594 if (file) {
e16a394c 3595 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3596 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3597 // Unless merging makes no sense
3598 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3599 mgr->Write();
3600 delete file;
fe2d7fc2 3601 // Enable termination for local jobs
3602 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3603 }
3604 if (cdir) cdir->cd();
f10e8481 3605 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3606 }
3607 Bool_t copy = kTRUE;
d3b18c4c 3608 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3609 if (copy) {
3610 CdWork();
3611 TString workdir = gGrid->GetHomeDirectory();
3612 workdir += fGridWorkingDir;
d3b18c4c 3613 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3614 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3615 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3616 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3617 }
3618}
3619
3620//______________________________________________________________________________
3621void AliAnalysisAlien::WriteAnalysisMacro()
3622{
3623// Write the analysis macro that will steer the analysis in grid mode.
3624 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3625 ofstream out;
3626 out.open(fAnalysisMacro.Data(), ios::out);
3627 if (!out.good()) {
3628 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3629 return;
3630 }
5b273635 3631 Bool_t hasSTEERBase = kFALSE;
3632 Bool_t hasESD = kFALSE;
3633 Bool_t hasAOD = kFALSE;
3634 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3635 Bool_t hasOADB = kFALSE;
5b273635 3636 Bool_t hasANALYSISalice = kFALSE;
3637 Bool_t hasCORRFW = kFALSE;
c57f56b7 3638 TString func = fAnalysisMacro;
3639 TString type = "ESD";
3640 TString comment = "// Analysis using ";
c85cfc0f 3641 if (IsUseMCchain()) {
3642 type = "MC";
3643 comment += "MC";
3644 } else {
3645 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3646 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3647 type = "AOD";
3648 comment += "AOD";
3649 }
3650 }
0df6ccf2 3651 if (type!="AOD" && fFriendChainName!="") {
3652 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3653 return;
3654 }
c57f56b7 3655 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3656 else comment += " data";
3657 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3658 func.ReplaceAll(".C", "");
3659 out << "void " << func.Data() << "()" << endl;
3660 out << "{" << endl;
3661 out << comment.Data() << endl;
3662 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3663 out << " TStopwatch timer;" << endl;
3664 out << " timer.Start();" << endl << endl;
13f0c00b 3665 // Change temp directory to current one
3666 out << "// Set temporary merging directory to current one" << endl;
3667 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3668 out << "// Set temporary compilation directory to current one" << endl;
3669 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3670 // Reset existing include path
3671 out << "// Reset existing include path and add current directory first in the search" << endl;
3672 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3673 if (!fExecutableCommand.Contains("aliroot")) {
3674 out << "// load base root libraries" << endl;
3675 out << " gSystem->Load(\"libTree\");" << endl;
3676 out << " gSystem->Load(\"libGeom\");" << endl;
3677 out << " gSystem->Load(\"libVMC\");" << endl;
3678 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3679 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3680 }
d5c6455a 3681 if (fAdditionalRootLibs.Length()) {
47a4137d 3682 // in principle libtree /lib geom libvmc etc. can go into this list, too
3683 out << "// Add aditional libraries" << endl;
3684 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3685 TIter next(list);
3686 TObjString *str;
3687 while((str=(TObjString*)next())) {
3688 if (str->GetString().Contains(".so"))
3689 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3690 }
47a4137d 3691 if (list) delete list;
d5c6455a 3692 }
57377eb5 3693 out << "// Load analysis framework libraries" << endl;
652561d5 3694 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3695 if (!fPackages) {
f790bc1b 3696 if (!fExecutableCommand.Contains("aliroot")) {
3697 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3698 out << " gSystem->Load(\"libESD\");" << endl;
3699 out << " gSystem->Load(\"libAOD\");" << endl;
3700 }
4e5c5506 3701 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3702 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3703 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3704 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3705 } else {
4e5c5506 3706 TIter next(fPackages);
3707 TObject *obj;
57377eb5 3708 TString pkgname;
57377eb5 3709 while ((obj=next())) {
3710 pkgname = obj->GetName();
4478e6f1 3711 if (pkgname == "STEERBase" ||
3712 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3713 if (pkgname == "ESD" ||
3714 pkgname == "ESD.par") hasESD = kTRUE;
3715 if (pkgname == "AOD" ||
3716 pkgname == "AOD.par") hasAOD = kTRUE;
3717 if (pkgname == "ANALYSIS" ||
3718 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 3719 if (pkgname == "OADB" ||
3720 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 3721 if (pkgname == "ANALYSISalice" ||
3722 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3723 if (pkgname == "CORRFW" ||
3724 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3725 }
3726 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3727 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3728 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3729 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3730 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3731 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3732 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3733 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3734 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 3735 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
3736 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 3737 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3738 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3739 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3740 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3741 out << "// Compile other par packages" << endl;
3742 next.Reset();
fcc9bb6f 3743 while ((obj=next())) {
3744 pkgname = obj->GetName();
4478e6f1 3745 if (pkgname == "STEERBase" ||
3746 pkgname == "STEERBase.par" ||
3747 pkgname == "ESD" ||
3748 pkgname == "ESD.par" ||
3749 pkgname == "AOD" ||
3750 pkgname == "AOD.par" ||
3751 pkgname == "ANALYSIS" ||
3752 pkgname == "ANALYSIS.par" ||
7ea40ac8 3753 pkgname == "OADB" ||
3754 pkgname == "OADB.par" ||
4478e6f1 3755 pkgname == "ANALYSISalice" ||
3756 pkgname == "ANALYSISalice.par" ||
3757 pkgname == "CORRFW" ||
3758 pkgname == "CORRFW.par") continue;
5b273635 3759 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3760 }
4e5c5506 3761 }
b3e07543 3762 out << "// include path" << endl;
3763 // Get the include path from the interpreter and remove entries pointing to AliRoot
3764 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3765 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3766 out << " TIter nextpath(listpaths);" << endl;
3767 out << " TObjString *pname;" << endl;
3768 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3769 out << " TString current = pname->GetName();" << endl;
3770 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3771 out << " gSystem->AddIncludePath(current);" << endl;
3772 out << " }" << endl;
3773 out << " if (listpaths) delete listpaths;" << endl;
3774 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3775 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3776 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3777 if (fAdditionalLibs.Length()) {
3778 out << "// Add aditional AliRoot libraries" << endl;
3779 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3780 TIter next(list);
3781 TObjString *str;
3782 while((str=(TObjString*)next())) {
3783 if (str->GetString().Contains(".so"))
3784 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3785 if (str->GetString().Contains(".par"))
3786 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3787 }
3788 if (list) delete list;
3789 }
3790 out << endl;
c57f56b7 3791 out << "// analysis source to be compiled at runtime (if any)" << endl;
3792 if (fAnalysisSource.Length()) {
3793 TObjArray *list = fAnalysisSource.Tokenize(" ");
3794 TIter next(list);
3795 TObjString *str;
3796 while((str=(TObjString*)next())) {
3797 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3798 }
3799 if (list) delete list;
3800 }
3801 out << endl;
b3e07543 3802// out << " printf(\"Currently load libraries:\\n\");" << endl;
3803// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3804 if (fFastReadOption) {
d3339be3 3805 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3806 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3807 out << "// fast xrootd reading enabled" << endl;
3808 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3809 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3810 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3811 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3812 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3813 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 3814 }
3815 if (!IsLocalTest()) {
3816 out << "// connect to AliEn and make the chain" << endl;
3817 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
0f389141 3818 }
c57f56b7 3819 out << "// read the analysis manager from file" << endl;
f10e8481 3820 TString analysisFile = fExecutable;
3821 analysisFile.ReplaceAll(".sh", ".root");
105631b5 3822 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
3823 << analysisFile << "\");" << endl;
3824 out << " if (!mgr) return;" << endl;
21ca8e59 3825 if (IsLocalTest()) {
3826 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
3827 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 3828 if (fFileForTestMode.IsNull())
3829 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
3830 else
3831 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 3832 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
21ca8e59 3833 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 3834 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 3835 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 3836 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
3837 } else {
7a366e0b 3838 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 3839 out << " mgr->SetNSysInfo(100);" << endl;
3840 }
21ca8e59 3841 }
c57f56b7 3842 out << " mgr->PrintStatus();" << endl;
52b6a92b 3843 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3844 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3845 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3846 } else {
9a8288ff 3847 if (TestBit(AliAnalysisGrid::kTest))
3848 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3849 else
3850 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3851 }
3852 }
21ca8e59 3853 if (!IsLocalTest()) {
242accb2 3854 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 3855 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
3856 } else {
3857 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 3858 }
f7498086 3859 out << " timer.Stop();" << endl;
3860 out << " timer.Print();" << endl;
c57f56b7 3861 out << "}" << endl << endl;
21ca8e59 3862 if (!IsLocalTest()) {
fcc9bb6f 3863 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3864 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3865 out << "{" << endl;
3866 out << "// Create a chain using url's from xml file" << endl;
242accb2 3867 out << " TString filename;" << endl;
3868 out << " Int_t run = 0;" << endl;
c85cfc0f 3869 if (IsUseMCchain()) {
3870 out << " TString treename = \"TE\";" << endl;
3871 } else {
3872 out << " TString treename = type;" << endl;
3873 out << " treename.ToLower();" << endl;
3874 out << " treename += \"Tree\";" << endl;
3875 }
e02fee64 3876 out << " printf(\"***************************************\\n\");" << endl;
3877 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3878 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3879 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3880 out << " if (!coll) {" << endl;
3881 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3882 out << " return NULL;" << endl;
3883 out << " }" << endl;
242accb2 3884 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3885 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3886 if(fFriendChainName!="") {
3887 out << " TChain *chainFriend = new TChain(treename);" << endl;
3888 }
c57f56b7 3889 out << " coll->Reset();" << endl;
0df6ccf2 3890 out << " while (coll->Next()) {" << endl;
242accb2 3891 out << " filename = coll->GetTURL("");" << endl;
3892 out << " if (mgr) {" << endl;
3893 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3894 out << " if (nrun && nrun != run) {" << endl;
3895 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3896 out << " mgr->SetRunFromPath(nrun);" << endl;
3897 out << " run = nrun;" << endl;
3898 out << " }" << endl;
3899 out << " }" << endl;
3900 out << " chain->Add(filename);" << endl;
0df6ccf2 3901 if(fFriendChainName!="") {
3902 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3903 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3904 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3905 out << " chainFriend->Add(fileFriend.Data());" << endl;
3906 }
3907 out << " }" << endl;
c57f56b7 3908 out << " if (!chain->GetNtrees()) {" << endl;
3909 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3910 out << " return NULL;" << endl;
3911 out << " }" << endl;
0df6ccf2 3912 if(fFriendChainName!="") {
3913 out << " chain->AddFriend(chainFriend);" << endl;
3914 }
c57f56b7 3915 out << " return chain;" << endl;
fcc9bb6f 3916 out << "}" << endl << endl;
c57f56b7 3917 }
5b273635 3918 if (hasANALYSISalice) {
3919 out <<"//________________________________________________________________________________" << endl;
3920 out << "Bool_t SetupPar(const char *package) {" << endl;
3921 out << "// Compile the package and set it up." << endl;
3922 out << " TString pkgdir = package;" << endl;
3923 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3924 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3925 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3926 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3927 out << " // Check for BUILD.sh and execute" << endl;
3928 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3929 out << " printf(\"*******************************\\n\");" << endl;
3930 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3931 out << " printf(\"*******************************\\n\");" << endl;
3932 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3933 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3934 out << " gSystem->ChangeDirectory(cdir);" << endl;
3935 out << " return kFALSE;" << endl;
3936 out << " }" << endl;
3937 out << " } else {" << endl;
3938 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3939 out << " gSystem->ChangeDirectory(cdir);" << endl;
3940 out << " return kFALSE;" << endl;
3941 out << " }" << endl;
3942 out << " // Check for SETUP.C and execute" << endl;
3943 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3944 out << " printf(\"*******************************\\n\");" << endl;
3945 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3946 out << " printf(\"*******************************\\n\");" << endl;
3947 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3948 out << " } else {" << endl;
3949 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3950 out << " gSystem->ChangeDirectory(cdir);" << endl;
3951 out << " return kFALSE;" << endl;
3952 out << " }" << endl;
3953 out << " // Restore original workdir" << endl;
3954 out << " gSystem->ChangeDirectory(cdir);" << endl;
3955 out << " return kTRUE;" << endl;
3956 out << "}" << endl;
3957 }
c57f56b7 3958 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3959 }
3960 Bool_t copy = kTRUE;
d3b18c4c 3961 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3962 if (copy) {
3963 CdWork();
3964 TString workdir = gGrid->GetHomeDirectory();
3965 workdir += fGridWorkingDir;
3966 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 3967 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 3968// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3969 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
3970 Form("alien://%s/%s", workdir.Data(),
3971 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 3972 }
3973}
3974
0f389141 3975//______________________________________________________________________________
3976void AliAnalysisAlien::WriteMergingMacro()
3977{
3978// Write a macro to merge the outputs per master job.
3979 if (!fMergeViaJDL) return;
3980 if (!fOutputFiles.Length()) {
3981 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3982 return;
3983 }
3984 TString mergingMacro = fExecutable;
3985 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 3986 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3987 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3988 ofstream out;
3989 out.open(mergingMacro.Data(), ios::out);
3990 if (!out.good()) {
3991 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3992 return;
3993 }
5b273635 3994 Bool_t hasSTEERBase = kFALSE;
3995 Bool_t hasESD = kFALSE;
3996 Bool_t hasAOD = kFALSE;
3997 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3998 Bool_t hasOADB = kFALSE;
5b273635 3999 Bool_t hasANALYSISalice = kFALSE;
4000 Bool_t hasCORRFW = kFALSE;
0f389141 4001 TString func = mergingMacro;
4002 TString comment;
4003 func.ReplaceAll(".C", "");
f47d5cb4 4004 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4005 out << "{" << endl;
4006 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4007 out << " TStopwatch timer;" << endl;
4008 out << " timer.Start();" << endl << endl;
b3e07543 4009 // Reset existing include path
4010 out << "// Reset existing include path and add current directory first in the search" << endl;
4011 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4012 if (!fExecutableCommand.Contains("aliroot")) {
4013 out << "// load base root libraries" << endl;
4014 out << " gSystem->Load(\"libTree\");" << endl;
4015 out << " gSystem->Load(\"libGeom\");" << endl;
4016 out << " gSystem->Load(\"libVMC\");" << endl;
4017 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4018 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4019 }
0f389141 4020 if (fAdditionalRootLibs.Length()) {
4021 // in principle libtree /lib geom libvmc etc. can go into this list, too
4022 out << "// Add aditional libraries" << endl;
4023 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4024 TIter next(list);
4025 TObjString *str;
4026 while((str=(TObjString*)next())) {
4027 if (str->GetString().Contains(".so"))
4028 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4029 }
4030 if (list) delete list;
4031 }
0f389141 4032 out << "// Load analysis framework libraries" << endl;
4033 if (!fPackages) {
7c2cd90a 4034 if (!fExecutableCommand.Contains("aliroot")) {
4035 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4036 out << " gSystem->Load(\"libESD\");" << endl;
4037 out << " gSystem->Load(\"libAOD\");" << endl;
4038 }
0f389141 4039 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4040 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4041 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4042 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4043 } else {
4044 TIter next(fPackages);
4045 TObject *obj;
4046 TString pkgname;
5b273635 4047 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4048 while ((obj=next())) {
4049 pkgname = obj->GetName();
4050 if (pkgname == "STEERBase" ||
4051 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4052 if (pkgname == "ESD" ||
4053 pkgname == "ESD.par") hasESD = kTRUE;
4054 if (pkgname == "AOD" ||
4055 pkgname == "AOD.par") hasAOD = kTRUE;
4056 if (pkgname == "ANALYSIS" ||
4057 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4058 if (pkgname == "OADB" ||
4059 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4060 if (pkgname == "ANALYSISalice" ||
4061 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4062 if (pkgname == "CORRFW" ||
4063 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4064 }
5b273635 4065 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4066 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4067 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4068 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4069 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4070 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4071 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4072 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4073 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4074 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4075 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4076 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4077 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4078 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4079 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4080 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4081 out << "// Compile other par packages" << endl;
4082 next.Reset();
4083 while ((obj=next())) {
4084 pkgname = obj->GetName();
4085 if (pkgname == "STEERBase" ||
4086 pkgname == "STEERBase.par" ||
4087 pkgname == "ESD" ||
4088 pkgname == "ESD.par" ||
4089 pkgname == "AOD" ||
4090 pkgname == "AOD.par" ||
4091 pkgname == "ANALYSIS" ||
4092 pkgname == "ANALYSIS.par" ||
7ea40ac8 4093 pkgname == "OADB" ||
4094 pkgname == "OADB.par" ||
0f389141 4095 pkgname == "ANALYSISalice" ||
4096 pkgname == "ANALYSISalice.par" ||
4097 pkgname == "CORRFW" ||
4098 pkgname == "CORRFW.par") continue;
5b273635 4099 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4100 }
4101 }
b3e07543 4102 out << "// include path" << endl;
4103 // Get the include path from the interpreter and remove entries pointing to AliRoot
4104 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4105 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4106 out << " TIter nextpath(listpaths);" << endl;
4107 out << " TObjString *pname;" << endl;
4108 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4109 out << " TString current = pname->GetName();" << endl;
4110 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4111 out << " gSystem->AddIncludePath(current);" << endl;
4112 out << " }" << endl;
4113 out << " if (listpaths) delete listpaths;" << endl;
4114 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4115 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4116 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4117 if (fAdditionalLibs.Length()) {
4118 out << "// Add aditional AliRoot libraries" << endl;
4119 TObjArray *list = fAdditionalLibs.Tokenize(" ");
4120 TIter next(list);
4121 TObjString *str;
4122 while((str=(TObjString*)next())) {
4123 if (str->GetString().Contains(".so"))
4124 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4125 }
4126 if (list) delete list;
4127 }
4128 out << endl;
4129 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4130 if (fAnalysisSource.Length()) {
4131 TObjArray *list = fAnalysisSource.Tokenize(" ");
4132 TIter next(list);
4133 TObjString *str;
4134 while((str=(TObjString*)next())) {
4135 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4136 }
4137 if (list) delete list;
4138 }
149d288c 4139 out << endl;
4140
0f389141 4141 if (fFastReadOption) {
4142 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4143 out << "// fast xrootd reading enabled" << endl;
4144 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4145 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4146 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4147 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4148 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4149 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4150 }
4151 // Change temp directory to current one
4152 out << "// Set temporary merging directory to current one" << endl;
4153 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4154 out << "// Set temporary compilation directory to current one" << endl;
4155 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
0f389141 4156 out << "// Connect to AliEn" << endl;
4157 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 4158 out << " TString outputDir = dir;" << endl;
f790bc1b 4159 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4160 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4161 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4162 out << " TIter *iter = new TIter(list);" << endl;
4163 out << " TObjString *str;" << endl;
a2f5fc01 4164 out << " TString outputFile;" << endl;
0f389141 4165 out << " Bool_t merged = kTRUE;" << endl;
4166 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4167 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4168 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4169 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4170 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4171 out << " // Skip already merged outputs" << endl;
a2f5fc01 4172 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
4173 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 4174 out << " continue;" << endl;
4175 out << " }" << endl;
a2f5fc01 4176 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4177 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4178 out << " if (!merged) {" << endl;
a2f5fc01 4179 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4180 out << " return;" << endl;
0f389141 4181 out << " }" << endl;
4182 out << " }" << endl;
7c2cd90a 4183 out << " // all outputs merged, validate" << endl;
4184 out << " ofstream out;" << endl;
4185 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4186 out << " out.close();" << endl;
4187 out << " // read the analysis manager from file" << endl;
f866cba5 4188 TString analysisFile = fExecutable;
4189 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 4190 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
105631b5 4191 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4192 << analysisFile << "\");" << endl;
4193 out << " if (!mgr) return;" << endl;
5c4250fc 4194 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4195 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4196 out << " mgr->PrintStatus();" << endl;
4197 if (AliAnalysisManager::GetAnalysisManager()) {
4198 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
4199 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4200 } else {
9a8288ff 4201 if (TestBit(AliAnalysisGrid::kTest))
4202 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4203 else
4204 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4205 }
4206 }
9a8288ff 4207 out << " TTree *tree = NULL;" << endl;
4208 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4209 out << "}" << endl << endl;
5b273635 4210 if (hasANALYSISalice) {
4211 out <<"//________________________________________________________________________________" << endl;
4212 out << "Bool_t SetupPar(const char *package) {" << endl;
4213 out << "// Compile the package and set it up." << endl;
4214 out << " TString pkgdir = package;" << endl;
4215 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4216 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4217 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4218 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4219 out << " // Check for BUILD.sh and execute" << endl;
4220 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4221 out << " printf(\"*******************************\\n\");" << endl;
4222 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4223 out << " printf(\"*******************************\\n\");" << endl;
4224 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4225 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4226 out << " gSystem->ChangeDirectory(cdir);" << endl;
4227 out << " return kFALSE;" << endl;
4228 out << " }" << endl;
4229 out << " } else {" << endl;
4230 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4231 out << " gSystem->ChangeDirectory(cdir);" << endl;
4232 out << " return kFALSE;" << endl;
4233 out << " }" << endl;
4234 out << " // Check for SETUP.C and execute" << endl;
4235 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4236 out << " printf(\"*******************************\\n\");" << endl;
4237 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4238 out << " printf(\"*******************************\\n\");" << endl;
4239 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4240 out << " } else {" << endl;
4241 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4242 out << " gSystem->ChangeDirectory(cdir);" << endl;
4243 out << " return kFALSE;" << endl;
4244 out << " }" << endl;
4245 out << " // Restore original workdir" << endl;
4246 out << " gSystem->ChangeDirectory(cdir);" << endl;
4247 out << " return kTRUE;" << endl;
4248 out << "}" << endl;
4249 }
0f389141 4250 }
4251 Bool_t copy = kTRUE;
d3b18c4c 4252 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4253 if (copy) {
4254 CdWork();
4255 TString workdir = gGrid->GetHomeDirectory();
4256 workdir += fGridWorkingDir;
4257 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4258 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4259// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4260 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4261 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4262 }
4263}
4264
4265//______________________________________________________________________________
4266Bool_t AliAnalysisAlien::SetupPar(const char *package)
4267{
205b201f 4268// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4269// Note that for loading the compiled library. The current directory should have precedence in
4270// LD_LIBRARY_PATH
4271 TString pkgdir = package;
4272 pkgdir.ReplaceAll(".par","");
b50a41c3 4273 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4274 TString cdir = gSystem->WorkingDirectory();
4275 gSystem->ChangeDirectory(pkgdir);
4276 // Check for BUILD.sh and execute
4277 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4278 printf("**************************************************\n");
4279 printf("*** Building PAR archive %s\n", package);
4280 printf("**************************************************\n");
4281 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4282 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4283 gSystem->ChangeDirectory(cdir);
4284 return kFALSE;
4285 }
4286 } else {
4287 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4288 gSystem->ChangeDirectory(cdir);
4289 return kFALSE;
4290 }
4291 // Check for SETUP.C and execute
4292 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4293 printf("**************************************************\n");
4294 printf("*** Setup PAR archive %s\n", package);
4295 printf("**************************************************\n");
4296 gROOT->Macro("PROOF-INF/SETUP.C");
4297 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4298 } else {
4299 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4300 gSystem->ChangeDirectory(cdir);
4301 return kFALSE;
4302 }
4303 // Restore original workdir
4304 gSystem->ChangeDirectory(cdir);
4305 return kTRUE;
4306}
4307
c57f56b7 4308//______________________________________________________________________________
4309void AliAnalysisAlien::WriteExecutable()
4310{
4311// Generate the alien executable script.
4312 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4313 ofstream out;
4314 out.open(fExecutable.Data(), ios::out);
4315 if (out.bad()) {
5513444a 4316 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4317 return;
4318 }
4319 out << "#!/bin/bash" << endl;
2a4d5166 4320 // Make sure we can properly compile par files
4321 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4322 out << "echo \"=========================================\"" << endl;
4323 out << "echo \"############## PATH : ##############\"" << endl;
4324 out << "echo $PATH" << endl;
4325 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4326 out << "echo $LD_LIBRARY_PATH" << endl;
4327 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4328 out << "echo $ROOTSYS" << endl;
4329 out << "echo \"############## which root : ##############\"" << endl;
4330 out << "which root" << endl;
4331 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4332 out << "echo $ALICE_ROOT" << endl;
4333 out << "echo \"############## which aliroot : ##############\"" << endl;
4334 out << "which aliroot" << endl;
9c5ddadc 4335 out << "echo \"############## system limits : ##############\"" << endl;
4336 out << "ulimit -a" << endl;
4337 out << "echo \"############## memory : ##############\"" << endl;
4338 out << "free -m" << endl;
c57f56b7 4339 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4340 out << fExecutableCommand << " ";
631c0b05 4341 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 4342 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
4343 out << "echo \"############## memory after: ##############\"" << endl;
4344 out << "free -m" << endl;
c57f56b7 4345 }
4346 Bool_t copy = kTRUE;
d3b18c4c 4347 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4348 if (copy) {
4349 CdWork();
4350 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4351 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4352 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4353 workdir += fGridWorkingDir;
4354 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4355 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4356 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4357// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4358 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4359 executable.Data())) Fatal("","Terminating");
c57f56b7 4360 }
4361}
4362
0f389141 4363//______________________________________________________________________________
4364void AliAnalysisAlien::WriteMergeExecutable()
4365{
4366// Generate the alien executable script for the merging job.
4367 if (!fMergeViaJDL) return;
4368 TString mergeExec = fExecutable;
4369 mergeExec.ReplaceAll(".sh", "_merge.sh");
4370 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4371 ofstream out;
4372 out.open(mergeExec.Data(), ios::out);
4373 if (out.bad()) {
4374 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4375 return;
4376 }
4377 out << "#!/bin/bash" << endl;
2a4d5166 4378 // Make sure we can properly compile par files
4379 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4380 out << "echo \"=========================================\"" << endl;
4381 out << "echo \"############## PATH : ##############\"" << endl;
4382 out << "echo $PATH" << endl;
4383 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4384 out << "echo $LD_LIBRARY_PATH" << endl;
4385 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4386 out << "echo $ROOTSYS" << endl;
4387 out << "echo \"############## which root : ##############\"" << endl;
4388 out << "which root" << endl;
4389 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4390 out << "echo $ALICE_ROOT" << endl;
4391 out << "echo \"############## which aliroot : ##############\"" << endl;
4392 out << "which aliroot" << endl;
4393 out << "echo \"############## system limits : ##############\"" << endl;
4394 out << "ulimit -a" << endl;
4395 out << "echo \"############## memory : ##############\"" << endl;
4396 out << "free -m" << endl;
4397 out << "echo \"=========================================\"" << endl << endl;
0f389141 4398 TString mergeMacro = fExecutable;
4399 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4400 if (IsOneStageMerging())
4401 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4402 else
f47d5cb4 4403 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4404 out << fExecutableCommand << " " << "$ARG" << endl;
4405 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4406 out << "echo \"############## memory after: ##############\"" << endl;
4407 out << "free -m" << endl;
0f389141 4408 }
4409 Bool_t copy = kTRUE;
d3b18c4c 4410 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4411 if (copy) {
4412 CdWork();
4413 TString workdir = gGrid->GetHomeDirectory();
4414 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4415 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4416 workdir += fGridWorkingDir;
4417 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4418 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4419 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4420// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4421 if (!copyLocal2Alien("WriteMergeExecutable",
4422 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4423 }
4424}
4425
c57f56b7 4426//______________________________________________________________________________
5513444a 4427void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4428{
4429// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4430// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4431// Next lines: full_path_to_dataset XXX (XXX is a string)
4432// To submit, one has to: submit jdl XXX for all lines
4433 ofstream out;
4434 out.open(filename, ios::out);
4435 if (out.bad()) {
4436 Error("WriteProductionFile", "Bad file name: %s", filename);
4437 return;
4438 }
d3b18c4c 4439 TString workdir;
4440 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4441 workdir = gGrid->GetHomeDirectory();
5513444a 4442 workdir += fGridWorkingDir;
f5e8c702 4443 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4444 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4445 out << locjdl << " " << njobspermaster << endl;
5513444a 4446 Int_t nmasterjobs = fInputFiles->GetEntries();
4447 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4448 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4449 runOutDir.ReplaceAll(".xml", "");
4450 if (fOutputToRunNo)
4451 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4452 else
4453 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4454 }
d3b18c4c 4455 if (gGrid) {
4456 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4457 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4458// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4459 if (!copyLocal2Alien("WriteProductionFile", filename,
4460 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4461 }
5513444a 4462}
4463
4464//______________________________________________________________________________
0f389141 4465void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4466{
4467// Generate the alien validation script.
4468 // Generate the validation script
4469 TObjString *os;
d3b18c4c 4470 if (fValidationScript.IsNull()) {
4471 fValidationScript = fExecutable;
4472 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4473 }
4474 TString validationScript = fValidationScript;
4475 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4476 if (!Connect()) {
4477 Error("WriteValidationScript", "Alien connection required");
4478 return;
4479 }
d3b18c4c 4480 if (!fTerminateFiles.IsNull()) {
4481 fTerminateFiles.Strip();
4482 fTerminateFiles.ReplaceAll(" ",",");
4483 }
a2f5fc01 4484 TString outStream = "";
4485 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4486 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4487 ofstream out;
0d5d317c 4488 out.open(validationScript, ios::out);
c57f56b7 4489 out << "#!/bin/bash" << endl;
4490 out << "##################################################" << endl;
4491 out << "validateout=`dirname $0`" << endl;
4492 out << "validatetime=`date`" << endl;
4493 out << "validated=\"0\";" << endl;
4494 out << "error=0" << endl;
4495 out << "if [ -z $validateout ]" << endl;
4496 out << "then" << endl;
4497 out << " validateout=\".\"" << endl;
4498 out << "fi" << endl << endl;
4499 out << "cd $validateout;" << endl;
4500 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4501 out << "echo \"*******************************************************\"" << outStream << endl;
4502 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4503 out << "" << endl;
a2f5fc01 4504 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4505 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4506 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4507 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4508 out << "ls -la ./" << outStream << endl;
4509 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4510 out << "##################################################" << endl;
ebec370a 4511 out << "" << endl;
4512
4513 out << "if [ ! -f stderr ] ; then" << endl;
4514 out << " error=1" << endl;
a2f5fc01 4515 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4516 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4517 out << "fi" << endl;
4518
b34c9f51 4519 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4520 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4521 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4522 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4523 out << "" << endl;
4524
ebec370a 4525 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4526 out << " error=1" << endl;
a2f5fc01 4527 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4528 out << " echo \"$parArch\" " << outStream << endl;
4529 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4530 out << "fi" << endl;
4531
4532 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4533 out << " error=1" << endl;
a2f5fc01 4534 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4535 out << " echo \"$segViol\" " << outStream << endl;
4536 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4537 out << "fi" << endl;
4538
4539 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4540 out << " error=1" << endl;
a2f5fc01 4541 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4542 out << " echo \"$segFault\" " << outStream << endl;
4543 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4544 out << "fi" << endl;
4545
b34c9f51 4546 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4547 out << " error=1" << endl;
a2f5fc01 4548 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4549 out << " echo \"$glibcErr\" " << outStream << endl;
4550 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4551 out << "fi" << endl;
4552
ebec370a 4553 // Part dedicated to the specific analyses running into the train
4554
d3b18c4c 4555 TString outputFiles = fOutputFiles;
4556 if (merge && !fTerminateFiles.IsNull()) {
4557 outputFiles += ",";
4558 outputFiles += fTerminateFiles;
4559 }
4560 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4561 TIter next1(arr);
a2f5fc01 4562 TString outputFile;
bbd88db8 4563 while (!merge && (os=(TObjString*)next1())) {
4564 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4565 outputFile = os->GetString();
4566 Int_t index = outputFile.Index("@");
4567 if (index > 0) outputFile.Remove(index);
bbd88db8 4568 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4569 if (outputFile.Contains("*")) continue;
4570 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4571 out << " error=1" << endl;
7c2cd90a 4572 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4573 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4574 out << "fi" << endl;
4575 }
4576 delete arr;
7c2cd90a 4577 out << "if ! [ -f outputs_valid ] ; then" << endl;
4578 out << " error=1" << endl;
4579 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4580 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4581 out << "fi" << endl;
923e2ca5 4582
c57f56b7 4583 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4584 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4585 if (!IsKeepLogs()) {
4586 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4587 outStream = "";
149d288c 4588 out << " rm -f std*" << endl;
4589 }
c57f56b7 4590 out << "fi" << endl;
4591
a2f5fc01 4592 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4593 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4594 out << "cd -" << endl;
4595 out << "exit $error" << endl;
4596 }
4597 Bool_t copy = kTRUE;
d3b18c4c 4598 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4599 if (copy) {
4600 CdWork();
4601 TString workdir = gGrid->GetHomeDirectory();
4602 workdir += fGridWorkingDir;
d3b18c4c 4603 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4604 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 4605// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4606 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
4607 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 4608 }
4609}