]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
adding ITS-SPD-vertexer to the inputs of GLOBAL-esd-converter (Hege)
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
51ClassImp(AliAnalysisAlien)
6c395669 52#if 0
53;
54#endif
c57f56b7 55
6c395669 56namespace {
57 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
58 {
59 TString sl(Form("file:%s", loc));
60 TString sr(Form("alien://%s", rem));
61 Bool_t ret = TFile::Cp(sl, sr);
62 if (!ret) {
63 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
64 }
65 return ret;
66 }
67}
68
c57f56b7 69//______________________________________________________________________________
70AliAnalysisAlien::AliAnalysisAlien()
71 :AliAnalysisGrid(),
72 fGridJDL(NULL),
0f389141 73 fMergingJDL(NULL),
c57f56b7 74 fPrice(0),
75 fTTL(0),
76 fSplitMaxInputFileNumber(0),
77 fMaxInitFailed(0),
78 fMasterResubmitThreshold(0),
bb885a9e 79 fNtestFiles(0),
319593fb 80 fNrunsPerMaster(0),
16a4353c 81 fMaxMergeFiles(0),
b3e07543 82 fMaxMergeStages(0),
d2a409b2 83 fNsubmitted(0),
a3e84053 84 fProductionMode(0),
cd11251e 85 fOutputToRunNo(0),
0f389141 86 fMergeViaJDL(0),
87 fFastReadOption(0),
e1c22e21 88 fOverwriteMode(1),
149d288c 89 fNreplicas(2),
3bdcb562 90 fNproofWorkers(0),
27734f0e 91 fNproofWorkersPerSlave(0),
3bdcb562 92 fProofReset(0),
c57f56b7 93 fRunNumbers(),
94 fExecutable(),
0a1c1f7f 95 fExecutableCommand(),
c57f56b7 96 fArguments(),
631c0b05 97 fExecutableArgs(),
c57f56b7 98 fAnalysisMacro(),
99 fAnalysisSource(),
d3b18c4c 100 fValidationScript(),
d5c6455a 101 fAdditionalRootLibs(),
c57f56b7 102 fAdditionalLibs(),
103 fSplitMode(),
104 fAPIVersion(),
105 fROOTVersion(),
106 fAliROOTVersion(),
648174cf 107 fExternalPackages(),
c57f56b7 108 fUser(),
109 fGridWorkingDir(),
110 fGridDataDir(),
111 fDataPattern(),
112 fGridOutputDir(),
113 fOutputArchive(),
114 fOutputFiles(),
115 fInputFormat(),
e7c71df0 116 fDatasetName(),
c57f56b7 117 fJDLName(),
d3b18c4c 118 fTerminateFiles(),
bb885a9e 119 fMergeExcludes(),
37d21c01 120 fRegisterExcludes(),
f965131e 121 fIncludePath(),
bb885a9e 122 fCloseSE(),
0df6ccf2 123 fFriendChainName(),
c6cb3634 124 fJobTag(),
648174cf 125 fOutputSingle(),
5fce53f4 126 fRunPrefix(),
3bdcb562 127 fProofCluster(),
128 fProofDataSet(),
129 fFileForTestMode(),
130 fRootVersionForProof(),
131 fAliRootMode(),
f47d5cb4 132 fMergeDirName(),
4e5c5506 133 fInputFiles(0),
28ce6c9c 134 fPackages(0),
77f34eae 135 fModules(0),
28ce6c9c 136 fProofParam()
c57f56b7 137{
138// Dummy ctor.
139 SetDefaults();
140}
141
142//______________________________________________________________________________
143AliAnalysisAlien::AliAnalysisAlien(const char *name)
144 :AliAnalysisGrid(name),
145 fGridJDL(NULL),
0f389141 146 fMergingJDL(NULL),
c57f56b7 147 fPrice(0),
148 fTTL(0),
149 fSplitMaxInputFileNumber(0),
150 fMaxInitFailed(0),
151 fMasterResubmitThreshold(0),
bb885a9e 152 fNtestFiles(0),
319593fb 153 fNrunsPerMaster(0),
16a4353c 154 fMaxMergeFiles(0),
b3e07543 155 fMaxMergeStages(0),
d2a409b2 156 fNsubmitted(0),
a3e84053 157 fProductionMode(0),
cd11251e 158 fOutputToRunNo(0),
0f389141 159 fMergeViaJDL(0),
160 fFastReadOption(0),
e1c22e21 161 fOverwriteMode(1),
149d288c 162 fNreplicas(2),
3bdcb562 163 fNproofWorkers(0),
27734f0e 164 fNproofWorkersPerSlave(0),
3bdcb562 165 fProofReset(0),
c57f56b7 166 fRunNumbers(),
167 fExecutable(),
0a1c1f7f 168 fExecutableCommand(),
c57f56b7 169 fArguments(),
631c0b05 170 fExecutableArgs(),
c57f56b7 171 fAnalysisMacro(),
172 fAnalysisSource(),
d3b18c4c 173 fValidationScript(),
d5c6455a 174 fAdditionalRootLibs(),
c57f56b7 175 fAdditionalLibs(),
176 fSplitMode(),
177 fAPIVersion(),
178 fROOTVersion(),
179 fAliROOTVersion(),
648174cf 180 fExternalPackages(),
c57f56b7 181 fUser(),
182 fGridWorkingDir(),
183 fGridDataDir(),
184 fDataPattern(),
185 fGridOutputDir(),
186 fOutputArchive(),
187 fOutputFiles(),
188 fInputFormat(),
e7c71df0 189 fDatasetName(),
c57f56b7 190 fJDLName(),
d3b18c4c 191 fTerminateFiles(),
bb885a9e 192 fMergeExcludes(),
37d21c01 193 fRegisterExcludes(),
f965131e 194 fIncludePath(),
bb885a9e 195 fCloseSE(),
0df6ccf2 196 fFriendChainName(),
c6cb3634 197 fJobTag(),
648174cf 198 fOutputSingle(),
5fce53f4 199 fRunPrefix(),
3bdcb562 200 fProofCluster(),
201 fProofDataSet(),
202 fFileForTestMode(),
203 fRootVersionForProof(),
204 fAliRootMode(),
f47d5cb4 205 fMergeDirName(),
4e5c5506 206 fInputFiles(0),
28ce6c9c 207 fPackages(0),
77f34eae 208 fModules(0),
28ce6c9c 209 fProofParam()
c57f56b7 210{
211// Default ctor.
212 SetDefaults();
213}
214
215//______________________________________________________________________________
216AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
217 :AliAnalysisGrid(other),
218 fGridJDL(NULL),
0f389141 219 fMergingJDL(NULL),
c57f56b7 220 fPrice(other.fPrice),
221 fTTL(other.fTTL),
222 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
223 fMaxInitFailed(other.fMaxInitFailed),
224 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 225 fNtestFiles(other.fNtestFiles),
319593fb 226 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 227 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 228 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 229 fNsubmitted(other.fNsubmitted),
a3e84053 230 fProductionMode(other.fProductionMode),
cd11251e 231 fOutputToRunNo(other.fOutputToRunNo),
0f389141 232 fMergeViaJDL(other.fMergeViaJDL),
233 fFastReadOption(other.fFastReadOption),
234 fOverwriteMode(other.fOverwriteMode),
149d288c 235 fNreplicas(other.fNreplicas),
3bdcb562 236 fNproofWorkers(other.fNproofWorkers),
27734f0e 237 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 238 fProofReset(other.fProofReset),
c57f56b7 239 fRunNumbers(other.fRunNumbers),
240 fExecutable(other.fExecutable),
0a1c1f7f 241 fExecutableCommand(other.fExecutableCommand),
c57f56b7 242 fArguments(other.fArguments),
631c0b05 243 fExecutableArgs(other.fExecutableArgs),
c57f56b7 244 fAnalysisMacro(other.fAnalysisMacro),
245 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 246 fValidationScript(other.fValidationScript),
d5c6455a 247 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 248 fAdditionalLibs(other.fAdditionalLibs),
249 fSplitMode(other.fSplitMode),
250 fAPIVersion(other.fAPIVersion),
251 fROOTVersion(other.fROOTVersion),
252 fAliROOTVersion(other.fAliROOTVersion),
648174cf 253 fExternalPackages(other.fExternalPackages),
c57f56b7 254 fUser(other.fUser),
255 fGridWorkingDir(other.fGridWorkingDir),
256 fGridDataDir(other.fGridDataDir),
257 fDataPattern(other.fDataPattern),
258 fGridOutputDir(other.fGridOutputDir),
259 fOutputArchive(other.fOutputArchive),
260 fOutputFiles(other.fOutputFiles),
261 fInputFormat(other.fInputFormat),
e7c71df0 262 fDatasetName(other.fDatasetName),
c57f56b7 263 fJDLName(other.fJDLName),
d3b18c4c 264 fTerminateFiles(other.fTerminateFiles),
bb885a9e 265 fMergeExcludes(other.fMergeExcludes),
37d21c01 266 fRegisterExcludes(other.fRegisterExcludes),
f965131e 267 fIncludePath(other.fIncludePath),
bb885a9e 268 fCloseSE(other.fCloseSE),
0df6ccf2 269 fFriendChainName(other.fFriendChainName),
c6cb3634 270 fJobTag(other.fJobTag),
648174cf 271 fOutputSingle(other.fOutputSingle),
5fce53f4 272 fRunPrefix(other.fRunPrefix),
3bdcb562 273 fProofCluster(other.fProofCluster),
274 fProofDataSet(other.fProofDataSet),
275 fFileForTestMode(other.fFileForTestMode),
276 fRootVersionForProof(other.fRootVersionForProof),
277 fAliRootMode(other.fAliRootMode),
f47d5cb4 278 fMergeDirName(other.fMergeDirName),
4e5c5506 279 fInputFiles(0),
28ce6c9c 280 fPackages(0),
77f34eae 281 fModules(0),
28ce6c9c 282 fProofParam()
c57f56b7 283{
284// Copy ctor.
285 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 286 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 287 fRunRange[0] = other.fRunRange[0];
288 fRunRange[1] = other.fRunRange[1];
c57f56b7 289 if (other.fInputFiles) {
290 fInputFiles = new TObjArray();
291 TIter next(other.fInputFiles);
292 TObject *obj;
293 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
294 fInputFiles->SetOwner();
295 }
4e5c5506 296 if (other.fPackages) {
297 fPackages = new TObjArray();
298 TIter next(other.fPackages);
299 TObject *obj;
300 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
301 fPackages->SetOwner();
302 }
77f34eae 303 if (other.fModules) {
304 fModules = new TObjArray();
305 fModules->SetOwner();
306 TIter next(other.fModules);
307 AliAnalysisTaskCfg *mod, *crt;
308 while ((crt=(AliAnalysisTaskCfg*)next())) {
309 mod = new AliAnalysisTaskCfg(*crt);
310 fModules->Add(mod);
311 }
312 }
c57f56b7 313}
314
315//______________________________________________________________________________
316AliAnalysisAlien::~AliAnalysisAlien()
317{
318// Destructor.
77f34eae 319 delete fGridJDL;
320 delete fMergingJDL;
321 delete fInputFiles;
322 delete fPackages;
323 delete fModules;
28ce6c9c 324 fProofParam.DeleteAll();
c57f56b7 325}
326
327//______________________________________________________________________________
328AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
329{
330// Assignment.
331 if (this != &other) {
332 AliAnalysisGrid::operator=(other);
333 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 334 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 335 fPrice = other.fPrice;
336 fTTL = other.fTTL;
337 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
338 fMaxInitFailed = other.fMaxInitFailed;
339 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 340 fNtestFiles = other.fNtestFiles;
a3e84053 341 fNrunsPerMaster = other.fNrunsPerMaster;
342 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 343 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 344 fNsubmitted = other.fNsubmitted;
345 fProductionMode = other.fProductionMode;
cd11251e 346 fOutputToRunNo = other.fOutputToRunNo;
0f389141 347 fMergeViaJDL = other.fMergeViaJDL;
348 fFastReadOption = other.fFastReadOption;
349 fOverwriteMode = other.fOverwriteMode;
149d288c 350 fNreplicas = other.fNreplicas;
3bdcb562 351 fNproofWorkers = other.fNproofWorkers;
27734f0e 352 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 353 fProofReset = other.fProofReset;
c57f56b7 354 fRunNumbers = other.fRunNumbers;
355 fExecutable = other.fExecutable;
0a1c1f7f 356 fExecutableCommand = other.fExecutableCommand;
c57f56b7 357 fArguments = other.fArguments;
631c0b05 358 fExecutableArgs = other.fExecutableArgs;
c57f56b7 359 fAnalysisMacro = other.fAnalysisMacro;
360 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 361 fValidationScript = other.fValidationScript;
d5c6455a 362 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 363 fAdditionalLibs = other.fAdditionalLibs;
364 fSplitMode = other.fSplitMode;
365 fAPIVersion = other.fAPIVersion;
366 fROOTVersion = other.fROOTVersion;
367 fAliROOTVersion = other.fAliROOTVersion;
648174cf 368 fExternalPackages = other.fExternalPackages;
c57f56b7 369 fUser = other.fUser;
370 fGridWorkingDir = other.fGridWorkingDir;
371 fGridDataDir = other.fGridDataDir;
372 fDataPattern = other.fDataPattern;
373 fGridOutputDir = other.fGridOutputDir;
374 fOutputArchive = other.fOutputArchive;
375 fOutputFiles = other.fOutputFiles;
376 fInputFormat = other.fInputFormat;
e7c71df0 377 fDatasetName = other.fDatasetName;
c57f56b7 378 fJDLName = other.fJDLName;
d3b18c4c 379 fTerminateFiles = other.fTerminateFiles;
bb885a9e 380 fMergeExcludes = other.fMergeExcludes;
37d21c01 381 fRegisterExcludes = other.fRegisterExcludes;
f965131e 382 fIncludePath = other.fIncludePath;
bb885a9e 383 fCloseSE = other.fCloseSE;
0df6ccf2 384 fFriendChainName = other.fFriendChainName;
c6cb3634 385 fJobTag = other.fJobTag;
648174cf 386 fOutputSingle = other.fOutputSingle;
5fce53f4 387 fRunPrefix = other.fRunPrefix;
3bdcb562 388 fProofCluster = other.fProofCluster;
389 fProofDataSet = other.fProofDataSet;
390 fFileForTestMode = other.fFileForTestMode;
391 fRootVersionForProof = other.fRootVersionForProof;
392 fAliRootMode = other.fAliRootMode;
f47d5cb4 393 fMergeDirName = other.fMergeDirName;
c57f56b7 394 if (other.fInputFiles) {
395 fInputFiles = new TObjArray();
396 TIter next(other.fInputFiles);
397 TObject *obj;
398 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
399 fInputFiles->SetOwner();
400 }
4e5c5506 401 if (other.fPackages) {
402 fPackages = new TObjArray();
403 TIter next(other.fPackages);
404 TObject *obj;
405 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
406 fPackages->SetOwner();
407 }
77f34eae 408 if (other.fModules) {
409 fModules = new TObjArray();
410 fModules->SetOwner();
411 TIter next(other.fModules);
412 AliAnalysisTaskCfg *mod, *crt;
413 while ((crt=(AliAnalysisTaskCfg*)next())) {
414 mod = new AliAnalysisTaskCfg(*crt);
415 fModules->Add(mod);
416 }
417 }
c57f56b7 418 }
419 return *this;
420}
421
37d21c01 422//______________________________________________________________________________
423void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
424{
425// Add a single additional library to be loaded. Extension must be present.
426 TString lib(name);
427 if (!lib.Contains(".")) {
428 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
429 return;
430 }
431 if (fAdditionalLibs.Contains(name)) {
432 Warning("AddAdditionalLibrary", "Library %s already added.", name);
433 return;
434 }
435 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
436 fAdditionalLibs += lib;
437}
438
77f34eae 439//______________________________________________________________________________
440void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
441{
442// Adding a module. Checks if already existing. Becomes owned by this.
443 if (!module) return;
444 if (GetModule(module->GetName())) {
445 Error("AddModule", "A module having the same name %s already added", module->GetName());
446 return;
447 }
448 if (!fModules) {
449 fModules = new TObjArray();
450 fModules->SetOwner();
451 }
452 fModules->Add(module);
453}
454
455//______________________________________________________________________________
456void AliAnalysisAlien::AddModules(TObjArray *list)
457{
458// Adding a list of modules. Checks if already existing. Becomes owned by this.
459 TIter next(list);
460 AliAnalysisTaskCfg *module;
461 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
462}
463
464//______________________________________________________________________________
465Bool_t AliAnalysisAlien::CheckDependencies()
466{
467// Check if all dependencies are satisfied. Reorder modules if needed.
468 Int_t nmodules = GetNmodules();
469 if (!nmodules) {
470 Warning("CheckDependencies", "No modules added yet to check their dependencies");
471 return kTRUE;
472 }
473 AliAnalysisTaskCfg *mod = 0;
474 AliAnalysisTaskCfg *dep = 0;
475 TString depname;
476 Int_t i, j, k;
477 for (i=0; i<nmodules; i++) {
478 mod = (AliAnalysisTaskCfg*) fModules->At(i);
479 Int_t ndeps = mod->GetNdeps();
480 Int_t istart = i;
481 for (j=0; j<ndeps; j++) {
482 depname = mod->GetDependency(j);
483 dep = GetModule(depname);
484 if (!dep) {
485 Error("CheckDependencies","Dependency %s not added for module %s",
486 depname.Data(), mod->GetName());
487 return kFALSE;
488 }
489 if (dep->NeedsDependency(mod->GetName())) {
490 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
491 mod->GetName(), dep->GetName());
492 return kFALSE;
493 }
494 Int_t idep = fModules->IndexOf(dep);
495 // The dependency task must come first
496 if (idep>i) {
497 // Remove at idep and move all objects below up one slot
498 // down to index i included.
499 fModules->RemoveAt(idep);
3e40fd4c 500 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 501 fModules->AddAt(dep, i++);
502 }
503 //Redo from istart if dependencies were inserted
504 if (i>istart) i=istart-1;
505 }
506 }
507 return kTRUE;
508}
509
3e40fd4c 510//______________________________________________________________________________
511AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
512{
513// Create the analysis manager and optionally execute the macro in filename.
514 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
515 if (mgr) return mgr;
516 mgr = new AliAnalysisManager(name);
517 mgr->SetGridHandler((AliAnalysisGrid*)this);
518 if (strlen(filename)) {
519 TString line = gSystem->ExpandPathName(filename);
520 line.Prepend(".x ");
521 gROOT->ProcessLine(line.Data());
522 }
523 return mgr;
524}
525
77f34eae 526//______________________________________________________________________________
527Int_t AliAnalysisAlien::GetNmodules() const
528{
529// Get number of modules.
530 if (!fModules) return 0;
531 return fModules->GetEntries();
532}
533
534//______________________________________________________________________________
535AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
536{
537// Get a module by name.
538 if (!fModules) return 0;
539 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
540}
541
542//______________________________________________________________________________
543Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
544{
545// Load a given module.
546 if (mod->IsLoaded()) return kTRUE;
547 Int_t ndeps = mod->GetNdeps();
548 TString depname;
549 for (Int_t j=0; j<ndeps; j++) {
550 depname = mod->GetDependency(j);
551 AliAnalysisTaskCfg *dep = GetModule(depname);
552 if (!dep) {
553 Error("LoadModule","Dependency %s not existing for module %s",
554 depname.Data(), mod->GetName());
555 return kFALSE;
556 }
557 if (!LoadModule(dep)) {
558 Error("LoadModule","Dependency %s for module %s could not be loaded",
559 depname.Data(), mod->GetName());
560 return kFALSE;
561 }
562 }
563 // Load libraries for the module
564 if (!mod->CheckLoadLibraries()) {
565 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
566 return kFALSE;
567 }
568 // Execute the macro
569 if (mod->ExecuteMacro()<0) {
570 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
571 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
572 return kFALSE;
573 }
574 // Configure dependencies
575 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
576 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
577 mod->GetConfigMacro()->GetTitle(), mod->GetName());
578 return kFALSE;
579 }
5e8c7c22 580 // Adjust extra libraries
581 Int_t nlibs = mod->GetNlibs();
582 TString lib;
583 for (Int_t i=0; i<nlibs; i++) {
584 lib = mod->GetLibrary(i);
585 if (fAdditionalLibs.Contains(lib)) continue;
586 lib = Form("lib%s.so", lib.Data());
587 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
588 fAdditionalLibs += lib;
589 }
77f34eae 590 return kTRUE;
591}
592
139fe6ba 593//______________________________________________________________________________
594Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
595{
596// Generate the full train.
597 fAdditionalLibs = "";
598 if (!LoadModules()) return kFALSE;
599 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
600 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 601 mgr->RunLocalInit();
139fe6ba 602 mgr->PrintStatus();
603 Int_t productionMode = fProductionMode;
604 SetProductionMode();
605 TString macro = fAnalysisMacro;
606 TString executable = fExecutable;
607 TString validation = fValidationScript;
608 TString execCommand = fExecutableCommand;
609 SetAnalysisMacro(Form("%s.C", name));
610 SetExecutable(Form("%s.sh", name));
611 SetExecutableCommand("aliroot -b -q ");
612 SetValidationScript(Form("%s_validation.sh", name));
613 StartAnalysis();
614 SetProductionMode(productionMode);
615 fAnalysisMacro = macro;
616 fExecutable = executable;
617 fExecutableCommand = execCommand;
618 fValidationScript = validation;
619 return kTRUE;
620}
621
77f34eae 622//______________________________________________________________________________
21ca8e59 623Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 624{
625// Generate test macros for a single module or for the full train.
5e8c7c22 626 fAdditionalLibs = "";
77f34eae 627 if (strlen(modname)) {
21ca8e59 628 if (!CheckDependencies()) return kFALSE;
629 AliAnalysisTaskCfg *mod = GetModule(modname);
630 if (!mod) {
631 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
632 return kFALSE;
633 }
634 if (!LoadModule(mod)) return kFALSE;
635 } else if (!LoadModules()) return kFALSE;
636 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
637 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 638 mgr->RunLocalInit();
21ca8e59 639 mgr->PrintStatus();
640 SetLocalTest(kTRUE);
641 Int_t productionMode = fProductionMode;
642 SetProductionMode();
643 TString macro = fAnalysisMacro;
644 TString executable = fExecutable;
645 TString validation = fValidationScript;
646 TString execCommand = fExecutableCommand;
647 SetAnalysisMacro(Form("%s.C", name));
648 SetExecutable(Form("%s.sh", name));
649 SetExecutableCommand("aliroot -b -q ");
650 SetValidationScript(Form("%s_validation.sh", name));
651 WriteAnalysisFile();
652 WriteAnalysisMacro();
653 WriteExecutable();
654 WriteValidationScript();
655 SetLocalTest(kFALSE);
656 SetProductionMode(productionMode);
657 fAnalysisMacro = macro;
658 fExecutable = executable;
659 fExecutableCommand = execCommand;
660 fValidationScript = validation;
77f34eae 661 return kTRUE;
662}
663
664//______________________________________________________________________________
665Bool_t AliAnalysisAlien::LoadModules()
666{
667// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 668 fAdditionalLibs = "";
21ca8e59 669 Int_t nmodules = GetNmodules();
670 if (!nmodules) {
671 Warning("LoadModules", "No module to be loaded");
672 return kTRUE;
673 }
3e40fd4c 674 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
675 if (!mgr) {
676 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
677 return kFALSE;
678 }
77f34eae 679 if (!CheckDependencies()) return kFALSE;
21ca8e59 680 nmodules = GetNmodules();
77f34eae 681 AliAnalysisTaskCfg *mod;
21ca8e59 682 for (Int_t imod=0; imod<nmodules; imod++) {
683 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 684 if (!LoadModule(mod)) return kFALSE;
685 }
77f34eae 686 return kTRUE;
687}
688
74013535 689//______________________________________________________________________________
690void AliAnalysisAlien::SetRunPrefix(const char *prefix)
691{
692// Set the run number format. Can be a prefix or a format like "%09d"
693 fRunPrefix = prefix;
694 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
695}
696
f965131e 697//______________________________________________________________________________
698void AliAnalysisAlien::AddIncludePath(const char *path)
699{
700// Add include path in the remote analysis macro.
701 TString p(path);
702 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
703 else fIncludePath += Form("-I%s ", path);
704}
705
c57f56b7 706//______________________________________________________________________________
707void AliAnalysisAlien::AddRunNumber(Int_t run)
708{
709// Add a run number to the list of runs to be processed.
710 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 711 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 712}
713
98690895 714//______________________________________________________________________________
715void AliAnalysisAlien::AddRunList(const char* runList)
716{
717// Add several runs into the list of runs; they are expected to be separated by a blank character.
718 TString sList = runList;
719 TObjArray *list = sList.Tokenize(" ");
720 Int_t n = list->GetEntries();
721 for (Int_t i = 0; i < n; i++) {
722 TObjString *os = (TObjString*)list->At(i);
723 AddRunNumber(os->GetString().Atoi());
724 }
725 delete list;
726}
727
ee75cfc3 728//______________________________________________________________________________
729void AliAnalysisAlien::AddRunNumber(const char* run)
730{
731// Add a run number to the list of runs to be processed.
7aa5d539 732 TString runs = run;
733 TObjString *os;
734 TObjArray *arr = runs.Tokenize(" ");
735 TIter next(arr);
736 TString prefix;
737 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
738 while ((os=(TObjString*)next())){
739 if (fRunNumbers.Length()) fRunNumbers += " ";
740 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
741 }
742 delete arr;
ee75cfc3 743}
744
c57f56b7 745//______________________________________________________________________________
746void AliAnalysisAlien::AddDataFile(const char *lfn)
747{
748// Adds a data file to the input to be analysed. The file should be a valid LFN
749// or point to an existing file in the alien workdir.
750 if (!fInputFiles) fInputFiles = new TObjArray();
751 fInputFiles->Add(new TObjString(lfn));
752}
648174cf 753
754//______________________________________________________________________________
755void AliAnalysisAlien::AddExternalPackage(const char *package)
756{
757// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
758 if (fExternalPackages) fExternalPackages += " ";
759 fExternalPackages += package;
760}
761
c57f56b7 762//______________________________________________________________________________
763Bool_t AliAnalysisAlien::Connect()
764{
765// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
766 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 767 if (fProductionMode) return kTRUE;
c57f56b7 768 if (!gGrid) {
769 Info("Connect", "Trying to connect to AliEn ...");
770 TGrid::Connect("alien://");
771 }
772 if (!gGrid || !gGrid->IsConnected()) {
773 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
774 return kFALSE;
775 }
776 fUser = gGrid->GetUser();
777 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
778 return kTRUE;
779}
780
781//______________________________________________________________________________
782void AliAnalysisAlien::CdWork()
783{
784// Check validity of alien workspace. Create directory if possible.
785 if (!Connect()) {
786 Error("CdWork", "Alien connection required");
787 return;
788 }
789 TString homedir = gGrid->GetHomeDirectory();
790 TString workdir = homedir + fGridWorkingDir;
923e2ca5 791 if (DirectoryExists(workdir)) {
792 gGrid->Cd(workdir);
793 return;
794 }
795 // Work directory not existing - create it
796 gGrid->Cd(homedir);
b93f8109 797 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 798 gGrid->Cd(fGridWorkingDir);
d3b18c4c 799 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 800 } else {
d3b18c4c 801 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 802 workdir.Data(), homedir.Data());
803 fGridWorkingDir = "";
804 }
c57f56b7 805}
806
348be253 807//______________________________________________________________________________
808Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
809{
810// Check if file copying is possible.
d3b18c4c 811 if (fProductionMode) return kTRUE;
348be253 812 if (!Connect()) {
813 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
814 return kFALSE;
815 }
d3339be3 816 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
817 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 818 // Check if alien_CLOSE_SE is defined
819 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
820 if (!closeSE.IsNull()) {
821 Info("CheckFileCopy", "Your current close storage is pointing to: \
822 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
823 } else {
824 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
825 }
826 // Check if grid directory exists.
827 if (!DirectoryExists(alienpath)) {
828 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
829 return kFALSE;
830 }
d3339be3 831 TFile f("plugin_test_copy", "RECREATE");
348be253 832 // User may not have write permissions to current directory
833 if (f.IsZombie()) {
834 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
835 gSystem->WorkingDirectory());
836 return kFALSE;
837 }
838 f.Close();
d3339be3 839 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
840 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
841 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 842 \n# 1. Make sure you have write permissions there. If this is the case: \
843 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
844 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
845 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
846 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
847 gSystem->Unlink(f.GetName());
848 return kFALSE;
849 }
850 gSystem->Unlink(f.GetName());
d3339be3 851 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
852 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 853 return kTRUE;
854}
855
c57f56b7 856//______________________________________________________________________________
857Bool_t AliAnalysisAlien::CheckInputData()
858{
859// Check validity of input data. If necessary, create xml files.
d3b18c4c 860 if (fProductionMode) return kTRUE;
a8739e8a 861 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 862 if (!fGridDataDir.Length()) {
863 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
864 return kFALSE;
865 }
9f020952 866 if (fMergeViaJDL) {
867 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
868 return kFALSE;
869 }
d2a409b2 870 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 871 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
872 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 873 return kTRUE;
c57f56b7 874 }
875 // Process declared files
a2f5fc01 876 Bool_t isCollection = kFALSE;
877 Bool_t isXml = kFALSE;
878 Bool_t useTags = kFALSE;
c57f56b7 879 Bool_t checked = kFALSE;
d3b18c4c 880 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 881 TString file;
882 TString workdir = gGrid->GetHomeDirectory();
883 workdir += fGridWorkingDir;
884 if (fInputFiles) {
885 TObjString *objstr;
886 TIter next(fInputFiles);
887 while ((objstr=(TObjString*)next())) {
888 file = workdir;
889 file += "/";
890 file += objstr->GetString();
891 // Store full lfn path
892 if (FileExists(file)) objstr->SetString(file);
893 else {
894 file = objstr->GetName();
895 if (!FileExists(objstr->GetName())) {
896 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
897 objstr->GetName(), workdir.Data());
898 return kFALSE;
899 }
900 }
901 Bool_t iscoll, isxml, usetags;
902 CheckDataType(file, iscoll, isxml, usetags);
903 if (!checked) {
904 checked = kTRUE;
a2f5fc01 905 isCollection = iscoll;
906 isXml = isxml;
907 useTags = usetags;
908 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 909 } else {
a2f5fc01 910 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 911 Error("CheckInputData", "Some conflict was found in the types of inputs");
912 return kFALSE;
913 }
914 }
915 }
916 }
917 // Process requested run numbers
a8739e8a 918 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 919 // Check validity of alien data directory
920 if (!fGridDataDir.Length()) {
921 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
922 return kFALSE;
923 }
923e2ca5 924 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 925 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
926 return kFALSE;
927 }
a2f5fc01 928 if (isCollection) {
c57f56b7 929 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
930 return kFALSE;
931 }
932
a2f5fc01 933 if (checked && !isXml) {
c57f56b7 934 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
935 return kFALSE;
936 }
937 // Check validity of run number(s)
938 TObjArray *arr;
939 TObjString *os;
74013535 940 TString format;
319593fb 941 Int_t nruns = 0;
904f9f5f 942 TString schunk, schunk2;
c57f56b7 943 TString path;
944 if (!checked) {
945 checked = kTRUE;
a2f5fc01 946 useTags = fDataPattern.Contains("tag");
947 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 948 }
a2f5fc01 949 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 950 Error("CheckInputData", "Cannot mix input files using/not using tags");
951 return kFALSE;
952 }
953 if (fRunNumbers.Length()) {
a8739e8a 954 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 955 arr = fRunNumbers.Tokenize(" ");
956 TIter next(arr);
957 while ((os=(TObjString*)next())) {
958 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 959 if (!DirectoryExists(path)) {
960 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 961 continue;
c57f56b7 962 }
963 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
964 TString msg = "\n##### file: ";
965 msg += path;
966 msg += " type: xml_collection;";
a2f5fc01 967 if (useTags) msg += " using_tags: Yes";
c57f56b7 968 else msg += " using_tags: No";
23329835 969 Info("CheckDataType", "%s", msg.Data());
319593fb 970 if (fNrunsPerMaster<2) {
d2a409b2 971 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 972 } else {
973 nruns++;
974 if (((nruns-1)%fNrunsPerMaster) == 0) {
975 schunk = os->GetString();
976 }
977 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
978 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 979 AddDataFile(schunk);
319593fb 980 }
c57f56b7 981 }
982 delete arr;
a8739e8a 983 } else {
984 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
985 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 986 format = Form("%%s/%s ", fRunPrefix.Data());
987 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 988 if (!DirectoryExists(path)) {
a8739e8a 989 continue;
990 }
74013535 991 format = Form("%%s/%s.xml", fRunPrefix.Data());
992 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 993 TString msg = "\n##### file: ";
994 msg += path;
995 msg += " type: xml_collection;";
a2f5fc01 996 if (useTags) msg += " using_tags: Yes";
a8739e8a 997 else msg += " using_tags: No";
23329835 998 Info("CheckDataType", "%s", msg.Data());
319593fb 999 if (fNrunsPerMaster<2) {
74013535 1000 format = Form("%s.xml", fRunPrefix.Data());
1001 AddDataFile(Form(format.Data(),irun));
319593fb 1002 } else {
1003 nruns++;
1004 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1005 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1006 }
74013535 1007 format = Form("_%s.xml", fRunPrefix.Data());
1008 schunk2 = Form(format.Data(), irun);
319593fb 1009 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1010 schunk += schunk2;
d2a409b2 1011 AddDataFile(schunk);
319593fb 1012 }
a8739e8a 1013 }
904f9f5f 1014 if (!fInputFiles) {
1015 schunk += schunk2;
1016 AddDataFile(schunk);
1017 }
c57f56b7 1018 }
1019 return kTRUE;
1020}
1021
21ca8e59 1022//______________________________________________________________________________
43b807f6 1023Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1024{
1025// Copy data from the given grid directory according a pattern and make a local
1026// dataset.
1027 if (!Connect()) {
1028 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
1029 return kFALSE;
1030 }
1031 if (!DirectoryExists(griddir)) {
1032 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
1033 return kFALSE;
1034 }
1035 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1036 printf("Running command: %s\n", command.Data());
1037 TGridResult *res = gGrid->Command(command);
1038 Int_t nfound = res->GetEntries();
1039 if (!nfound) {
1040 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
1041 return kFALSE;
1042 }
1043 printf("... found %d files. Copying locally ...\n", nfound);
1044 // Copy files locally
1045 ofstream out;
1046 out.open(output, ios::out);
1047 TMap *map;
1048 TString turl, dirname, filename, temp;
1049 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1050 gSystem->MakeDirectory(outputdir);
1051 gSystem->ChangeDirectory(outputdir);
21ca8e59 1052 for (Int_t i=0; i<nfound; i++) {
1053 map = (TMap*)res->At(i);
1054 turl = map->GetValue("turl")->GetName();
1055 filename = gSystem->BaseName(turl.Data());
1056 dirname = gSystem->DirName(turl.Data());
1057 dirname = gSystem->BaseName(dirname.Data());
1058 gSystem->MakeDirectory(dirname);
43b807f6 1059
1060 TString source(turl);
1061 TString targetFileName(filename);
1062
1063 if (strlen(archivefile) > 0) {
1064 // TODO here the archive in which the file resides should be determined
1065 // however whereis returns only a guid, and guid2lfn does not work
1066 // Therefore we use the one provided as argument for now
1067 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1068 targetFileName = archivefile;
1069 }
1070 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
1071 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1072 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
21ca8e59 1073 }
1074 }
1075 gSystem->ChangeDirectory(cdir);
1076 delete res;
1077 return kTRUE;
1078}
1079
c57f56b7 1080//______________________________________________________________________________
1081Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1082{
1083// Create dataset for the grid data directory + run number.
3af861ff 1084 const Int_t gMaxEntries = 15000;
d3b18c4c 1085 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1086 if (!Connect()) {
1087 Error("CreateDataset", "Cannot create dataset with no grid connection");
1088 return kFALSE;
1089 }
1090
1091 // Cd workspace
d3b18c4c 1092 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1093 TString workdir = gGrid->GetHomeDirectory();
1094 workdir += fGridWorkingDir;
1095
1096 // Compose the 'find' command arguments
74013535 1097 TString format;
c57f56b7 1098 TString command;
1099 TString options = "-x collection ";
bb885a9e 1100 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1101 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1102 TString conditions = "";
3af861ff 1103 Int_t nstart = 0;
1104 Int_t ncount = 0;
1105 Int_t stage = 0;
c57f56b7 1106 TString file;
1107 TString path;
319593fb 1108 Int_t nruns = 0;
904f9f5f 1109 TString schunk, schunk2;
ab254fd1 1110 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1111 if (!fRunNumbers.Length() && !fRunRange[0]) {
1112 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1113 // Make a single data collection from data directory.
1114 path = fGridDataDir;
923e2ca5 1115 if (!DirectoryExists(path)) {
d2a409b2 1116 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1117 return kFALSE;
1118 }
923e2ca5 1119// CdWork();
d2a409b2 1120 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1121 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1122 while (1) {
1123 ncount = 0;
1124 stage++;
1125 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1126 command = "find ";
1127 command += Form("%s -o %d ",options.Data(), nstart);
1128 command += path;
1129 command += " ";
1130 command += pattern;
1131 command += conditions;
1132 printf("command: %s\n", command.Data());
1133 TGridResult *res = gGrid->Command(command);
1134 if (res) delete res;
1135 // Write standard output to file
1136 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1137 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1138 Bool_t nullFile = kFALSE;
1139 if (!hasGrep) {
1140 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1141 } else {
1142 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1143 if (nullFile) {
1144 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1145 gSystem->Exec("rm -f __tmp*");
1146 return kFALSE;
1147 }
1148 TString line;
1149 ifstream in;
1150 in.open("__tmp__");
1151 in >> line;
1152 in.close();
1153 gSystem->Exec("rm -f __tmp__");
1154 ncount = line.Atoi();
1155 }
1156 }
1157 if (ncount == gMaxEntries) {
1158 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1159 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1160 if (!cbase) cbase = cadd;
1161 else {
1162 cbase->Add(cadd);
1163 delete cadd;
defd7a3a 1164 }
3af861ff 1165 nstart += ncount;
1166 } else {
1167 if (cbase) {
1168 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1169 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1170 cbase->Add(cadd);
1171 delete cadd;
1172 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1173 delete cbase; cbase = 0;
1174 } else {
1175 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1176 }
1177 gSystem->Exec("rm -f __tmp*");
1178 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1179 break;
1180 }
0f389141 1181 }
1182 Bool_t fileExists = FileExists(file);
1183 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1184 // Copy xml file to alien space
0f389141 1185 if (fileExists) gGrid->Rm(file);
d2a409b2 1186 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1187 if (!FileExists(file)) {
1188 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1189 return kFALSE;
1190 }
1191 // Update list of files to be processed.
1192 }
1193 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1194 return kTRUE;
1195 }
c57f56b7 1196 // Several runs
a2f5fc01 1197 Bool_t nullResult = kTRUE;
a8739e8a 1198 if (fRunNumbers.Length()) {
1199 TObjArray *arr = fRunNumbers.Tokenize(" ");
1200 TObjString *os;
1201 TIter next(arr);
1202 while ((os=(TObjString*)next())) {
3af861ff 1203 nstart = 0;
1204 stage = 0;
dc030a24 1205 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1206 if (!DirectoryExists(path)) continue;
1207// CdWork();
a8739e8a 1208 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1209 else file = Form("%s.xml", os->GetString().Data());
319593fb 1210 // If local collection file does not exist, create it via 'find' command.
3af861ff 1211 while (1) {
1212 ncount = 0;
1213 stage++;
1214 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1215 command = "find ";
1216 command += Form("%s -o %d ",options.Data(), nstart);
1217 command += path;
1218 command += pattern;
1219 command += conditions;
1220 TGridResult *res = gGrid->Command(command);
1221 if (res) delete res;
1222 // Write standard output to file
1223 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1224 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1225 Bool_t nullFile = kFALSE;
1226 if (!hasGrep) {
1227 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1228 } else {
1229 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1230 if (nullFile) {
1231 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1232 gSystem->Exec("rm -f __tmp*");
1233 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1234 break;
3af861ff 1235 }
1236 TString line;
1237 ifstream in;
1238 in.open("__tmp__");
1239 in >> line;
1240 in.close();
1241 gSystem->Exec("rm -f __tmp__");
1242 ncount = line.Atoi();
1243 }
1244 nullResult = kFALSE;
1245 }
1246 if (ncount == gMaxEntries) {
1247 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1248 if (fNrunsPerMaster > 1) {
1249 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1250 file.Data(),gMaxEntries);
1251 return kFALSE;
1252 }
1253 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1254 if (!cbase) cbase = cadd;
1255 else {
1256 cbase->Add(cadd);
1257 delete cadd;
defd7a3a 1258 }
3af861ff 1259 nstart += ncount;
1260 } else {
c89a84bc 1261 if (cbase && fNrunsPerMaster<2) {
3af861ff 1262 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1263 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1264 cbase->Add(cadd);
1265 delete cadd;
1266 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1267 delete cbase; cbase = 0;
1268 } else {
1269 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1270 }
1271 gSystem->Exec("rm -f __tmp*");
1272 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1273 break;
d3339be3 1274 }
3af861ff 1275 }
a8739e8a 1276 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1277 // Check if there is one run per master job.
1278 if (fNrunsPerMaster<2) {
1279 if (FileExists(file)) {
0f389141 1280 if (fOverwriteMode) gGrid->Rm(file);
1281 else {
1282 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1283 continue;
1284 }
319593fb 1285 }
1286 // Copy xml file to alien space
1287 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1288 if (!FileExists(file)) {
1289 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1290 delete arr;
1291 return kFALSE;
1292 }
1293 } else {
1294 nruns++;
1295 if (((nruns-1)%fNrunsPerMaster) == 0) {
1296 schunk = os->GetString();
1297 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1298 } else {
1299 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1300 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1301 cbase->Add(cadd);
1302 delete cadd;
1303 }
1304 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1305 continue;
1306 }
1307 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1308 if (FileExists(schunk)) {
1309 if (fOverwriteMode) gGrid->Rm(file);
1310 else {
1311 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1312 continue;
1313 }
319593fb 1314 }
84fcd93f 1315 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1316 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1317 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1318 if (!FileExists(schunk)) {
1319 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1320 delete arr;
1321 return kFALSE;
1322 }
d3339be3 1323 }
a8739e8a 1324 }
1325 delete arr;
a2f5fc01 1326 if (nullResult) {
d3339be3 1327 Error("CreateDataset", "No valid dataset corresponding to the query!");
1328 return kFALSE;
1329 }
a8739e8a 1330 } else {
1331 // Process a full run range.
1332 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1333 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1334 nstart = 0;
1335 stage = 0;
74013535 1336 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1337 if (!DirectoryExists(path)) continue;
1338// CdWork();
74013535 1339 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1340 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1341 else file = Form(format.Data(), irun);
0f389141 1342 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1343 if (fOverwriteMode) gGrid->Rm(file);
1344 else {
1345 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1346 continue;
1347 }
a8739e8a 1348 }
319593fb 1349 // If local collection file does not exist, create it via 'find' command.
3af861ff 1350 while (1) {
1351 ncount = 0;
1352 stage++;
1353 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1354 command = "find ";
1355 command += Form("%s -o %d ",options.Data(), nstart);
1356 command += path;
1357 command += pattern;
1358 command += conditions;
1359 TGridResult *res = gGrid->Command(command);
1360 if (res) delete res;
1361 // Write standard output to file
1362 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1363 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1364 Bool_t nullFile = kFALSE;
1365 if (!hasGrep) {
1366 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1367 } else {
1368 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1369 if (nullFile) {
1370 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1371 gSystem->Exec("rm -f __tmp*");
1604c9ed 1372 break;
3af861ff 1373 }
1374 TString line;
1375 ifstream in;
1376 in.open("__tmp__");
1377 in >> line;
1378 in.close();
1379 gSystem->Exec("rm -f __tmp__");
1380 ncount = line.Atoi();
1381 }
1382 nullResult = kFALSE;
1383 }
1384 if (ncount == gMaxEntries) {
1385 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1386 if (fNrunsPerMaster > 1) {
1387 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1388 file.Data(),gMaxEntries);
1389 return kFALSE;
1390 }
1391 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1392 if (!cbase) cbase = cadd;
1393 else {
1394 cbase->Add(cadd);
1395 delete cadd;
defd7a3a 1396 }
3af861ff 1397 nstart += ncount;
1398 } else {
c89a84bc 1399 if (cbase && fNrunsPerMaster<2) {
3af861ff 1400 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1401 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1402 cbase->Add(cadd);
1403 delete cadd;
1404 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1405 delete cbase; cbase = 0;
1406 } else {
1407 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1408 }
1409 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1410 break;
d3339be3 1411 }
319593fb 1412 }
a8739e8a 1413 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1414 // Check if there is one run per master job.
1415 if (fNrunsPerMaster<2) {
1416 if (FileExists(file)) {
0f389141 1417 if (fOverwriteMode) gGrid->Rm(file);
1418 else {
1419 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1420 continue;
1421 }
319593fb 1422 }
1423 // Copy xml file to alien space
1424 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1425 if (!FileExists(file)) {
1426 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1427 return kFALSE;
1428 }
1429 } else {
1430 nruns++;
95e5b448 1431 // Check if the collection for the chunk exist locally.
1432 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1433 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1434 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1435 else continue;
1436 }
84fcd93f 1437 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1438 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1439 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1440 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1441 } else {
1442 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1443 cbase->Add(cadd);
1444 delete cadd;
1445 }
74013535 1446 format = Form("%%s_%s.xml", fRunPrefix.Data());
1447 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1448 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1449 continue;
1450 }
904f9f5f 1451 schunk = schunk2;
319593fb 1452 if (FileExists(schunk)) {
0f389141 1453 if (fOverwriteMode) gGrid->Rm(schunk);
1454 else {
1455 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1456 continue;
1457 }
319593fb 1458 }
84fcd93f 1459 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1460 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1461 if (FileExists(schunk)) {
0f389141 1462 if (fOverwriteMode) gGrid->Rm(schunk);
1463 else {
1464 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1465 continue;
1466 }
95e5b448 1467 }
319593fb 1468 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1469 if (!FileExists(schunk)) {
1470 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1471 return kFALSE;
1472 }
1473 }
c57f56b7 1474 }
a2f5fc01 1475 if (nullResult) {
d3339be3 1476 Error("CreateDataset", "No valid dataset corresponding to the query!");
1477 return kFALSE;
1478 }
a8739e8a 1479 }
c57f56b7 1480 return kTRUE;
1481}
1482
1483//______________________________________________________________________________
1484Bool_t AliAnalysisAlien::CreateJDL()
1485{
1486// Generate a JDL file according to current settings. The name of the file is
1487// specified by fJDLName.
1488 Bool_t error = kFALSE;
1489 TObjArray *arr = 0;
1490 Bool_t copy = kTRUE;
d3b18c4c 1491 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1492 Bool_t generate = kTRUE;
1493 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1494 if (!Connect()) {
1495 Error("CreateJDL", "Alien connection required");
1496 return kFALSE;
1497 }
1498 // Check validity of alien workspace
d3b18c4c 1499 TString workdir;
1500 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1501 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1502 workdir += fGridWorkingDir;
1503 if (generate) {
1504 TObjString *os;
1505 if (!fInputFiles) {
1506 Error("CreateJDL()", "Define some input files for your analysis.");
1507 error = kTRUE;
1508 }
1509 // Compose list of input files
1510 // Check if output files were defined
1511 if (!fOutputFiles.Length()) {
1512 Error("CreateJDL", "You must define at least one output file");
1513 error = kTRUE;
1514 }
1515 // Check if an output directory was defined and valid
1516 if (!fGridOutputDir.Length()) {
1517 Error("CreateJDL", "You must define AliEn output directory");
1518 error = kTRUE;
1519 } else {
d3b18c4c 1520 if (!fProductionMode) {
1521 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1522 if (!DirectoryExists(fGridOutputDir)) {
1523 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1524 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1525 } else {
1526 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1527 // error = kTRUE;
1528 }
67a3b254 1529 } else {
1530 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1531 }
1532 gGrid->Cd(workdir);
1533 }
c57f56b7 1534 }
1535 // Exit if any error up to now
1536 if (error) return kFALSE;
1537 // Set JDL fields
0f389141 1538 if (!fUser.IsNull()) {
1539 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1540 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1541 }
1542 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1543 TString mergeExec = fExecutable;
1544 mergeExec.ReplaceAll(".sh", "_merge.sh");
1545 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1546 mergeExec.ReplaceAll(".sh", ".C");
1547 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1548 if (!fArguments.IsNull())
1549 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1550 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1551 else {
1552 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1553 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1554 }
b3e07543 1555
1f0d1ca2 1556 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1557 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1558 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1559 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1560
0f389141 1561 if (fMaxInitFailed > 0) {
c57f56b7 1562 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1563 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1564 }
1565 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1566 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1567 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1568 }
1569 if (!IsOneStageMerging()) {
f47d5cb4 1570 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1571 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1572 }
1573 if (fSplitMode.Length()) {
c57f56b7 1574 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1575 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1576 }
1577 fMergingJDL->SetValue("Split", "\"se\"");
1578 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1579 if (!fAliROOTVersion.IsNull()) {
1580 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1581 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1582 }
1583 if (!fROOTVersion.IsNull()) {
c57f56b7 1584 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1585 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1586 }
1587 if (!fAPIVersion.IsNull()) {
c57f56b7 1588 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1589 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1590 }
648174cf 1591 if (!fExternalPackages.IsNull()) {
1592 arr = fExternalPackages.Tokenize(" ");
1593 TIter next(arr);
1594 while ((os=(TObjString*)next())) {
1595 TString pkgname = os->GetString();
1596 Int_t index = pkgname.Index("::");
1597 TString pkgversion = pkgname(index+2, pkgname.Length());
1598 pkgname.Remove(index);
1599 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1600 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1601 }
1602 delete arr;
1603 }
0f389141 1604 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1605 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1606 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1607 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1608 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1609 TString analysisFile = fExecutable;
1610 analysisFile.ReplaceAll(".sh", ".root");
1611 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1612 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1613 if (fAdditionalLibs.Length()) {
1614 arr = fAdditionalLibs.Tokenize(" ");
1615 TIter next(arr);
1616 while ((os=(TObjString*)next())) {
1617 if (os->GetString().Contains(".so")) continue;
1618 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1619 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1620 }
1621 delete arr;
1622 }
4e5c5506 1623 if (fPackages) {
1624 TIter next(fPackages);
1625 TObject *obj;
0f389141 1626 while ((obj=next())) {
4e5c5506 1627 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1628 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1629 }
4e5c5506 1630 }
c57f56b7 1631 if (fOutputArchive.Length()) {
1632 arr = fOutputArchive.Tokenize(" ");
1633 TIter next(arr);
0f389141 1634 Bool_t first = kTRUE;
1635 const char *comment = "Files to be archived";
1636 const char *comment1 = comment;
1637 while ((os=(TObjString*)next())) {
1638 if (!first) comment = NULL;
1639 if (!os->GetString().Contains("@") && fCloseSE.Length())
1640 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1641 else
1642 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1643 first = kFALSE;
1644 }
c57f56b7 1645 delete arr;
f790bc1b 1646 // Output archive for the merging jdl
1647 TString outputArchive;
1648 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1649 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1650 // Add normal output files, extra files + terminate files
1651 TString files = GetListOfFiles("outextter");
37d21c01 1652 // Do not register files in fRegisterExcludes
1653 if (!fRegisterExcludes.IsNull()) {
1654 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1655 TIter next1(arr);
1656 while ((os=(TObjString*)next1())) {
1657 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1658 files.ReplaceAll(os->GetString(),"");
1659 }
66d71516 1660 delete arr;
0f389141 1661 }
f790bc1b 1662 files.ReplaceAll(".root", "*.root");
b83f84f0 1663 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1664 } else {
95f84779 1665 TString files = fOutputArchive;
1666 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1667 outputArchive = files;
f790bc1b 1668 }
0f389141 1669 arr = outputArchive.Tokenize(" ");
705adb3e 1670 TIter next2(arr);
0f389141 1671 comment = comment1;
1672 first = kTRUE;
705adb3e 1673 while ((os=(TObjString*)next2())) {
0f389141 1674 if (!first) comment = NULL;
0cdf65a8 1675 TString currentfile = os->GetString();
0cdf65a8 1676 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1677 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1678 else
0cdf65a8 1679 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1680 first = kFALSE;
1681 }
1682 delete arr;
c57f56b7 1683 }
149d288c 1684 arr = fOutputFiles.Tokenize(",");
c57f56b7 1685 TIter next(arr);
0f389141 1686 Bool_t first = kTRUE;
f790bc1b 1687 const char *comment = "Files to be saved";
43da816a 1688 while ((os=(TObjString*)next())) {
e1eaf596 1689 // Ignore ouputs in jdl that are also in outputarchive
1690 TString sout = os->GetString();
f790bc1b 1691 sout.ReplaceAll("*", "");
1692 sout.ReplaceAll(".root", "");
e1eaf596 1693 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1694 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1695 // Ignore fRegisterExcludes
1696 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1697 if (!first) comment = NULL;
43da816a 1698 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1699 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1700 else
0f389141 1701 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1702 first = kFALSE;
f790bc1b 1703 if (fMergeExcludes.Contains(sout)) continue;
1704 if (!os->GetString().Contains("@") && fCloseSE.Length())
1705 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1706 else
1707 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1708 }
c57f56b7 1709 delete arr;
0f389141 1710 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1711 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1712 TString validationScript = fValidationScript;
0f389141 1713 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1714 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1715 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1716 if (fMasterResubmitThreshold) {
1717 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1718 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1719 }
d2a409b2 1720 // Write a jdl with 2 input parameters: collection name and output dir name.
1721 WriteJDL(copy);
c57f56b7 1722 }
1723 // Copy jdl to grid workspace
a8739e8a 1724 if (copy) {
b5e4aaa7 1725 // Check if an output directory was defined and valid
1726 if (!fGridOutputDir.Length()) {
1727 Error("CreateJDL", "You must define AliEn output directory");
1728 return kFALSE;
1729 } else {
1730 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1731 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1732 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1733 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1734 } else {
1735 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1736 return kFALSE;
1737 }
1738 }
1739 gGrid->Cd(workdir);
1740 }
648174cf 1741 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1742 TString mergeJDLName = fExecutable;
1743 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1744 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1745 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1746 if (fProductionMode) {
648174cf 1747 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1748 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1749 }
648174cf 1750 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1751 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1752 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1753 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1754 Fatal("","Terminating");
1755// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1756 if (fMergeViaJDL) {
1757 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1758// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1759 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1760 Fatal("","Terminating");
0f389141 1761 }
648174cf 1762 }
c57f56b7 1763 if (fAdditionalLibs.Length()) {
e7c71df0 1764 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1765 TObjString *os;
1766 TIter next(arr);
1767 while ((os=(TObjString*)next())) {
c57f56b7 1768 if (os->GetString().Contains(".so")) continue;
4e5c5506 1769 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1770 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1771// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1772 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1773 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1774 Fatal("","Terminating");
c57f56b7 1775 }
1776 delete arr;
1777 }
4e5c5506 1778 if (fPackages) {
1779 TIter next(fPackages);
1780 TObject *obj;
1781 while ((obj=next())) {
fdbbc7be 1782 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1783 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1784// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1785 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1786 Form("%s/%s", workdir.Data(), obj->GetName())))
1787 Fatal("","Terminating");
4e5c5506 1788 }
1789 }
c57f56b7 1790 }
1791 return kTRUE;
1792}
1793
a8739e8a 1794//______________________________________________________________________________
d2a409b2 1795Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1796{
1797// Writes one or more JDL's corresponding to findex. If findex is negative,
1798// all run numbers are considered in one go (jdl). For non-negative indices
1799// they correspond to the indices in the array fInputFiles.
1800 if (!fInputFiles) return kFALSE;
d3b18c4c 1801 TObject *os;
1802 TString workdir;
1803 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1804 workdir += fGridWorkingDir;
f47d5cb4 1805 TString stageName = "$2";
1806 if (fProductionMode) stageName = "$4";
1807 if (!fMergeDirName.IsNull()) {
1808 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1809 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1810 } else {
1811 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1812 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1813 }
d3b18c4c 1814 if (fProductionMode) {
a8739e8a 1815 TIter next(fInputFiles);
b3e07543 1816 while ((os=next())) {
d3b18c4c 1817 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1818 }
56754bf6 1819 if (!fOutputToRunNo)
1820 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1821 else
1822 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1823 } else {
1824 if (!fRunNumbers.Length() && !fRunRange[0]) {
1825 // One jdl with no parameters in case input data is specified by name.
1826 TIter next(fInputFiles);
1827 while ((os=next()))
1828 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1829 if (!fOutputSingle.IsNull())
1830 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1831 else {
1832 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1833 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1834 }
1835 } else {
1836 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1837 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1838 if (!fOutputSingle.IsNull()) {
1839 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1840 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1841 } else {
1842 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1843 }
1844 }
a8739e8a 1845 }
1846
a8739e8a 1847 // Generate the JDL as a string
1848 TString sjdl = fGridJDL->Generate();
0f389141 1849 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1850 // Final merge jdl
1851 if (!fMergeDirName.IsNull()) {
1852 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1853 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1854 } else {
1855 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1856 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1857 }
b3e07543 1858 TString sjdl2 = fMergingJDL->Generate();
1859 Int_t index, index1;
a8739e8a 1860 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1861 sjdl.ReplaceAll("(member", "\n (member");
1862 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1863 sjdl.ReplaceAll("{", "{\n ");
1864 sjdl.ReplaceAll("};", "\n};");
1865 sjdl.ReplaceAll("{\n \n", "{\n");
1866 sjdl.ReplaceAll("\n\n", "\n");
1867 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1868 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1869 sjdl1.ReplaceAll("(member", "\n (member");
1870 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1871 sjdl1.ReplaceAll("{", "{\n ");
1872 sjdl1.ReplaceAll("};", "\n};");
1873 sjdl1.ReplaceAll("{\n \n", "{\n");
1874 sjdl1.ReplaceAll("\n\n", "\n");
1875 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1876 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1877 sjdl2.ReplaceAll("(member", "\n (member");
1878 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1879 sjdl2.ReplaceAll("{", "{\n ");
1880 sjdl2.ReplaceAll("};", "\n};");
1881 sjdl2.ReplaceAll("{\n \n", "{\n");
1882 sjdl2.ReplaceAll("\n\n", "\n");
1883 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1884 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1885 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1886 index = sjdl.Index("JDLVariables");
1887 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1888 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1889 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1890 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1891 index = fJobTag.Index(":");
1892 if (index < 0) index = fJobTag.Length();
1893 TString jobTag = fJobTag;
f47d5cb4 1894 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1895 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1896 if (fProductionMode) {
1897 sjdl1.Prepend("# Generated merging jdl (production mode) \
1898 \n# $1 = full alien path to output directory to be merged \
1899 \n# $2 = train number \
1900 \n# $3 = production (like LHC10b) \
1901 \n# $4 = merging stage \
1902 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1903 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1904 sjdl2.Prepend("# Generated merging jdl \
1905 \n# $1 = full alien path to output directory to be merged \
1906 \n# $2 = train number \
1907 \n# $3 = production (like LHC10b) \
1908 \n# $4 = merging stage \
1909 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1910 } else {
1911 sjdl1.Prepend("# Generated merging jdl \
1912 \n# $1 = full alien path to output directory to be merged \
1913 \n# $2 = merging stage \
03335324 1914 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1915 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1916 sjdl2.Prepend("# Generated merging jdl \
1917 \n# $1 = full alien path to output directory to be merged \
1918 \n# $2 = merging stage \
03335324 1919 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1920 }
0f389141 1921 index = sjdl1.Index("JDLVariables");
1922 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1923 index = sjdl2.Index("JDLVariables");
1924 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1925 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1926 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1927 index = sjdl2.Index("Split =");
1928 if (index>=0) {
1929 index1 = sjdl2.Index("\n", index);
1930 sjdl2.Remove(index, index1-index+1);
1931 }
1932 index = sjdl2.Index("SplitMaxInputFileNumber");
1933 if (index>=0) {
1934 index1 = sjdl2.Index("\n", index);
1935 sjdl2.Remove(index, index1-index+1);
1936 }
1937 index = sjdl2.Index("InputDataCollection");
1938 if (index>=0) {
1939 index1 = sjdl2.Index(";", index);
1940 sjdl2.Remove(index, index1-index+1);
1941 }
1942 index = sjdl2.Index("InputDataListFormat");
1943 if (index>=0) {
1944 index1 = sjdl2.Index("\n", index);
1945 sjdl2.Remove(index, index1-index+1);
1946 }
1947 index = sjdl2.Index("InputDataList");
1948 if (index>=0) {
1949 index1 = sjdl2.Index("\n", index);
1950 sjdl2.Remove(index, index1-index+1);
1951 }
f47d5cb4 1952 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1953 // Write jdl to file
a8739e8a 1954 ofstream out;
d2a409b2 1955 out.open(fJDLName.Data(), ios::out);
a8739e8a 1956 if (out.bad()) {
d3b18c4c 1957 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1958 return kFALSE;
1959 }
1960 out << sjdl << endl;
b3e07543 1961 out.close();
0f389141 1962 TString mergeJDLName = fExecutable;
1963 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1964 if (fMergeViaJDL) {
1965 ofstream out1;
1966 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1967 if (out1.bad()) {
d3b18c4c 1968 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1969 return kFALSE;
1970 }
1971 out1 << sjdl1 << endl;
b3e07543 1972 out1.close();
1973 ofstream out2;
1974 TString finalJDL = mergeJDLName;
1975 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1976 out2.open(finalJDL.Data(), ios::out);
1977 if (out2.bad()) {
1978 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1979 return kFALSE;
1980 }
1981 out2 << sjdl2 << endl;
1982 out2.close();
0f389141 1983 }
a8739e8a 1984
1985 // Copy jdl to grid workspace
1986 if (!copy) {
d3b18c4c 1987 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1988 } else {
d2a409b2 1989 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1990 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1991 TString finalJDL = mergeJDLName;
1992 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1993 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1994 if (fProductionMode) {
b5fe9cba 1995 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1996 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1997 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1998 }
d2a409b2 1999 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2000 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2001 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2002 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2003// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2004 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2005 Fatal("","Terminating");
0f389141 2006 if (fMergeViaJDL) {
b3e07543 2007 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2008// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2009// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2010 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2011 Fatal("","Terminating");
2012 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2013 Fatal("","Terminating");
0f389141 2014 }
a8739e8a 2015 }
2016 return kTRUE;
2017}
2018
c57f56b7 2019//______________________________________________________________________________
5513444a 2020Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2021{
2022// Returns true if file exists.
5513444a 2023 if (!gGrid) return kFALSE;
2ea55496 2024 TString slfn = lfn;
2025 slfn.ReplaceAll("alien://","");
2026 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2027 if (!res) return kFALSE;
2028 TMap *map = dynamic_cast<TMap*>(res->At(0));
2029 if (!map) {
2030 delete res;
2031 return kFALSE;
2032 }
2033 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2034 if (!objs || !objs->GetString().Length()) {
2035 delete res;
2036 return kFALSE;
2037 }
2038 delete res;
2039 return kTRUE;
2040}
2041
923e2ca5 2042//______________________________________________________________________________
2043Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2044{
2045// Returns true if directory exists. Can be also a path.
2046 if (!gGrid) return kFALSE;
2047 // Check if dirname is a path
2048 TString dirstripped = dirname;
2049 dirstripped = dirstripped.Strip();
2050 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2051 TString dir = gSystem->BaseName(dirstripped);
2052 dir += "/";
2053 TString path = gSystem->DirName(dirstripped);
2054 TGridResult *res = gGrid->Ls(path, "-F");
2055 if (!res) return kFALSE;
2056 TIter next(res);
2057 TMap *map;
2058 TObject *obj;
2059 while ((map=dynamic_cast<TMap*>(next()))) {
2060 obj = map->GetValue("name");
2061 if (!obj) break;
2062 if (dir == obj->GetName()) {
2063 delete res;
2064 return kTRUE;
2065 }
2066 }
2067 delete res;
2068 return kFALSE;
2069}
2070
c57f56b7 2071//______________________________________________________________________________
a2f5fc01 2072void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2073{
2074// Check input data type.
a2f5fc01 2075 isCollection = kFALSE;
2076 isXml = kFALSE;
2077 useTags = kFALSE;
c57f56b7 2078 if (!gGrid) {
2079 Error("CheckDataType", "No connection to grid");
2080 return;
2081 }
a2f5fc01 2082 isCollection = IsCollection(lfn);
c57f56b7 2083 TString msg = "\n##### file: ";
2084 msg += lfn;
a2f5fc01 2085 if (isCollection) {
c57f56b7 2086 msg += " type: raw_collection;";
2087 // special treatment for collections
a2f5fc01 2088 isXml = kFALSE;
c57f56b7 2089 // check for tag files in the collection
2090 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2091 if (!res) {
2092 msg += " using_tags: No (unknown)";
23329835 2093 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2094 return;
2095 }
2096 const char* typeStr = res->GetKey(0, "origLFN");
2097 if (!typeStr || !strlen(typeStr)) {
2098 msg += " using_tags: No (unknown)";
23329835 2099 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2100 return;
2101 }
2102 TString file = typeStr;
a2f5fc01 2103 useTags = file.Contains(".tag");
2104 if (useTags) msg += " using_tags: Yes";
c57f56b7 2105 else msg += " using_tags: No";
23329835 2106 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2107 return;
2108 }
2109 TString slfn(lfn);
2110 slfn.ToLower();
a2f5fc01 2111 isXml = slfn.Contains(".xml");
2112 if (isXml) {
c57f56b7 2113 // Open xml collection and check if there are tag files inside
2114 msg += " type: xml_collection;";
2115 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2116 if (!coll) {
2117 msg += " using_tags: No (unknown)";
23329835 2118 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2119 return;
2120 }
2121 TMap *map = coll->Next();
2122 if (!map) {
2123 msg += " using_tags: No (unknown)";
23329835 2124 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2125 return;
2126 }
2127 map = (TMap*)map->GetValue("");
2128 TString file;
2129 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2130 useTags = file.Contains(".tag");
c57f56b7 2131 delete coll;
a2f5fc01 2132 if (useTags) msg += " using_tags: Yes";
c57f56b7 2133 else msg += " using_tags: No";
23329835 2134 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2135 return;
2136 }
a2f5fc01 2137 useTags = slfn.Contains(".tag");
c57f56b7 2138 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2139 else msg += " type: unknown file;";
a2f5fc01 2140 if (useTags) msg += " using_tags: Yes";
c57f56b7 2141 else msg += " using_tags: No";
23329835 2142 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2143}
2144
4e5c5506 2145//______________________________________________________________________________
2146void AliAnalysisAlien::EnablePackage(const char *package)
2147{
2148// Enables a par file supposed to exist in the current directory.
2149 TString pkg(package);
2150 pkg.ReplaceAll(".par", "");
2151 pkg += ".par";
2152 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2153 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2154 return;
2155 }
2156 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2157 Info("EnablePackage", "AliEn plugin will use .par packages");
2158 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2159 if (!fPackages) {
2160 fPackages = new TObjArray();
2161 fPackages->SetOwner();
2162 }
2163 fPackages->Add(new TObjString(pkg));
2164}
2165
3bdcb562 2166//______________________________________________________________________________
2167TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2168{
2169// Make a tree from files having the location specified in fFileForTestMode.
2170// Inspired from JF's CreateESDChain.
2171 if (fFileForTestMode.IsNull()) {
2172 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2173 return NULL;
2174 }
2175 if (gSystem->AccessPathName(fFileForTestMode)) {
2176 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2177 return NULL;
2178 }
2179 // Open the file
2180 ifstream in;
2181 in.open(fFileForTestMode);
2182 Int_t count = 0;
2183 // Read the input list of files and add them to the chain
2184 TString line;
2185 TChain *chain = new TChain(treeName);
2186 while (in.good())
2187 {
2188 in >> line;
2189 if (line.IsNull()) continue;
2190 if (count++ == fNtestFiles) break;
2191 TString esdFile(line);
2192 TFile *file = TFile::Open(esdFile);
2193 if (file) {
2194 if (!file->IsZombie()) chain->Add(esdFile);
2195 file->Close();
2196 } else {
2197 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2198 }
2199 }
2200 in.close();
2201 if (!chain->GetListOfFiles()->GetEntries()) {
2202 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2203 delete chain;
2204 return NULL;
2205 }
2206// chain->ls();
2207 return chain;
2208}
2209
d2a409b2 2210//______________________________________________________________________________
2211const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2212{
2213// Get job status for all jobs with jobid>jobidstart.
2214 static char mstatus[20];
2215 mstatus[0] = '\0';
2216 nrunning = 0;
2217 nwaiting = 0;
2218 nerror = 0;
2219 ndone = 0;
2220 TGridJobStatusList *list = gGrid->Ps("");
2221 if (!list) return mstatus;
2222 Int_t nentries = list->GetSize();
2223 TGridJobStatus *status;
2224 Int_t pid;
2225 for (Int_t ijob=0; ijob<nentries; ijob++) {
2226 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2227 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2228 if (pid<jobidstart) continue;
2229 if (pid == lastid) {
8ddad121 2230 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2231 }
2232 switch (status->GetStatus()) {
2233 case TGridJobStatus::kWAITING:
2234 nwaiting++; break;
2235 case TGridJobStatus::kRUNNING:
2236 nrunning++; break;
2237 case TGridJobStatus::kABORTED:
2238 case TGridJobStatus::kFAIL:
2239 case TGridJobStatus::kUNKNOWN:
2240 nerror++; break;
2241 case TGridJobStatus::kDONE:
2242 ndone++;
2243 }
2244 }
2245 list->Delete();
2246 delete list;
2247 return mstatus;
2248}
2249
c57f56b7 2250//______________________________________________________________________________
2251Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2252{
2253// Returns true if file is a collection. Functionality duplicated from
2254// TAlien::Type() because we don't want to directly depend on TAlien.
2255 if (!gGrid) {
2256 Error("IsCollection", "No connection to grid");
2257 return kFALSE;
2258 }
2259 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2260 if (!res) return kFALSE;
2261 const char* typeStr = res->GetKey(0, "type");
2262 if (!typeStr || !strlen(typeStr)) return kFALSE;
2263 if (!strcmp(typeStr, "collection")) return kTRUE;
2264 delete res;
2265 return kFALSE;
2266}
2267
fe2d7fc2 2268//______________________________________________________________________________
2269Bool_t AliAnalysisAlien::IsSingleOutput() const
2270{
2271// Check if single-ouput option is on.
2272 return (!fOutputSingle.IsNull());
2273}
2274
16a4353c 2275//______________________________________________________________________________
2276void AliAnalysisAlien::Print(Option_t *) const
2277{
2278// Print current plugin settings.
84fcd93f 2279 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2280 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2281 if (mgr && mgr->IsProofMode()) {
2282 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2283 if (TestBit(AliAnalysisGrid::kTest))
2284 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2285 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2286 if (!fProofDataSet.IsNull())
2287 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2288 if (fProofReset==1)
2289 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2290 if (fProofReset>1)
2291 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2292 if (!fRootVersionForProof.IsNull())
2293 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2294 else
2295 printf("= ROOT version requested________________________ default\n");
2296 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2297 if (!fAliRootMode.IsNull())
2298 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2299 if (fNproofWorkers)
2300 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2301 if (fNproofWorkersPerSlave)
2302 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2303 if (TestSpecialBit(kClearPackages))
2304 printf("= ClearPackages requested...\n");
2305 if (fIncludePath.Data())
2306 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2307 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2308 if (fPackages && fPackages->GetEntries()) {
2309 TIter next(fPackages);
2310 TObject *obj;
2311 TString list;
2312 while ((obj=next())) list += obj->GetName();
2313 printf("= Par files to be used: ________________________ %s\n", list.Data());
2314 }
2315 if (TestSpecialBit(kProofConnectGrid))
2316 printf("= Requested PROOF connection to grid\n");
2317 return;
2318 }
e1c22e21 2319 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2320 if (fOverwriteMode) {
2321 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2322 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2323 }
348be253 2324 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2325 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2326 printf("= Production mode:______________________________ %d\n", fProductionMode);
2327 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2328 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2329 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2330 if (fUser.Length())
84fcd93f 2331 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2332 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2333 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2334 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2335 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2336 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2337 if (fRunNumbers.Length())
84fcd93f 2338 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2339 if (fRunRange[0])
74013535 2340 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2341 if (!fRunRange[0] && !fRunNumbers.Length()) {
2342 TIter next(fInputFiles);
2343 TObject *obj;
2344 TString list;
2345 while ((obj=next())) list += obj->GetName();
84fcd93f 2346 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2347 }
2348 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2349 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2350 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2351 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2352 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2353 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2354 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2355 printf("=====================================================================\n");
2356 printf("= Job price: ___________________________________ %d\n", fPrice);
2357 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2358 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2359 if (fMaxInitFailed>0)
84fcd93f 2360 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2361 if (fMasterResubmitThreshold>0)
84fcd93f 2362 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2363 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2364 if (fNrunsPerMaster>0)
84fcd93f 2365 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2366 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2367 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2368 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2369 if (fArguments.Length())
84fcd93f 2370 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2371 if (fExecutableArgs.Length())
2372 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2373 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2374 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2375 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2376 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2377 if (fDatasetName)
84fcd93f 2378 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2379 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2380 if (fIncludePath.Data())
84fcd93f 2381 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2382 if (fCloseSE.Length())
84fcd93f 2383 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2384 if (fFriendChainName.Length())
84fcd93f 2385 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2386 if (fPackages && fPackages->GetEntries()) {
16a4353c 2387 TIter next(fPackages);
2388 TObject *obj;
2389 TString list;
2390 while ((obj=next())) list += obj->GetName();
84fcd93f 2391 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2392 }
2393}
2394
c57f56b7 2395//______________________________________________________________________________
2396void AliAnalysisAlien::SetDefaults()
2397{
2398// Set default values for everything. What cannot be filled will be left empty.
2399 if (fGridJDL) delete fGridJDL;
2400 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2401 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2402 fPrice = 1;
2403 fTTL = 30000;
2404 fSplitMaxInputFileNumber = 100;
2405 fMaxInitFailed = 0;
2406 fMasterResubmitThreshold = 0;
bb885a9e 2407 fNtestFiles = 10;
149d288c 2408 fNreplicas = 2;
a8739e8a 2409 fRunRange[0] = 0;
2410 fRunRange[1] = 0;
8e3bd06a 2411 fRunPrefix = "%d";
319593fb 2412 fNrunsPerMaster = 1;
16a4353c 2413 fMaxMergeFiles = 100;
c57f56b7 2414 fRunNumbers = "";
2415 fExecutable = "analysis.sh";
0a1c1f7f 2416 fExecutableCommand = "root -b -q";
c57f56b7 2417 fArguments = "";
631c0b05 2418 fExecutableArgs = "";
c57f56b7 2419 fAnalysisMacro = "myAnalysis.C";
2420 fAnalysisSource = "";
2421 fAdditionalLibs = "";
2422 fSplitMode = "se";
2423 fAPIVersion = "";
2424 fROOTVersion = "";
2425 fAliROOTVersion = "";
2426 fUser = ""; // Your alien user name
2427 fGridWorkingDir = "";
2428 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2429 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2430 fFriendChainName = "";
c57f56b7 2431 fGridOutputDir = "output";
b83f84f0 2432 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2433 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2434 fInputFormat = "xml-single";
2435 fJDLName = "analysis.jdl";
c6cb3634 2436 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2437 fMergeExcludes = "";
0f389141 2438 fMergeViaJDL = 0;
348be253 2439 SetUseCopy(kTRUE);
2440 SetCheckCopy(kTRUE);
149d288c 2441 SetDefaultOutputs(kTRUE);
e1c22e21 2442 fOverwriteMode = 1;
c57f56b7 2443}
2444
0f389141 2445//______________________________________________________________________________
b3e07543 2446Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2447{
b3e07543 2448// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2449 // First check if the result is already in the output directory.
2450 if (FileExists(Form("%s/%s",aliendir,filename))) {
2451 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2452 return kFALSE;
2453 }
b3e07543 2454 // Now check the last stage done.
2455 Int_t stage = 0;
2456 while (1) {
2457 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2458 stage++;
2459 }
2460 // Next stage of merging
2461 stage++;
2462 TString pattern = "*root_archive.zip";
2463 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2464 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2465 if (res) delete res;
2466 // Write standard output to file
2467 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2468 // Count the number of files inside
2469 ifstream ifile;
2470 ifile.open(Form("Stage_%d.xml",stage));
2471 if (!ifile.good()) {
2472 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2473 return kFALSE;
b3e07543 2474 }
2475 TString line;
2476 Int_t nfiles = 0;
2477 while (!ifile.eof()) {
2478 ifile >> line;
2479 if (line.Contains("/event")) nfiles++;
7c2cd90a 2480 }
b3e07543 2481 ifile.close();
2482 if (!nfiles) {
2483 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2484 return kFALSE;
2485 } else {
2486 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2487 }
b3e07543 2488 // Copy the file in the output directory
2489 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2490// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2491 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2492 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2493 // Check if this is the last stage to be done.
2494 Bool_t laststage = (nfiles<nperchunk);
2495 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2496 if (laststage) {
2497 printf("### Submiting final merging stage %d\n", stage);
2498 TString finalJDL = jdl;
2499 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2500 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2501 Int_t jobId = SubmitSingleJob(query);
2502 if (!jobId) return kFALSE;
2503 } else {
2504 printf("### Submiting merging stage %d\n", stage);
03335324 2505 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2506 Int_t jobId = SubmitSingleJob(query);
2507 if (!jobId) return kFALSE;
7c2cd90a 2508 }
b3e07543 2509 return kTRUE;
2510}
7c2cd90a 2511
105631b5 2512//______________________________________________________________________________
2513AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2514{
2515// Loat the analysis manager from a file.
2516 TFile *file = TFile::Open(fname);
2517 if (!file) {
2518 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2519 return 0;
2520 }
2521 TIter nextkey(file->GetListOfKeys());
2522 AliAnalysisManager *mgr = 0;
2523 TKey *key;
2524 while ((key=(TKey*)nextkey())) {
2525 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2526 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2527 }
2528 if (!mgr)
2529 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2530 return mgr;
2531}
2532
7c2cd90a 2533//______________________________________________________________________________
2534Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2535{
2536// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2537 if (!gGrid) return 0;
2538 printf("=> %s ------> ",query);
2539 TGridResult *res = gGrid->Command(query);
2540 if (!res) return 0;
2541 TString jobId = res->GetKey(0,"jobId");
2542 delete res;
2543 if (jobId.IsNull()) {
2544 printf("submission failed. Reason:\n");
2545 gGrid->Stdout();
2546 gGrid->Stderr();
2547 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2548 return 0;
2549 }
2550 printf(" Job id: %s\n", jobId.Data());
2551 return atoi(jobId);
2552}
2553
2554//______________________________________________________________________________
b3e07543 2555Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2556{
b3e07543 2557// Merge given output files from basedir. Basedir can be an alien output directory
2558// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2559// files in a group (ignored for xml input). Merging can be done in stages:
2560// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2561// stage=1 : works with an xml of all root_archive.zip in the output directory
2562// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2563 TString outputFile = output;
0f389141 2564 TString command;
a2f5fc01 2565 TString outputChunk;
2566 TString previousChunk = "";
b3e07543 2567 TObjArray *listoffiles = new TObjArray();
2568// listoffiles->SetOwner();
a2f5fc01 2569 Int_t countChunk = 0;
2570 Int_t countZero = nmaxmerge;
0f389141 2571 Bool_t merged = kTRUE;
a2f5fc01 2572 Int_t index = outputFile.Index("@");
2573 if (index > 0) outputFile.Remove(index);
7c2cd90a 2574 TString inputFile = outputFile;
b3e07543 2575 TString sbasedir = basedir;
2576 if (sbasedir.Contains(".xml")) {
2577 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2578 nmaxmerge = 9999999;
2579 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2580 if (!coll) {
2581 ::Error("MergeOutput", "Input XML collection empty.");
2582 return kFALSE;
2583 }
2584 // Iterate grid collection
2585 while (coll->Next()) {
2586 TString fname = gSystem->DirName(coll->GetTURL());
2587 fname += "/";
2588 fname += inputFile;
2589 listoffiles->Add(new TNamed(fname.Data(),""));
2590 }
2591 } else {
2592 command = Form("find %s/ *%s", basedir, inputFile.Data());
2593 printf("command: %s\n", command.Data());
2594 TGridResult *res = gGrid->Command(command);
2595 if (!res) {
2596 ::Error("MergeOutput","No result for the find command\n");
2597 delete listoffiles;
2598 return kFALSE;
2599 }
2600 TIter nextmap(res);
2601 TMap *map = 0;
2602 while ((map=(TMap*)nextmap())) {
2603 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2604 if (!objs || !objs->GetString().Length()) {
2605 // Nothing found - skip this output
2606 delete res;
2607 delete listoffiles;
2608 return kFALSE;
2609 }
2610 listoffiles->Add(new TNamed(objs->GetName(),""));
2611 }
2612 delete res;
2613 }
2614 if (!listoffiles->GetEntries()) {
7c2cd90a 2615 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2616 delete listoffiles;
0f389141 2617 return kFALSE;
2618 }
2619
2620 TFileMerger *fm = 0;
b3e07543 2621 TIter next0(listoffiles);
2622 TObjArray *listoffilestmp = new TObjArray();
2623 listoffilestmp->SetOwner();
2624 TObject *nextfile;
2625 TString snextfile;
2626 // Keep only the files at upper level
2627 Int_t countChar = 0;
2628 while ((nextfile=next0())) {
2629 snextfile = nextfile->GetName();
2630 Int_t crtCount = snextfile.CountChar('/');
2631 if (nextfile == listoffiles->First()) countChar = crtCount;
2632 if (crtCount < countChar) countChar = crtCount;
2633 }
2634 next0.Reset();
2635 while ((nextfile=next0())) {
2636 snextfile = nextfile->GetName();
2637 Int_t crtCount = snextfile.CountChar('/');
2638 if (crtCount > countChar) {
2639 delete nextfile;
2640 continue;
2641 }
2642 listoffilestmp->Add(nextfile);
2643 }
2644 delete listoffiles;
2645 listoffiles = listoffilestmp; // Now contains 'good' files
2646 listoffiles->Print();
2647 TIter next(listoffiles);
7c2cd90a 2648 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2649 outputChunk = outputFile;
2650 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2651 // Check for existent temporary merge files
e1c22e21 2652 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2653 // Preserve old merging functionality for stage 0.
2654 if (stage==0) {
2655 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2656 while (1) {
2657 // Skip as many input files as in a chunk
2ea55496 2658 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2659 nextfile = next();
2660 if (!nextfile) {
2ea55496 2661 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2662 delete listoffiles;
2ea55496 2663 return kFALSE;
2664 }
b3e07543 2665 snextfile = nextfile->GetName();
7c2cd90a 2666 }
2667 outputChunk = outputFile;
2668 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2669 countChunk++;
2670 if (gSystem->AccessPathName(outputChunk)) continue;
2671 // Merged file with chunks up to <countChunk> found
2672 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2673 previousChunk = outputChunk;
2674 break;
2675 }
2676 }
2677 countZero = nmaxmerge;
2678
b3e07543 2679 while ((nextfile=next())) {
2680 snextfile = nextfile->GetName();
2ea55496 2681 // Loop 'find' results and get next LFN
7c2cd90a 2682 if (countZero == nmaxmerge) {
2683 // First file in chunk - create file merger and add previous chunk if any.
2b895f34 2684 fm = new TFileMerger(kTRUE);
7c2cd90a 2685 fm->SetFastMethod(kTRUE);
2686 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2687 outputChunk = outputFile;
2688 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2689 }
2690 // If last file found, put merged results in the output file
b3e07543 2691 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2692 // Add file to be merged and decrement chunk counter.
b3e07543 2693 fm->AddFile(snextfile);
7c2cd90a 2694 countZero--;
b3e07543 2695 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2696 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2697 // Nothing found - skip this output
2698 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2699 merged = kFALSE;
2700 break;
7c2cd90a 2701 }
2702 fm->OutputFile(outputChunk);
2703 // Merge the outputs, then go to next chunk
2704 if (!fm->Merge()) {
2705 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2706 merged = kFALSE;
2707 break;
7c2cd90a 2708 } else {
2709 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2710 gSystem->Unlink(previousChunk);
2711 }
b3e07543 2712 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2713 countChunk++;
2714 countZero = nmaxmerge;
2715 previousChunk = outputChunk;
0f389141 2716 }
0f389141 2717 }
b3e07543 2718 delete listoffiles;
2719 delete fm;
7c2cd90a 2720 return merged;
2721 }
2722 // Merging stage different than 0.
2723 // Move to the begining of the requested chunk.
2b895f34 2724 fm = new TFileMerger(kTRUE);
7c2cd90a 2725 fm->SetFastMethod(kTRUE);
b3e07543 2726 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2727 delete listoffiles;
7c2cd90a 2728 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2729 // Nothing found - skip this output
2730 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2731 delete fm;
2732 return kFALSE;
2733 }
b3e07543 2734 fm->OutputFile(outputFile);
7c2cd90a 2735 // Merge the outputs
2736 if (!fm->Merge()) {
2737 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2738 delete fm;
2739 return kFALSE;
2740 } else {
b3e07543 2741 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2742 }
7c2cd90a 2743 delete fm;
2744 return kTRUE;
0f389141 2745}
2746
c57f56b7 2747//______________________________________________________________________________
2748Bool_t AliAnalysisAlien::MergeOutputs()
2749{
2750// Merge analysis outputs existing in the AliEn space.
2751 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2752 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2753 if (!Connect()) {
2754 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2755 return kFALSE;
0f389141 2756 }
c9e8f7fd 2757 if (fMergeViaJDL) {
2758 if (!TestBit(AliAnalysisGrid::kMerge)) {
2759 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2760 return kFALSE;
2761 }
2762 if (fProductionMode) {
2763 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2764 return kFALSE;
2765 }
0f389141 2766 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2767 if (!SubmitMerging()) return kFALSE;
0f389141 2768 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2769 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2770 return kFALSE;
c57f56b7 2771 }
2772 // Get the output path
dc030a24 2773 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2774 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2775 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2776 return kFALSE;
2777 }
2778 if (!fOutputFiles.Length()) {
2779 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2780 return kFALSE;
0f389141 2781 }
2782 // Check if fast read option was requested
e1c22e21 2783 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2784 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2785 if (fFastReadOption) {
d3339be3 2786 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2787 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2788 gEnv->SetValue("XNet.ConnectTimeout",50);
2789 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2790 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2791 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2792 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2793 }
e8b839ab 2794 // Make sure we change the temporary directory
2795 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2796 // Set temporary compilation directory to current one
2797 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2798 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2799 TIter next(list);
2800 TObjString *str;
a2f5fc01 2801 TString outputFile;
c57f56b7 2802 Bool_t merged = kTRUE;
2803 while((str=(TObjString*)next())) {
a2f5fc01 2804 outputFile = str->GetString();
2805 Int_t index = outputFile.Index("@");
2806 if (index > 0) outputFile.Remove(index);
2807 TString outputChunk = outputFile;
2808 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2809 // Skip already merged outputs
a2f5fc01 2810 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2811 if (fOverwriteMode) {
a2f5fc01 2812 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2813 gSystem->Unlink(outputFile);
2814 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2815 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2816 outputChunk.Data());
2817 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2818 }
2819 } else {
a2f5fc01 2820 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2821 continue;
2822 }
2823 } else {
a2f5fc01 2824 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2825 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2826 outputChunk.Data());
2827 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2828 }
2829 }
37d21c01 2830 if (fMergeExcludes.Contains(outputFile.Data()) ||
2831 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 2832 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2833 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2834 if (!merged) {
2835 Error("MergeOutputs", "Terminate() will NOT be executed");
2836 return kFALSE;
ff07ec61 2837 }
2838 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2839 if (fileOpened) fileOpened->Close();
c57f56b7 2840 }
0f389141 2841 return kTRUE;
c57f56b7 2842}
2843
bb885a9e 2844//______________________________________________________________________________
2845void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2846{
2847// Use the output files connected to output containers from the analysis manager
2848// rather than the files defined by SetOutputFiles
2849 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2850 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2851 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2852}
2853
149d288c 2854//______________________________________________________________________________
2855void AliAnalysisAlien::SetOutputFiles(const char *list)
2856{
2857// Manually set the output files list.
2858// Removes duplicates. Not allowed if default outputs are not disabled.
2859 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2860 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2861 return;
2862 }
2863 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2864 fOutputFiles = "";
2865 TString slist = list;
2866 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2867 TObjArray *arr = slist.Tokenize(" ");
2868 TObjString *os;
2869 TIter next(arr);
2870 TString sout;
2871 while ((os=(TObjString*)next())) {
2872 sout = os->GetString();
2873 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2874 if (fOutputFiles.Contains(sout)) continue;
2875 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2876 fOutputFiles += sout;
2877 }
2878 delete arr;
f790bc1b 2879}
149d288c 2880
2881//______________________________________________________________________________
2882void AliAnalysisAlien::SetOutputArchive(const char *list)
2883{
2884// Manually set the output archive list. Free text - you are on your own...
2885// Not allowed if default outputs are not disabled.
2886 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2887 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2888 return;
2889 }
2890 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2891 fOutputArchive = list;
2892}
2893
2894//______________________________________________________________________________
2895void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2896{
2897// Setting a prefered output SE is not allowed anymore.
2898 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2899}
2900
28ce6c9c 2901//______________________________________________________________________________
2902void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
2903{
2904// Set some PROOF special parameter.
2905 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2906 if (pair) {
2907 TObject *old = pair->Key();
2908 TObject *val = pair->Value();
2909 fProofParam.Remove(old);
2910 delete old;
2911 delete val;
2912 }
2913 fProofParam.Add(new TObjString(pname), new TObjString(value));
2914}
2915
2916//______________________________________________________________________________
2917const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
2918{
2919// Returns a special PROOF parameter.
2920 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2921 if (!pair) return 0;
2922 return pair->Value()->GetName();
2923}
2924
5513444a 2925//______________________________________________________________________________
2926Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2927{
2928// Start remote grid analysis.
3bdcb562 2929 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2930 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2931 if (!mgr || !mgr->IsInitialized()) {
2932 Error("StartAnalysis", "You need an initialized analysis manager for this");
2933 return kFALSE;
2934 }
2935 // Are we in PROOF mode ?
2936 if (mgr->IsProofMode()) {
baa34407 2937 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
2938 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2939 if (fProofCluster.IsNull()) {
2940 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2941 return kFALSE;
2942 }
2943 if (fProofDataSet.IsNull() && !testMode) {
2944 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2945 return kFALSE;
2946 }
2947 // Set the needed environment
2948 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2949 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2950 if (fProofReset && !testMode) {
2951 if (fProofReset==1) {
2952 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2953 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2954 } else {
2955 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2956 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2957 }
2958 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2959 return kFALSE;
2960 }
baa34407 2961
2962 if (!testMode) {
2963 // Check if there is an old active session
2964 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
2965 if (nsessions) {
2966 Error("StartAnalysis","You have to reset your old session first\n");
2967 return kFALSE;
2968 }
53911ca6 2969 }
3bdcb562 2970 // Do we need to change the ROOT version ? The success of this cannot be checked.
2971 if (!fRootVersionForProof.IsNull() && !testMode) {
2972 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2973 fProofCluster.Data(), fRootVersionForProof.Data()));
2974 }
2975 // Connect to PROOF and check the status
2976 Long_t proof = 0;
27734f0e 2977 TString sworkers;
2978 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2979 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2980 if (!testMode) {
27734f0e 2981 if (!sworkers.IsNull())
2982 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2983 else
2984 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2985 } else {
2986 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2987 if (!proof) {
2988 Error("StartAnalysis", "Could not start PROOF in test mode");
2989 return kFALSE;
2990 }
2991 }
2992 if (!proof) {
2993 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2994 return kFALSE;
2995 }
27734f0e 2996 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2997 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 2998 // Set proof special parameters if any
2999 TIter nextpp(&fProofParam);
3000 TObject *proofparam;
3001 while ((proofparam=nextpp())) {
3002 TString svalue = GetProofParameter(proofparam->GetName());
3003 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3004 }
3bdcb562 3005 // Is dataset existing ?
3006 if (!testMode) {
3007 TString dataset = fProofDataSet;
3008 Int_t index = dataset.Index("#");
3009 if (index>=0) dataset.Remove(index);
830acc4c 3010// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3011// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3012// return kFALSE;
3013// }
3014// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3015 }
3016 // Is ClearPackages() needed ?
3017 if (TestSpecialBit(kClearPackages)) {
3018 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3019 gROOT->ProcessLine("gProof->ClearPackages();");
3020 }
3021 // Is a given aliroot mode requested ?
3022 TList optionsList;
1ac846e9 3023 TString parLibs;
830acc4c 3024 if (!fAliRootMode.IsNull()) {
3bdcb562 3025 TString alirootMode = fAliRootMode;
3026 if (alirootMode == "default") alirootMode = "";
3027 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3028 optionsList.SetOwner();
3029 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3030 // Check the additional libs to be loaded
3031 TString extraLibs;
1ac846e9 3032 Bool_t parMode = kFALSE;
dc1ffbcb 3033 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3034 // Parse the extra libs for .so
3035 if (fAdditionalLibs.Length()) {
3036 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3037 TIter next(list);
3038 TObjString *str;
1ac846e9 3039 while((str=(TObjString*)next())) {
3040 if (str->GetString().Contains(".so")) {
3041 if (parMode) {
3042 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3043 break;
3044 }
3045 TString stmp = str->GetName();
3046 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3047 stmp.ReplaceAll(".so","");
3048 if (!extraLibs.IsNull()) extraLibs += ":";
3049 extraLibs += stmp;
3050 continue;
3051 }
3052 if (str->GetString().Contains(".par")) {
3053 // The first par file found in the list will not allow any further .so
3054 parMode = kTRUE;
3055 if (!parLibs.IsNull()) parLibs += ":";
3056 parLibs += str->GetName();
3057 continue;
3058 }
3bdcb562 3059 }
3060 if (list) delete list;
3061 }
dc030a24 3062 if (!extraLibs.IsNull()) {
3063 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3064 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3065 }
830acc4c 3066 // Check extra includes
3067 if (!fIncludePath.IsNull()) {
3068 TString includePath = fIncludePath;
3069 includePath.ReplaceAll(" ",":");
dc030a24 3070 includePath.ReplaceAll("$ALICE_ROOT/","");
3071 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3072 includePath.ReplaceAll("-I","");
dc030a24 3073 includePath.Remove(TString::kTrailing, ':');
830acc4c 3074 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3075 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3076 }
27734f0e 3077 // Check if connection to grid is requested
3078 if (TestSpecialBit(kProofConnectGrid))
3079 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3080 // Enable AliRoot par
3081 if (testMode) {
3082 // Enable proof lite package
3083 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3084 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3085 TNamed *obj = (TNamed*)optionsList.At(i);
3086 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3087 }
3088 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3089 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3090 Info("StartAnalysis", "AliRootProofLite enabled");
3091 } else {
3092 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3093 return kFALSE;
3094 }
3095 } else {
dc030a24 3096 if ( ! fAliROOTVersion.IsNull() ) {
3097 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3098 fAliROOTVersion.Data(), &optionsList))) {
3099 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3100 return kFALSE;
3101 }
3102 }
27734f0e 3103 }
1ac846e9 3104 // Enable first par files from fAdditionalLibs
3105 if (!parLibs.IsNull()) {
3106 TObjArray *list = parLibs.Tokenize(":");
3107 TIter next(list);
3108 TObjString *package;
3109 while((package=(TObjString*)next())) {
40501bb7 3110 TString spkg = package->GetName();
3111 spkg.ReplaceAll(".par", "");
3112 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3113 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3114 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3115 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3116 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3117 return kFALSE;
3118 }
3119 } else {
3120 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3121 return kFALSE;
3122 }
3123 }
3124 if (list) delete list;
3125 }
3bdcb562 3126 } else {
3127 if (fAdditionalLibs.Contains(".so") && !testMode) {
3128 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3129 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3130 return kFALSE;
3131 }
3132 }
3133 // Enable par files if requested
3134 if (fPackages && fPackages->GetEntries()) {
3135 TIter next(fPackages);
3136 TObject *package;
3137 while ((package=next())) {
1ac846e9 3138 // Skip packages already enabled
3139 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3140 TString spkg = package->GetName();
3141 spkg.ReplaceAll(".par", "");
3142 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3143 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3144 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3145 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3146 return kFALSE;
3147 }
3148 } else {
3149 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3150 return kFALSE;
3151 }
3152 }
3153 }
3154 // Do we need to load analysis source files ?
3155 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3156 if (fAnalysisSource.Length()) {
3157 TObjArray *list = fAnalysisSource.Tokenize(" ");
3158 TIter next(list);
3159 TObjString *str;
3160 while((str=(TObjString*)next())) {
3161 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3162 }
3bdcb562 3163 if (list) delete list;
3164 }
830acc4c 3165 if (testMode) {
3166 // Register dataset to proof lite.
3167 if (fFileForTestMode.IsNull()) {
3168 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3169 return kFALSE;
3170 }
3171 if (gSystem->AccessPathName(fFileForTestMode)) {
3172 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3173 return kFALSE;
3174 }
3175 TFileCollection *coll = new TFileCollection();
3176 coll->AddFromFile(fFileForTestMode);
8ddad121 3177 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3178 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3179 }
3bdcb562 3180 return kTRUE;
3181 }
c57f56b7 3182
43da816a 3183 // Check if output files have to be taken from the analysis manager
3184 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3185 // Add output files and AOD files
3186 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3187 // Add extra files registered to the analysis manager
f790bc1b 3188 TString extra = GetListOfFiles("ext");
3189 if (!extra.IsNull()) {
149d288c 3190 extra.ReplaceAll(".root", "*.root");
f790bc1b 3191 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3192 fOutputFiles += extra;
84fcd93f 3193 }
149d288c 3194 // Compose the output archive.
b83f84f0 3195 fOutputArchive = "log_archive.zip:std*@disk=1 ";
3196 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3197 }
f7b1cbc2 3198// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3199 if (TestBit(AliAnalysisGrid::kOffline)) {
3200 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3201 \n there nor any job run. You can revise the JDL and analysis \
3202 \n macro then run the same in \"submit\" mode.");
3203 } else if (TestBit(AliAnalysisGrid::kTest)) {
3204 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3205 \n dataset.");
3206 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3207 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3208 \n space and job submitted.");
3209 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3210 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3211 if (fMergeViaJDL) CheckInputData();
5513444a 3212 return kTRUE;
c57f56b7 3213 } else {
3214 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3215 }
3216
348be253 3217 Print();
c57f56b7 3218 if (!Connect()) {
3219 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3220 return kFALSE;
16a4353c 3221 }
d3b18c4c 3222 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3223 if (!CheckInputData()) {
3224 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3225 return kFALSE;
c57f56b7 3226 }
d3339be3 3227 if (!CreateDataset(fDataPattern)) {
3228 TString serror;
3229 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3230 if (fRunNumbers.Length()) serror = "run numbers";
3231 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3232 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3233 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3234 return kFALSE;
3235 }
21ca8e59 3236 WriteAnalysisFile();
c57f56b7 3237 WriteAnalysisMacro();
3238 WriteExecutable();
3239 WriteValidationScript();
0f389141 3240 if (fMergeViaJDL) {
3241 WriteMergingMacro();
3242 WriteMergeExecutable();
3243 WriteValidationScript(kTRUE);
3244 }
5513444a 3245 if (!CreateJDL()) return kFALSE;
3246 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3247 if (testMode) {
c57f56b7 3248 // Locally testing the analysis
3249 Info("StartAnalysis", "\n_______________________________________________________________________ \
3250 \n Running analysis script in a daughter shell as on a worker node \
3251 \n_______________________________________________________________________");
149d288c 3252 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3253 TIter next(list);
3254 TObjString *str;
a2f5fc01 3255 TString outputFile;
c57f56b7 3256 while((str=(TObjString*)next())) {
a2f5fc01 3257 outputFile = str->GetString();
3258 Int_t index = outputFile.Index("@");
3259 if (index > 0) outputFile.Remove(index);
3260 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3261 }
3262 delete list;
3263 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3264 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3265// gSystem->Exec("cat stdout");
5513444a 3266 return kFALSE;
c57f56b7 3267 }
5513444a 3268 // Check if submitting is managed by LPM manager
a3e84053 3269 if (fProductionMode) {
5513444a 3270 TString prodfile = fJDLName;
3271 prodfile.ReplaceAll(".jdl", ".prod");
3272 WriteProductionFile(prodfile);
3273 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3274 return kFALSE;
3275 }
a8739e8a 3276 // Submit AliEn job(s)
d2a409b2 3277 gGrid->Cd(fGridOutputDir);
a8739e8a 3278 TGridResult *res;
c57f56b7 3279 TString jobID = "";
d2a409b2 3280 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3281 // Submit a given xml or a set of runs
a8739e8a 3282 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3283 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3284 if (res) {
3285 const char *cjobId = res->GetKey(0,"jobId");
3286 if (!cjobId) {
a03be957 3287 gGrid->Stdout();
3288 gGrid->Stderr();
a8739e8a 3289 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3290 return kFALSE;
a8739e8a 3291 } else {
3292 Info("StartAnalysis", "\n_______________________________________________________________________ \
3293 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3294 \n_______________________________________________________________________",
3295 fJDLName.Data(), cjobId);
3296 jobID = cjobId;
3297 }
3298 delete res;
a03be957 3299 } else {
3300 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3301 return kFALSE;
a8739e8a 3302 }
3303 } else {
d2a409b2 3304 // Submit for a range of enumeration of runs.
a03be957 3305 if (!Submit()) return kFALSE;
c57f56b7 3306 }
a8739e8a 3307
c57f56b7 3308 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3309 \n You may exit at any time and terminate the job later using the option <terminate> \
3310 \n ##################################################################################", jobID.Data());
bb885a9e 3311 gSystem->Exec("aliensh");
5513444a 3312 return kTRUE;
c57f56b7 3313}
3314
f790bc1b 3315//______________________________________________________________________________
3316const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3317{
3318// Get a comma-separated list of output files of the requested type.
3319// Type can be (case unsensitive):
3320// aod - list of aod files (std, extensions and filters)
3321// out - list of output files connected to containers (but not aod's or extras)
3322// ext - list of extra files registered to the manager
3323// ter - list of files produced in terminate
3324 static TString files;
3325 files = "";
3326 TString stype = type;
3327 stype.ToLower();
3328 TString aodfiles, extra;
3329 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3330 if (!mgr) {
3331 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3332 return files.Data();
3333 }
3334 if (mgr->GetOutputEventHandler()) {
3335 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3336 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3337 if (!extraaod.IsNull()) {
3338 aodfiles += ",";
3339 aodfiles += extraaod;
3340 }
3341 }
3342 if (stype.Contains("aod")) {
3343 files = aodfiles;
3344 if (stype == "aod") return files.Data();
3345 }
3346 // Add output files that are not in the list of AOD files
3347 TString outputfiles = "";
3348 TIter next(mgr->GetOutputs());
3349 AliAnalysisDataContainer *output;
3350 const char *filename = 0;
3351 while ((output=(AliAnalysisDataContainer*)next())) {
3352 filename = output->GetFileName();
3353 if (!(strcmp(filename, "default"))) continue;
3354 if (outputfiles.Contains(filename)) continue;
3355 if (aodfiles.Contains(filename)) continue;
3356 if (!outputfiles.IsNull()) outputfiles += ",";
3357 outputfiles += filename;
3358 }
3359 if (stype.Contains("out")) {
3360 if (!files.IsNull()) files += ",";
3361 files += outputfiles;
3362 if (stype == "out") return files.Data();
3363 }
3364 // Add extra files registered to the analysis manager
3365 TString sextra;
3366 extra = mgr->GetExtraFiles();
3367 if (!extra.IsNull()) {
3368 extra.Strip();
3369 extra.ReplaceAll(" ", ",");
3370 TObjArray *fextra = extra.Tokenize(",");
3371 TIter nextx(fextra);
3372 TObject *obj;
3373 while ((obj=nextx())) {
3374 if (aodfiles.Contains(obj->GetName())) continue;
3375 if (outputfiles.Contains(obj->GetName())) continue;
3376 if (sextra.Contains(obj->GetName())) continue;
3377 if (!sextra.IsNull()) sextra += ",";
3378 sextra += obj->GetName();
3379 }
3380 delete fextra;
3381 if (stype.Contains("ext")) {
3382 if (!files.IsNull()) files += ",";
3383 files += sextra;
3384 }
3385 }
3386 if (stype == "ext") return files.Data();
3387 TString termfiles;
3388 if (!fTerminateFiles.IsNull()) {
3389 fTerminateFiles.Strip();
3390 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3391 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3392 TIter nextx(fextra);
3393 TObject *obj;
3394 while ((obj=nextx())) {
3395 if (aodfiles.Contains(obj->GetName())) continue;
3396 if (outputfiles.Contains(obj->GetName())) continue;
3397 if (termfiles.Contains(obj->GetName())) continue;
3398 if (sextra.Contains(obj->GetName())) continue;
3399 if (!termfiles.IsNull()) termfiles += ",";
3400 termfiles += obj->GetName();
3401 }
3402 delete fextra;
3403 }
3404 if (stype.Contains("ter")) {
3405 if (!files.IsNull() && !termfiles.IsNull()) {
3406 files += ",";
3407 files += termfiles;
3408 }
3409 }
3410 return files.Data();
3411}
3412
d2a409b2 3413//______________________________________________________________________________
a03be957 3414Bool_t AliAnalysisAlien::Submit()
d2a409b2 3415{
3416// Submit all master jobs.
3417 Int_t nmasterjobs = fInputFiles->GetEntries();
3418 Long_t tshoot = gSystem->Now();
a03be957 3419 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3420 while (fNsubmitted < nmasterjobs) {
3421 Long_t now = gSystem->Now();
3422 if ((now-tshoot)>30000) {
3423 tshoot = now;
a03be957 3424 if (!SubmitNext()) return kFALSE;
d2a409b2 3425 }
3426 }
a03be957 3427 return kTRUE;
d2a409b2 3428}
3429
0f389141 3430//______________________________________________________________________________
a03be957 3431Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3432{
3433// Submit all merging jobs.
dc030a24 3434 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3435 gGrid->Cd(fGridOutputDir);
3436 TString mergeJDLName = fExecutable;
3437 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3438 if (!fInputFiles) {
3439 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3440 return kFALSE;
3441 }
0f389141 3442 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3443 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3444 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3445 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3446 if (fOutputToRunNo) {
3447 // The output directory is the run number
3448 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3449 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3450 } else {
17b4e9a4 3451 if (!fRunNumbers.Length() && !fRunRange[0]) {
3452 // The output directory is the grid outdir
3453 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3454 runOutDir = fGridOutputDir;
3455 } else {
3456 // The output directory is the master number in 3 digits format
3457 printf("### Submitting merging job for master <%03d>\n", i);
3458 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3459 }
7c2cd90a 3460 }
3461 // Check now the number of merging stages.
2b222f7c 3462 TObjArray *list = fOutputFiles.Tokenize(",");
3463 TIter next(list);
3464 TObjString *str;
3465 TString outputFile;
3466 while((str=(TObjString*)next())) {
3467 outputFile = str->GetString();
3468 Int_t index = outputFile.Index("@");
3469 if (index > 0) outputFile.Remove(index);
37d21c01 3470 if (!fMergeExcludes.Contains(outputFile) &&
3471 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3472 }
3473 delete list;
b3e07543 3474 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3475 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3476 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3477 }
a03be957 3478 if (!ntosubmit) return kTRUE;
9f020952 3479 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3480 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3481 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3482 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3483 \n ################################################################################################################");
0f389141 3484 gSystem->Exec("aliensh");
a03be957 3485 return kTRUE;
0f389141 3486}
3487
d2a409b2 3488//______________________________________________________________________________
a03be957 3489Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3490{
08d5b699 3491// Submit next bunch of master jobs if the queue is free. The first master job is
3492// submitted right away, while the next will not be unless the previous was split.
3493// The plugin will not submit new master jobs if there are more that 500 jobs in
3494// waiting phase.
d2a409b2 3495 static Bool_t iscalled = kFALSE;
3496 static Int_t firstmaster = 0;
3497 static Int_t lastmaster = 0;
3498 static Int_t npermaster = 0;
a03be957 3499 if (iscalled) return kTRUE;
d2a409b2 3500 iscalled = kTRUE;
3501 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3502 Int_t ntosubmit = 0;
3503 TGridResult *res;
3504 TString jobID = "";
3bdcb562 3505 Int_t nmasterjobs = fInputFiles->GetEntries();
3506 if (!fNsubmitted) {
3507 ntosubmit = 1;
3508 if (!IsUseSubmitPolicy()) {
90d50a8c 3509 if (nmasterjobs>5)
3bdcb562 3510 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3511 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3512 ntosubmit = nmasterjobs;
3513 }
3514 } else {
d2a409b2 3515 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3516 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3517 // If last master not split, just return
a03be957 3518 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3519 // No more than 100 waiting jobs
08d5b699 3520 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3521 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3522 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3523 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3524 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3525 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3526 }
d2a409b2 3527 for (Int_t i=0; i<ntosubmit; i++) {
3528 // Submit for a range of enumeration of runs.
a03be957 3529 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3530 TString query;
cd11251e 3531 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3532 runOutDir.ReplaceAll(".xml", "");
3533 if (fOutputToRunNo)
3534 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3535 else
3536 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3537 printf("********* %s\n",query.Data());
d2a409b2 3538 res = gGrid->Command(query);
3539 if (res) {
98ca124f 3540 TString cjobId1 = res->GetKey(0,"jobId");
3541 if (!cjobId1.Length()) {
d2a409b2 3542 iscalled = kFALSE;
a03be957 3543 gGrid->Stdout();
3544 gGrid->Stderr();
3545 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3546 return kFALSE;
d2a409b2 3547 } else {
3548 Info("StartAnalysis", "\n_______________________________________________________________________ \
3549 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3550 \n_______________________________________________________________________",
98ca124f 3551 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 3552 jobID += cjobId1;
3553 jobID += " ";
98ca124f 3554 lastmaster = cjobId1.Atoi();
d2a409b2 3555 if (!firstmaster) firstmaster = lastmaster;
3556 fNsubmitted++;
3557 }
3558 delete res;
a03be957 3559 } else {
3560 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3561 return kFALSE;
d2a409b2 3562 }
3563 }
3564 iscalled = kFALSE;
a03be957 3565 return kTRUE;
d2a409b2 3566}
3567
c57f56b7 3568//______________________________________________________________________________
3569void AliAnalysisAlien::WriteAnalysisFile()
3570{
f10e8481 3571// Write current analysis manager into the file <analysisFile>
3572 TString analysisFile = fExecutable;
3573 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3574 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3575 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3576 if (!mgr || !mgr->IsInitialized()) {
3577 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3578 return;
3579 }
3580 // Check analysis type
3581 TObject *handler;
3582 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3583 handler = (TObject*)mgr->GetInputEventHandler();
3584 if (handler) {
65eb22f8 3585 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3586 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3587 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3588 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3589 } else {
3590 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3591 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3592 }
c57f56b7 3593 }
3594 TDirectory *cdir = gDirectory;
f10e8481 3595 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3596 if (file) {
e16a394c 3597 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3598 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3599 // Unless merging makes no sense
3600 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3601 mgr->Write();
3602 delete file;
fe2d7fc2 3603 // Enable termination for local jobs
3604 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3605 }
3606 if (cdir) cdir->cd();
f10e8481 3607 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3608 }
3609 Bool_t copy = kTRUE;
d3b18c4c 3610 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3611 if (copy) {
3612 CdWork();
3613 TString workdir = gGrid->GetHomeDirectory();
3614 workdir += fGridWorkingDir;
d3b18c4c 3615 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3616 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3617 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3618 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3619 }
3620}
3621
3622//______________________________________________________________________________
3623void AliAnalysisAlien::WriteAnalysisMacro()
3624{
3625// Write the analysis macro that will steer the analysis in grid mode.
3626 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3627 ofstream out;
3628 out.open(fAnalysisMacro.Data(), ios::out);
3629 if (!out.good()) {
3630 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3631 return;
3632 }
5b273635 3633 Bool_t hasSTEERBase = kFALSE;
3634 Bool_t hasESD = kFALSE;
3635 Bool_t hasAOD = kFALSE;
3636 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3637 Bool_t hasOADB = kFALSE;
5b273635 3638 Bool_t hasANALYSISalice = kFALSE;
3639 Bool_t hasCORRFW = kFALSE;
c57f56b7 3640 TString func = fAnalysisMacro;
3641 TString type = "ESD";
3642 TString comment = "// Analysis using ";
c85cfc0f 3643 if (IsUseMCchain()) {
3644 type = "MC";
3645 comment += "MC";
3646 } else {
3647 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3648 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3649 type = "AOD";
3650 comment += "AOD";
3651 }
3652 }
0df6ccf2 3653 if (type!="AOD" && fFriendChainName!="") {
3654 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3655 return;
3656 }
c57f56b7 3657 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3658 else comment += " data";
3659 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3660 func.ReplaceAll(".C", "");
3661 out << "void " << func.Data() << "()" << endl;
3662 out << "{" << endl;
3663 out << comment.Data() << endl;
3664 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3665 out << " TStopwatch timer;" << endl;
3666 out << " timer.Start();" << endl << endl;
13f0c00b 3667 // Change temp directory to current one
3668 out << "// Set temporary merging directory to current one" << endl;
3669 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3670 out << "// Set temporary compilation directory to current one" << endl;
3671 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3672 // Reset existing include path
3673 out << "// Reset existing include path and add current directory first in the search" << endl;
3674 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3675 if (!fExecutableCommand.Contains("aliroot")) {
3676 out << "// load base root libraries" << endl;
3677 out << " gSystem->Load(\"libTree\");" << endl;
3678 out << " gSystem->Load(\"libGeom\");" << endl;
3679 out << " gSystem->Load(\"libVMC\");" << endl;
3680 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3681 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3682 }
d5c6455a 3683 if (fAdditionalRootLibs.Length()) {
47a4137d 3684 // in principle libtree /lib geom libvmc etc. can go into this list, too
3685 out << "// Add aditional libraries" << endl;
3686 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3687 TIter next(list);
3688 TObjString *str;
3689 while((str=(TObjString*)next())) {
3690 if (str->GetString().Contains(".so"))
3691 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3692 }
47a4137d 3693 if (list) delete list;
d5c6455a 3694 }
57377eb5 3695 out << "// Load analysis framework libraries" << endl;
652561d5 3696 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3697 if (!fPackages) {
f790bc1b 3698 if (!fExecutableCommand.Contains("aliroot")) {
3699 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3700 out << " gSystem->Load(\"libESD\");" << endl;
3701 out << " gSystem->Load(\"libAOD\");" << endl;
3702 }
4e5c5506 3703 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3704 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3705 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3706 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3707 } else {
4e5c5506 3708 TIter next(fPackages);
3709 TObject *obj;
57377eb5 3710 TString pkgname;
57377eb5 3711 while ((obj=next())) {
3712 pkgname = obj->GetName();
4478e6f1 3713 if (pkgname == "STEERBase" ||
3714 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3715 if (pkgname == "ESD" ||
3716 pkgname == "ESD.par") hasESD = kTRUE;
3717 if (pkgname == "AOD" ||
3718 pkgname == "AOD.par") hasAOD = kTRUE;
3719 if (pkgname == "ANALYSIS" ||
3720 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 3721 if (pkgname == "OADB" ||
3722 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 3723 if (pkgname == "ANALYSISalice" ||
3724 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3725 if (pkgname == "CORRFW" ||
3726 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3727 }
3728 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3729 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3730 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3731 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3732 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3733 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3734 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3735 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3736 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 3737 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
3738 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 3739 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3740 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3741 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3742 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3743 out << "// Compile other par packages" << endl;
3744 next.Reset();
fcc9bb6f 3745 while ((obj=next())) {
3746 pkgname = obj->GetName();
4478e6f1 3747 if (pkgname == "STEERBase" ||
3748 pkgname == "STEERBase.par" ||
3749 pkgname == "ESD" ||
3750 pkgname == "ESD.par" ||
3751 pkgname == "AOD" ||
3752 pkgname == "AOD.par" ||
3753 pkgname == "ANALYSIS" ||
3754 pkgname == "ANALYSIS.par" ||
7ea40ac8 3755 pkgname == "OADB" ||
3756 pkgname == "OADB.par" ||
4478e6f1 3757 pkgname == "ANALYSISalice" ||
3758 pkgname == "ANALYSISalice.par" ||
3759 pkgname == "CORRFW" ||
3760 pkgname == "CORRFW.par") continue;
5b273635 3761 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3762 }
4e5c5506 3763 }
b3e07543 3764 out << "// include path" << endl;
3765 // Get the include path from the interpreter and remove entries pointing to AliRoot
3766 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3767 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3768 out << " TIter nextpath(listpaths);" << endl;
3769 out << " TObjString *pname;" << endl;
3770 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3771 out << " TString current = pname->GetName();" << endl;
3772 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3773 out << " gSystem->AddIncludePath(current);" << endl;
3774 out << " }" << endl;
3775 out << " if (listpaths) delete listpaths;" << endl;
3776 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3777 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3778 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3779 if (fAdditionalLibs.Length()) {
3780 out << "// Add aditional AliRoot libraries" << endl;
3781 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3782 TIter next(list);
3783 TObjString *str;
3784 while((str=(TObjString*)next())) {
3785 if (str->GetString().Contains(".so"))
3786 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3787 if (str->GetString().Contains(".par"))
3788 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3789 }
3790 if (list) delete list;
3791 }
3792 out << endl;
c57f56b7 3793 out << "// analysis source to be compiled at runtime (if any)" << endl;
3794 if (fAnalysisSource.Length()) {
3795 TObjArray *list = fAnalysisSource.Tokenize(" ");
3796 TIter next(list);
3797 TObjString *str;
3798 while((str=(TObjString*)next())) {
3799 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3800 }
3801 if (list) delete list;
3802 }
3803 out << endl;
b3e07543 3804// out << " printf(\"Currently load libraries:\\n\");" << endl;
3805// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3806 if (fFastReadOption) {
d3339be3 3807 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3808 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3809 out << "// fast xrootd reading enabled" << endl;
3810 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3811 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3812 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3813 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3814 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3815 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 3816 }
3817 if (!IsLocalTest()) {
3818 out << "// connect to AliEn and make the chain" << endl;
3819 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
0f389141 3820 }
c57f56b7 3821 out << "// read the analysis manager from file" << endl;
f10e8481 3822 TString analysisFile = fExecutable;
3823 analysisFile.ReplaceAll(".sh", ".root");
105631b5 3824 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
3825 << analysisFile << "\");" << endl;
3826 out << " if (!mgr) return;" << endl;
21ca8e59 3827 if (IsLocalTest()) {
3828 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
3829 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 3830 if (fFileForTestMode.IsNull())
3831 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
3832 else
3833 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 3834 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
21ca8e59 3835 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 3836 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 3837 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 3838 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
3839 } else {
7a366e0b 3840 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 3841 out << " mgr->SetNSysInfo(100);" << endl;
3842 }
21ca8e59 3843 }
c57f56b7 3844 out << " mgr->PrintStatus();" << endl;
52b6a92b 3845 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3846 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3847 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3848 } else {
9a8288ff 3849 if (TestBit(AliAnalysisGrid::kTest))
3850 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3851 else
3852 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3853 }
3854 }
21ca8e59 3855 if (!IsLocalTest()) {
242accb2 3856 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 3857 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
3858 } else {
3859 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 3860 }
f7498086 3861 out << " timer.Stop();" << endl;
3862 out << " timer.Print();" << endl;
c57f56b7 3863 out << "}" << endl << endl;
21ca8e59 3864 if (!IsLocalTest()) {
fcc9bb6f 3865 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3866 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3867 out << "{" << endl;
3868 out << "// Create a chain using url's from xml file" << endl;
242accb2 3869 out << " TString filename;" << endl;
3870 out << " Int_t run = 0;" << endl;
c85cfc0f 3871 if (IsUseMCchain()) {
3872 out << " TString treename = \"TE\";" << endl;
3873 } else {
3874 out << " TString treename = type;" << endl;
3875 out << " treename.ToLower();" << endl;
3876 out << " treename += \"Tree\";" << endl;
3877 }
e02fee64 3878 out << " printf(\"***************************************\\n\");" << endl;
3879 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3880 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3881 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3882 out << " if (!coll) {" << endl;
3883 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3884 out << " return NULL;" << endl;
3885 out << " }" << endl;
242accb2 3886 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3887 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3888 if(fFriendChainName!="") {
3889 out << " TChain *chainFriend = new TChain(treename);" << endl;
3890 }
c57f56b7 3891 out << " coll->Reset();" << endl;
0df6ccf2 3892 out << " while (coll->Next()) {" << endl;
242accb2 3893 out << " filename = coll->GetTURL("");" << endl;
3894 out << " if (mgr) {" << endl;
3895 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3896 out << " if (nrun && nrun != run) {" << endl;
3897 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3898 out << " mgr->SetRunFromPath(nrun);" << endl;
3899 out << " run = nrun;" << endl;
3900 out << " }" << endl;
3901 out << " }" << endl;
3902 out << " chain->Add(filename);" << endl;
0df6ccf2 3903 if(fFriendChainName!="") {
3904 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3905 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3906 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3907 out << " chainFriend->Add(fileFriend.Data());" << endl;
3908 }
3909 out << " }" << endl;
c57f56b7 3910 out << " if (!chain->GetNtrees()) {" << endl;
3911 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3912 out << " return NULL;" << endl;
3913 out << " }" << endl;
0df6ccf2 3914 if(fFriendChainName!="") {
3915 out << " chain->AddFriend(chainFriend);" << endl;
3916 }
c57f56b7 3917 out << " return chain;" << endl;
fcc9bb6f 3918 out << "}" << endl << endl;
c57f56b7 3919 }
5b273635 3920 if (hasANALYSISalice) {
3921 out <<"//________________________________________________________________________________" << endl;
3922 out << "Bool_t SetupPar(const char *package) {" << endl;
3923 out << "// Compile the package and set it up." << endl;
3924 out << " TString pkgdir = package;" << endl;
3925 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3926 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3927 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3928 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3929 out << " // Check for BUILD.sh and execute" << endl;
3930 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3931 out << " printf(\"*******************************\\n\");" << endl;
3932 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3933 out << " printf(\"*******************************\\n\");" << endl;
3934 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3935 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3936 out << " gSystem->ChangeDirectory(cdir);" << endl;
3937 out << " return kFALSE;" << endl;
3938 out << " }" << endl;
3939 out << " } else {" << endl;
3940 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3941 out << " gSystem->ChangeDirectory(cdir);" << endl;
3942 out << " return kFALSE;" << endl;
3943 out << " }" << endl;
3944 out << " // Check for SETUP.C and execute" << endl;
3945 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3946 out << " printf(\"*******************************\\n\");" << endl;
3947 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3948 out << " printf(\"*******************************\\n\");" << endl;
3949 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3950 out << " } else {" << endl;
3951 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3952 out << " gSystem->ChangeDirectory(cdir);" << endl;
3953 out << " return kFALSE;" << endl;
3954 out << " }" << endl;
3955 out << " // Restore original workdir" << endl;
3956 out << " gSystem->ChangeDirectory(cdir);" << endl;
3957 out << " return kTRUE;" << endl;
3958 out << "}" << endl;
3959 }
c57f56b7 3960 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3961 }
3962 Bool_t copy = kTRUE;
d3b18c4c 3963 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3964 if (copy) {
3965 CdWork();
3966 TString workdir = gGrid->GetHomeDirectory();
3967 workdir += fGridWorkingDir;
3968 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 3969 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 3970// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3971 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
3972 Form("alien://%s/%s", workdir.Data(),
3973 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 3974 }
3975}
3976
0f389141 3977//______________________________________________________________________________
3978void AliAnalysisAlien::WriteMergingMacro()
3979{
3980// Write a macro to merge the outputs per master job.
3981 if (!fMergeViaJDL) return;
3982 if (!fOutputFiles.Length()) {
3983 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3984 return;
3985 }
3986 TString mergingMacro = fExecutable;
3987 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 3988 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3989 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3990 ofstream out;
3991 out.open(mergingMacro.Data(), ios::out);
3992 if (!out.good()) {
3993 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3994 return;
3995 }
5b273635 3996 Bool_t hasSTEERBase = kFALSE;
3997 Bool_t hasESD = kFALSE;
3998 Bool_t hasAOD = kFALSE;
3999 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4000 Bool_t hasOADB = kFALSE;
5b273635 4001 Bool_t hasANALYSISalice = kFALSE;
4002 Bool_t hasCORRFW = kFALSE;
0f389141 4003 TString func = mergingMacro;
4004 TString comment;
4005 func.ReplaceAll(".C", "");
f47d5cb4 4006 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4007 out << "{" << endl;
4008 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4009 out << " TStopwatch timer;" << endl;
4010 out << " timer.Start();" << endl << endl;
b3e07543 4011 // Reset existing include path
4012 out << "// Reset existing include path and add current directory first in the search" << endl;
4013 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4014 if (!fExecutableCommand.Contains("aliroot")) {
4015 out << "// load base root libraries" << endl;
4016 out << " gSystem->Load(\"libTree\");" << endl;
4017 out << " gSystem->Load(\"libGeom\");" << endl;
4018 out << " gSystem->Load(\"libVMC\");" << endl;
4019 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4020 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4021 }
0f389141 4022 if (fAdditionalRootLibs.Length()) {
4023 // in principle libtree /lib geom libvmc etc. can go into this list, too
4024 out << "// Add aditional libraries" << endl;
4025 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4026 TIter next(list);
4027 TObjString *str;
4028 while((str=(TObjString*)next())) {
4029 if (str->GetString().Contains(".so"))
4030 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4031 }
4032 if (list) delete list;
4033 }
0f389141 4034 out << "// Load analysis framework libraries" << endl;
4035 if (!fPackages) {
7c2cd90a 4036 if (!fExecutableCommand.Contains("aliroot")) {
4037 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4038 out << " gSystem->Load(\"libESD\");" << endl;
4039 out << " gSystem->Load(\"libAOD\");" << endl;
4040 }
0f389141 4041 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4042 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4043 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4044 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4045 } else {
4046 TIter next(fPackages);
4047 TObject *obj;
4048 TString pkgname;
5b273635 4049 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4050 while ((obj=next())) {
4051 pkgname = obj->GetName();
4052 if (pkgname == "STEERBase" ||
4053 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4054 if (pkgname == "ESD" ||
4055 pkgname == "ESD.par") hasESD = kTRUE;
4056 if (pkgname == "AOD" ||
4057 pkgname == "AOD.par") hasAOD = kTRUE;
4058 if (pkgname == "ANALYSIS" ||
4059 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4060 if (pkgname == "OADB" ||
4061 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4062 if (pkgname == "ANALYSISalice" ||
4063 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4064 if (pkgname == "CORRFW" ||
4065 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4066 }
5b273635 4067 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4068 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4069 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4070 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4071 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4072 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4073 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4074 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4075 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4076 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4077 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4078 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4079 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4080 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4081 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4082 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4083 out << "// Compile other par packages" << endl;
4084 next.Reset();
4085 while ((obj=next())) {
4086 pkgname = obj->GetName();
4087 if (pkgname == "STEERBase" ||
4088 pkgname == "STEERBase.par" ||
4089 pkgname == "ESD" ||
4090 pkgname == "ESD.par" ||
4091 pkgname == "AOD" ||
4092 pkgname == "AOD.par" ||
4093 pkgname == "ANALYSIS" ||
4094 pkgname == "ANALYSIS.par" ||
7ea40ac8 4095 pkgname == "OADB" ||
4096 pkgname == "OADB.par" ||
0f389141 4097 pkgname == "ANALYSISalice" ||
4098 pkgname == "ANALYSISalice.par" ||
4099 pkgname == "CORRFW" ||
4100 pkgname == "CORRFW.par") continue;
5b273635 4101 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4102 }
4103 }
b3e07543 4104 out << "// include path" << endl;
4105 // Get the include path from the interpreter and remove entries pointing to AliRoot
4106 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4107 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4108 out << " TIter nextpath(listpaths);" << endl;
4109 out << " TObjString *pname;" << endl;
4110 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4111 out << " TString current = pname->GetName();" << endl;
4112 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4113 out << " gSystem->AddIncludePath(current);" << endl;
4114 out << " }" << endl;
4115 out << " if (listpaths) delete listpaths;" << endl;
4116 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4117 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4118 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4119 if (fAdditionalLibs.Length()) {
4120 out << "// Add aditional AliRoot libraries" << endl;
4121 TObjArray *list = fAdditionalLibs.Tokenize(" ");
4122 TIter next(list);
4123 TObjString *str;
4124 while((str=(TObjString*)next())) {
4125 if (str->GetString().Contains(".so"))
4126 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4127 }
4128 if (list) delete list;
4129 }
4130 out << endl;
4131 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4132 if (fAnalysisSource.Length()) {
4133 TObjArray *list = fAnalysisSource.Tokenize(" ");
4134 TIter next(list);
4135 TObjString *str;
4136 while((str=(TObjString*)next())) {
4137 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4138 }
4139 if (list) delete list;
4140 }
149d288c 4141 out << endl;
4142
0f389141 4143 if (fFastReadOption) {
4144 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4145 out << "// fast xrootd reading enabled" << endl;
4146 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4147 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4148 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4149 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4150 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4151 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4152 }
4153 // Change temp directory to current one
4154 out << "// Set temporary merging directory to current one" << endl;
4155 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4156 out << "// Set temporary compilation directory to current one" << endl;
4157 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
0f389141 4158 out << "// Connect to AliEn" << endl;
4159 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 4160 out << " TString outputDir = dir;" << endl;
f790bc1b 4161 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4162 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4163 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4164 out << " TIter *iter = new TIter(list);" << endl;
4165 out << " TObjString *str;" << endl;
a2f5fc01 4166 out << " TString outputFile;" << endl;
0f389141 4167 out << " Bool_t merged = kTRUE;" << endl;
4168 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4169 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4170 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4171 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4172 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4173 out << " // Skip already merged outputs" << endl;
a2f5fc01 4174 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
4175 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 4176 out << " continue;" << endl;
4177 out << " }" << endl;
a2f5fc01 4178 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4179 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4180 out << " if (!merged) {" << endl;
a2f5fc01 4181 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4182 out << " return;" << endl;
0f389141 4183 out << " }" << endl;
4184 out << " }" << endl;
7c2cd90a 4185 out << " // all outputs merged, validate" << endl;
4186 out << " ofstream out;" << endl;
4187 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4188 out << " out.close();" << endl;
4189 out << " // read the analysis manager from file" << endl;
f866cba5 4190 TString analysisFile = fExecutable;
4191 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 4192 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
105631b5 4193 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4194 << analysisFile << "\");" << endl;
4195 out << " if (!mgr) return;" << endl;
5c4250fc 4196 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4197 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4198 out << " mgr->PrintStatus();" << endl;
4199 if (AliAnalysisManager::GetAnalysisManager()) {
4200 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
4201 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4202 } else {
9a8288ff 4203 if (TestBit(AliAnalysisGrid::kTest))
4204 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4205 else
4206 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4207 }
4208 }
9a8288ff 4209 out << " TTree *tree = NULL;" << endl;
4210 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4211 out << "}" << endl << endl;
5b273635 4212 if (hasANALYSISalice) {
4213 out <<"//________________________________________________________________________________" << endl;
4214 out << "Bool_t SetupPar(const char *package) {" << endl;
4215 out << "// Compile the package and set it up." << endl;
4216 out << " TString pkgdir = package;" << endl;
4217 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4218 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4219 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4220 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4221 out << " // Check for BUILD.sh and execute" << endl;
4222 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4223 out << " printf(\"*******************************\\n\");" << endl;
4224 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4225 out << " printf(\"*******************************\\n\");" << endl;
4226 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4227 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4228 out << " gSystem->ChangeDirectory(cdir);" << endl;
4229 out << " return kFALSE;" << endl;
4230 out << " }" << endl;
4231 out << " } else {" << endl;
4232 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4233 out << " gSystem->ChangeDirectory(cdir);" << endl;
4234 out << " return kFALSE;" << endl;
4235 out << " }" << endl;
4236 out << " // Check for SETUP.C and execute" << endl;
4237 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4238 out << " printf(\"*******************************\\n\");" << endl;
4239 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4240 out << " printf(\"*******************************\\n\");" << endl;
4241 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4242 out << " } else {" << endl;
4243 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4244 out << " gSystem->ChangeDirectory(cdir);" << endl;
4245 out << " return kFALSE;" << endl;
4246 out << " }" << endl;
4247 out << " // Restore original workdir" << endl;
4248 out << " gSystem->ChangeDirectory(cdir);" << endl;
4249 out << " return kTRUE;" << endl;
4250 out << "}" << endl;
4251 }
0f389141 4252 }
4253 Bool_t copy = kTRUE;
d3b18c4c 4254 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4255 if (copy) {
4256 CdWork();
4257 TString workdir = gGrid->GetHomeDirectory();
4258 workdir += fGridWorkingDir;
4259 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4260 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4261// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4262 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4263 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4264 }
4265}
4266
4267//______________________________________________________________________________
4268Bool_t AliAnalysisAlien::SetupPar(const char *package)
4269{
205b201f 4270// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4271// Note that for loading the compiled library. The current directory should have precedence in
4272// LD_LIBRARY_PATH
4273 TString pkgdir = package;
4274 pkgdir.ReplaceAll(".par","");
b50a41c3 4275 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4276 TString cdir = gSystem->WorkingDirectory();
4277 gSystem->ChangeDirectory(pkgdir);
4278 // Check for BUILD.sh and execute
4279 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4280 printf("**************************************************\n");
4281 printf("*** Building PAR archive %s\n", package);
4282 printf("**************************************************\n");
4283 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4284 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4285 gSystem->ChangeDirectory(cdir);
4286 return kFALSE;
4287 }
4288 } else {
4289 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4290 gSystem->ChangeDirectory(cdir);
4291 return kFALSE;
4292 }
4293 // Check for SETUP.C and execute
4294 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4295 printf("**************************************************\n");
4296 printf("*** Setup PAR archive %s\n", package);
4297 printf("**************************************************\n");
4298 gROOT->Macro("PROOF-INF/SETUP.C");
4299 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4300 } else {
4301 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4302 gSystem->ChangeDirectory(cdir);
4303 return kFALSE;
4304 }
4305 // Restore original workdir
4306 gSystem->ChangeDirectory(cdir);
4307 return kTRUE;
4308}
4309
c57f56b7 4310//______________________________________________________________________________
4311void AliAnalysisAlien::WriteExecutable()
4312{
4313// Generate the alien executable script.
4314 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4315 ofstream out;
4316 out.open(fExecutable.Data(), ios::out);
4317 if (out.bad()) {
5513444a 4318 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4319 return;
4320 }
4321 out << "#!/bin/bash" << endl;
2a4d5166 4322 // Make sure we can properly compile par files
4323 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4324 out << "echo \"=========================================\"" << endl;
4325 out << "echo \"############## PATH : ##############\"" << endl;
4326 out << "echo $PATH" << endl;
4327 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4328 out << "echo $LD_LIBRARY_PATH" << endl;
4329 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4330 out << "echo $ROOTSYS" << endl;
4331 out << "echo \"############## which root : ##############\"" << endl;
4332 out << "which root" << endl;
4333 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4334 out << "echo $ALICE_ROOT" << endl;
4335 out << "echo \"############## which aliroot : ##############\"" << endl;
4336 out << "which aliroot" << endl;
9c5ddadc 4337 out << "echo \"############## system limits : ##############\"" << endl;
4338 out << "ulimit -a" << endl;
4339 out << "echo \"############## memory : ##############\"" << endl;
4340 out << "free -m" << endl;
c57f56b7 4341 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4342 out << fExecutableCommand << " ";
370c9880 4343 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4344 out << "RET=$?" << endl;
4345 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4346 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4347 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4348 out << " let sig=\"$RET - 128\""<<endl;
4349 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4350 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4351 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4352 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4353 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4354 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4355 out << " fi"<<endl;
4356 out << " exit $RET"<< endl;
4357 out << "fi" << endl << endl ;
4358 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4359 out << "echo \"############## memory after: ##############\"" << endl;
4360 out << "free -m" << endl;
c57f56b7 4361 }
4362 Bool_t copy = kTRUE;
d3b18c4c 4363 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4364 if (copy) {
4365 CdWork();
4366 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4367 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4368 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4369 workdir += fGridWorkingDir;
4370 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4371 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4372 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4373// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4374 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4375 executable.Data())) Fatal("","Terminating");
c57f56b7 4376 }
4377}
4378
0f389141 4379//______________________________________________________________________________
4380void AliAnalysisAlien::WriteMergeExecutable()
4381{
4382// Generate the alien executable script for the merging job.
4383 if (!fMergeViaJDL) return;
4384 TString mergeExec = fExecutable;
4385 mergeExec.ReplaceAll(".sh", "_merge.sh");
4386 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4387 ofstream out;
4388 out.open(mergeExec.Data(), ios::out);
4389 if (out.bad()) {
4390 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4391 return;
4392 }
4393 out << "#!/bin/bash" << endl;
2a4d5166 4394 // Make sure we can properly compile par files
4395 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4396 out << "echo \"=========================================\"" << endl;
4397 out << "echo \"############## PATH : ##############\"" << endl;
4398 out << "echo $PATH" << endl;
4399 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4400 out << "echo $LD_LIBRARY_PATH" << endl;
4401 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4402 out << "echo $ROOTSYS" << endl;
4403 out << "echo \"############## which root : ##############\"" << endl;
4404 out << "which root" << endl;
4405 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4406 out << "echo $ALICE_ROOT" << endl;
4407 out << "echo \"############## which aliroot : ##############\"" << endl;
4408 out << "which aliroot" << endl;
4409 out << "echo \"############## system limits : ##############\"" << endl;
4410 out << "ulimit -a" << endl;
4411 out << "echo \"############## memory : ##############\"" << endl;
4412 out << "free -m" << endl;
4413 out << "echo \"=========================================\"" << endl << endl;
0f389141 4414 TString mergeMacro = fExecutable;
4415 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4416 if (IsOneStageMerging())
4417 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4418 else
f47d5cb4 4419 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4420 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4421 out << "RET=$?" << endl;
4422 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4423 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4424 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4425 out << " let sig=\"$RET - 128\""<<endl;
4426 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4427 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4428 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4429 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4430 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4431 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4432 out << " fi"<<endl;
4433 out << " exit $RET"<< endl;
4434 out << "fi" << endl << endl ;
0f389141 4435 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4436 out << "echo \"############## memory after: ##############\"" << endl;
4437 out << "free -m" << endl;
0f389141 4438 }
4439 Bool_t copy = kTRUE;
d3b18c4c 4440 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4441 if (copy) {
4442 CdWork();
4443 TString workdir = gGrid->GetHomeDirectory();
4444 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4445 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4446 workdir += fGridWorkingDir;
4447 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4448 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4449 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4450// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4451 if (!copyLocal2Alien("WriteMergeExecutable",
4452 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4453 }
4454}
4455
c57f56b7 4456//______________________________________________________________________________
5513444a 4457void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4458{
4459// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4460// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4461// Next lines: full_path_to_dataset XXX (XXX is a string)
4462// To submit, one has to: submit jdl XXX for all lines
4463 ofstream out;
4464 out.open(filename, ios::out);
4465 if (out.bad()) {
4466 Error("WriteProductionFile", "Bad file name: %s", filename);
4467 return;
4468 }
d3b18c4c 4469 TString workdir;
4470 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4471 workdir = gGrid->GetHomeDirectory();
5513444a 4472 workdir += fGridWorkingDir;
f5e8c702 4473 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4474 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4475 out << locjdl << " " << njobspermaster << endl;
5513444a 4476 Int_t nmasterjobs = fInputFiles->GetEntries();
4477 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4478 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4479 runOutDir.ReplaceAll(".xml", "");
4480 if (fOutputToRunNo)
4481 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4482 else
4483 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4484 }
d3b18c4c 4485 if (gGrid) {
4486 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4487 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4488// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4489 if (!copyLocal2Alien("WriteProductionFile", filename,
4490 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4491 }
5513444a 4492}
4493
4494//______________________________________________________________________________
0f389141 4495void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4496{
4497// Generate the alien validation script.
4498 // Generate the validation script
4499 TObjString *os;
d3b18c4c 4500 if (fValidationScript.IsNull()) {
4501 fValidationScript = fExecutable;
4502 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4503 }
4504 TString validationScript = fValidationScript;
4505 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4506 if (!Connect()) {
4507 Error("WriteValidationScript", "Alien connection required");
4508 return;
4509 }
d3b18c4c 4510 if (!fTerminateFiles.IsNull()) {
4511 fTerminateFiles.Strip();
4512 fTerminateFiles.ReplaceAll(" ",",");
4513 }
a2f5fc01 4514 TString outStream = "";
4515 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4516 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4517 ofstream out;
0d5d317c 4518 out.open(validationScript, ios::out);
c57f56b7 4519 out << "#!/bin/bash" << endl;
4520 out << "##################################################" << endl;
4521 out << "validateout=`dirname $0`" << endl;
4522 out << "validatetime=`date`" << endl;
4523 out << "validated=\"0\";" << endl;
4524 out << "error=0" << endl;
4525 out << "if [ -z $validateout ]" << endl;
4526 out << "then" << endl;
4527 out << " validateout=\".\"" << endl;
4528 out << "fi" << endl << endl;
4529 out << "cd $validateout;" << endl;
4530 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4531 out << "echo \"*******************************************************\"" << outStream << endl;
4532 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4533 out << "" << endl;
a2f5fc01 4534 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4535 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4536 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4537 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4538 out << "ls -la ./" << outStream << endl;
4539 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4540 out << "##################################################" << endl;
ebec370a 4541 out << "" << endl;
4542
4543 out << "if [ ! -f stderr ] ; then" << endl;
4544 out << " error=1" << endl;
a2f5fc01 4545 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4546 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4547 out << "fi" << endl;
4548
b34c9f51 4549 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4550 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4551 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4552 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4553 out << "" << endl;
4554
ebec370a 4555 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4556 out << " error=1" << endl;
a2f5fc01 4557 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4558 out << " echo \"$parArch\" " << outStream << endl;
4559 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4560 out << "fi" << endl;
4561
4562 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4563 out << " error=1" << endl;
a2f5fc01 4564 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4565 out << " echo \"$segViol\" " << outStream << endl;
4566 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4567 out << "fi" << endl;
4568
4569 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4570 out << " error=1" << endl;
a2f5fc01 4571 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4572 out << " echo \"$segFault\" " << outStream << endl;
4573 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4574 out << "fi" << endl;
4575
b34c9f51 4576 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4577 out << " error=1" << endl;
a2f5fc01 4578 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4579 out << " echo \"$glibcErr\" " << outStream << endl;
4580 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4581 out << "fi" << endl;
4582
ebec370a 4583 // Part dedicated to the specific analyses running into the train
4584
d3b18c4c 4585 TString outputFiles = fOutputFiles;
4586 if (merge && !fTerminateFiles.IsNull()) {
4587 outputFiles += ",";
4588 outputFiles += fTerminateFiles;
4589 }
4590 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4591 TIter next1(arr);
a2f5fc01 4592 TString outputFile;
bbd88db8 4593 while (!merge && (os=(TObjString*)next1())) {
4594 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4595 outputFile = os->GetString();
4596 Int_t index = outputFile.Index("@");
4597 if (index > 0) outputFile.Remove(index);
bbd88db8 4598 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4599 if (outputFile.Contains("*")) continue;
4600 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4601 out << " error=1" << endl;
7c2cd90a 4602 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4603 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4604 out << "fi" << endl;
4605 }
4606 delete arr;
7c2cd90a 4607 out << "if ! [ -f outputs_valid ] ; then" << endl;
4608 out << " error=1" << endl;
4609 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4610 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4611 out << "fi" << endl;
923e2ca5 4612
c57f56b7 4613 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4614 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4615 if (!IsKeepLogs()) {
4616 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4617 outStream = "";
149d288c 4618 out << " rm -f std*" << endl;
4619 }
c57f56b7 4620 out << "fi" << endl;
4621
a2f5fc01 4622 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4623 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4624 out << "cd -" << endl;
4625 out << "exit $error" << endl;
4626 }
4627 Bool_t copy = kTRUE;
d3b18c4c 4628 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4629 if (copy) {
4630 CdWork();
4631 TString workdir = gGrid->GetHomeDirectory();
4632 workdir += fGridWorkingDir;
d3b18c4c 4633 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4634 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 4635// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4636 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
4637 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 4638 }
4639}