]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Coverity warning corrected.
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
51ClassImp(AliAnalysisAlien)
6c395669 52#if 0
53;
54#endif
c57f56b7 55
6c395669 56namespace {
57 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
58 {
59 TString sl(Form("file:%s", loc));
60 TString sr(Form("alien://%s", rem));
61 Bool_t ret = TFile::Cp(sl, sr);
62 if (!ret) {
63 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
64 }
65 return ret;
66 }
67}
68
c57f56b7 69//______________________________________________________________________________
70AliAnalysisAlien::AliAnalysisAlien()
71 :AliAnalysisGrid(),
72 fGridJDL(NULL),
0f389141 73 fMergingJDL(NULL),
c57f56b7 74 fPrice(0),
75 fTTL(0),
76 fSplitMaxInputFileNumber(0),
77 fMaxInitFailed(0),
78 fMasterResubmitThreshold(0),
bb885a9e 79 fNtestFiles(0),
319593fb 80 fNrunsPerMaster(0),
16a4353c 81 fMaxMergeFiles(0),
b3e07543 82 fMaxMergeStages(0),
d2a409b2 83 fNsubmitted(0),
a3e84053 84 fProductionMode(0),
cd11251e 85 fOutputToRunNo(0),
0f389141 86 fMergeViaJDL(0),
87 fFastReadOption(0),
e1c22e21 88 fOverwriteMode(1),
149d288c 89 fNreplicas(2),
3bdcb562 90 fNproofWorkers(0),
27734f0e 91 fNproofWorkersPerSlave(0),
3bdcb562 92 fProofReset(0),
c57f56b7 93 fRunNumbers(),
94 fExecutable(),
0a1c1f7f 95 fExecutableCommand(),
c57f56b7 96 fArguments(),
631c0b05 97 fExecutableArgs(),
c57f56b7 98 fAnalysisMacro(),
99 fAnalysisSource(),
d3b18c4c 100 fValidationScript(),
d5c6455a 101 fAdditionalRootLibs(),
c57f56b7 102 fAdditionalLibs(),
103 fSplitMode(),
104 fAPIVersion(),
105 fROOTVersion(),
106 fAliROOTVersion(),
648174cf 107 fExternalPackages(),
c57f56b7 108 fUser(),
109 fGridWorkingDir(),
110 fGridDataDir(),
111 fDataPattern(),
112 fGridOutputDir(),
113 fOutputArchive(),
114 fOutputFiles(),
115 fInputFormat(),
e7c71df0 116 fDatasetName(),
c57f56b7 117 fJDLName(),
d3b18c4c 118 fTerminateFiles(),
bb885a9e 119 fMergeExcludes(),
37d21c01 120 fRegisterExcludes(),
f965131e 121 fIncludePath(),
bb885a9e 122 fCloseSE(),
0df6ccf2 123 fFriendChainName(),
c6cb3634 124 fJobTag(),
648174cf 125 fOutputSingle(),
5fce53f4 126 fRunPrefix(),
3bdcb562 127 fProofCluster(),
128 fProofDataSet(),
129 fFileForTestMode(),
130 fRootVersionForProof(),
131 fAliRootMode(),
f47d5cb4 132 fMergeDirName(),
4e5c5506 133 fInputFiles(0),
28ce6c9c 134 fPackages(0),
77f34eae 135 fModules(0),
28ce6c9c 136 fProofParam()
c57f56b7 137{
138// Dummy ctor.
139 SetDefaults();
140}
141
142//______________________________________________________________________________
143AliAnalysisAlien::AliAnalysisAlien(const char *name)
144 :AliAnalysisGrid(name),
145 fGridJDL(NULL),
0f389141 146 fMergingJDL(NULL),
c57f56b7 147 fPrice(0),
148 fTTL(0),
149 fSplitMaxInputFileNumber(0),
150 fMaxInitFailed(0),
151 fMasterResubmitThreshold(0),
bb885a9e 152 fNtestFiles(0),
319593fb 153 fNrunsPerMaster(0),
16a4353c 154 fMaxMergeFiles(0),
b3e07543 155 fMaxMergeStages(0),
d2a409b2 156 fNsubmitted(0),
a3e84053 157 fProductionMode(0),
cd11251e 158 fOutputToRunNo(0),
0f389141 159 fMergeViaJDL(0),
160 fFastReadOption(0),
e1c22e21 161 fOverwriteMode(1),
149d288c 162 fNreplicas(2),
3bdcb562 163 fNproofWorkers(0),
27734f0e 164 fNproofWorkersPerSlave(0),
3bdcb562 165 fProofReset(0),
c57f56b7 166 fRunNumbers(),
167 fExecutable(),
0a1c1f7f 168 fExecutableCommand(),
c57f56b7 169 fArguments(),
631c0b05 170 fExecutableArgs(),
c57f56b7 171 fAnalysisMacro(),
172 fAnalysisSource(),
d3b18c4c 173 fValidationScript(),
d5c6455a 174 fAdditionalRootLibs(),
c57f56b7 175 fAdditionalLibs(),
176 fSplitMode(),
177 fAPIVersion(),
178 fROOTVersion(),
179 fAliROOTVersion(),
648174cf 180 fExternalPackages(),
c57f56b7 181 fUser(),
182 fGridWorkingDir(),
183 fGridDataDir(),
184 fDataPattern(),
185 fGridOutputDir(),
186 fOutputArchive(),
187 fOutputFiles(),
188 fInputFormat(),
e7c71df0 189 fDatasetName(),
c57f56b7 190 fJDLName(),
d3b18c4c 191 fTerminateFiles(),
bb885a9e 192 fMergeExcludes(),
37d21c01 193 fRegisterExcludes(),
f965131e 194 fIncludePath(),
bb885a9e 195 fCloseSE(),
0df6ccf2 196 fFriendChainName(),
c6cb3634 197 fJobTag(),
648174cf 198 fOutputSingle(),
5fce53f4 199 fRunPrefix(),
3bdcb562 200 fProofCluster(),
201 fProofDataSet(),
202 fFileForTestMode(),
203 fRootVersionForProof(),
204 fAliRootMode(),
f47d5cb4 205 fMergeDirName(),
4e5c5506 206 fInputFiles(0),
28ce6c9c 207 fPackages(0),
77f34eae 208 fModules(0),
28ce6c9c 209 fProofParam()
c57f56b7 210{
211// Default ctor.
212 SetDefaults();
213}
214
215//______________________________________________________________________________
216AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
217 :AliAnalysisGrid(other),
218 fGridJDL(NULL),
0f389141 219 fMergingJDL(NULL),
c57f56b7 220 fPrice(other.fPrice),
221 fTTL(other.fTTL),
222 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
223 fMaxInitFailed(other.fMaxInitFailed),
224 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 225 fNtestFiles(other.fNtestFiles),
319593fb 226 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 227 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 228 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 229 fNsubmitted(other.fNsubmitted),
a3e84053 230 fProductionMode(other.fProductionMode),
cd11251e 231 fOutputToRunNo(other.fOutputToRunNo),
0f389141 232 fMergeViaJDL(other.fMergeViaJDL),
233 fFastReadOption(other.fFastReadOption),
234 fOverwriteMode(other.fOverwriteMode),
149d288c 235 fNreplicas(other.fNreplicas),
3bdcb562 236 fNproofWorkers(other.fNproofWorkers),
27734f0e 237 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 238 fProofReset(other.fProofReset),
c57f56b7 239 fRunNumbers(other.fRunNumbers),
240 fExecutable(other.fExecutable),
0a1c1f7f 241 fExecutableCommand(other.fExecutableCommand),
c57f56b7 242 fArguments(other.fArguments),
631c0b05 243 fExecutableArgs(other.fExecutableArgs),
c57f56b7 244 fAnalysisMacro(other.fAnalysisMacro),
245 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 246 fValidationScript(other.fValidationScript),
d5c6455a 247 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 248 fAdditionalLibs(other.fAdditionalLibs),
249 fSplitMode(other.fSplitMode),
250 fAPIVersion(other.fAPIVersion),
251 fROOTVersion(other.fROOTVersion),
252 fAliROOTVersion(other.fAliROOTVersion),
648174cf 253 fExternalPackages(other.fExternalPackages),
c57f56b7 254 fUser(other.fUser),
255 fGridWorkingDir(other.fGridWorkingDir),
256 fGridDataDir(other.fGridDataDir),
257 fDataPattern(other.fDataPattern),
258 fGridOutputDir(other.fGridOutputDir),
259 fOutputArchive(other.fOutputArchive),
260 fOutputFiles(other.fOutputFiles),
261 fInputFormat(other.fInputFormat),
e7c71df0 262 fDatasetName(other.fDatasetName),
c57f56b7 263 fJDLName(other.fJDLName),
d3b18c4c 264 fTerminateFiles(other.fTerminateFiles),
bb885a9e 265 fMergeExcludes(other.fMergeExcludes),
37d21c01 266 fRegisterExcludes(other.fRegisterExcludes),
f965131e 267 fIncludePath(other.fIncludePath),
bb885a9e 268 fCloseSE(other.fCloseSE),
0df6ccf2 269 fFriendChainName(other.fFriendChainName),
c6cb3634 270 fJobTag(other.fJobTag),
648174cf 271 fOutputSingle(other.fOutputSingle),
5fce53f4 272 fRunPrefix(other.fRunPrefix),
3bdcb562 273 fProofCluster(other.fProofCluster),
274 fProofDataSet(other.fProofDataSet),
275 fFileForTestMode(other.fFileForTestMode),
276 fRootVersionForProof(other.fRootVersionForProof),
277 fAliRootMode(other.fAliRootMode),
f47d5cb4 278 fMergeDirName(other.fMergeDirName),
4e5c5506 279 fInputFiles(0),
28ce6c9c 280 fPackages(0),
77f34eae 281 fModules(0),
28ce6c9c 282 fProofParam()
c57f56b7 283{
284// Copy ctor.
285 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 286 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 287 fRunRange[0] = other.fRunRange[0];
288 fRunRange[1] = other.fRunRange[1];
c57f56b7 289 if (other.fInputFiles) {
290 fInputFiles = new TObjArray();
291 TIter next(other.fInputFiles);
292 TObject *obj;
293 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
294 fInputFiles->SetOwner();
295 }
4e5c5506 296 if (other.fPackages) {
297 fPackages = new TObjArray();
298 TIter next(other.fPackages);
299 TObject *obj;
300 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
301 fPackages->SetOwner();
302 }
77f34eae 303 if (other.fModules) {
304 fModules = new TObjArray();
305 fModules->SetOwner();
306 TIter next(other.fModules);
307 AliAnalysisTaskCfg *mod, *crt;
308 while ((crt=(AliAnalysisTaskCfg*)next())) {
309 mod = new AliAnalysisTaskCfg(*crt);
310 fModules->Add(mod);
311 }
312 }
c57f56b7 313}
314
315//______________________________________________________________________________
316AliAnalysisAlien::~AliAnalysisAlien()
317{
318// Destructor.
77f34eae 319 delete fGridJDL;
320 delete fMergingJDL;
321 delete fInputFiles;
322 delete fPackages;
323 delete fModules;
28ce6c9c 324 fProofParam.DeleteAll();
c57f56b7 325}
326
327//______________________________________________________________________________
328AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
329{
330// Assignment.
331 if (this != &other) {
332 AliAnalysisGrid::operator=(other);
333 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 334 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 335 fPrice = other.fPrice;
336 fTTL = other.fTTL;
337 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
338 fMaxInitFailed = other.fMaxInitFailed;
339 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 340 fNtestFiles = other.fNtestFiles;
a3e84053 341 fNrunsPerMaster = other.fNrunsPerMaster;
342 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 343 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 344 fNsubmitted = other.fNsubmitted;
345 fProductionMode = other.fProductionMode;
cd11251e 346 fOutputToRunNo = other.fOutputToRunNo;
0f389141 347 fMergeViaJDL = other.fMergeViaJDL;
348 fFastReadOption = other.fFastReadOption;
349 fOverwriteMode = other.fOverwriteMode;
149d288c 350 fNreplicas = other.fNreplicas;
3bdcb562 351 fNproofWorkers = other.fNproofWorkers;
27734f0e 352 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 353 fProofReset = other.fProofReset;
c57f56b7 354 fRunNumbers = other.fRunNumbers;
355 fExecutable = other.fExecutable;
0a1c1f7f 356 fExecutableCommand = other.fExecutableCommand;
c57f56b7 357 fArguments = other.fArguments;
631c0b05 358 fExecutableArgs = other.fExecutableArgs;
c57f56b7 359 fAnalysisMacro = other.fAnalysisMacro;
360 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 361 fValidationScript = other.fValidationScript;
d5c6455a 362 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 363 fAdditionalLibs = other.fAdditionalLibs;
364 fSplitMode = other.fSplitMode;
365 fAPIVersion = other.fAPIVersion;
366 fROOTVersion = other.fROOTVersion;
367 fAliROOTVersion = other.fAliROOTVersion;
648174cf 368 fExternalPackages = other.fExternalPackages;
c57f56b7 369 fUser = other.fUser;
370 fGridWorkingDir = other.fGridWorkingDir;
371 fGridDataDir = other.fGridDataDir;
372 fDataPattern = other.fDataPattern;
373 fGridOutputDir = other.fGridOutputDir;
374 fOutputArchive = other.fOutputArchive;
375 fOutputFiles = other.fOutputFiles;
376 fInputFormat = other.fInputFormat;
e7c71df0 377 fDatasetName = other.fDatasetName;
c57f56b7 378 fJDLName = other.fJDLName;
d3b18c4c 379 fTerminateFiles = other.fTerminateFiles;
bb885a9e 380 fMergeExcludes = other.fMergeExcludes;
37d21c01 381 fRegisterExcludes = other.fRegisterExcludes;
f965131e 382 fIncludePath = other.fIncludePath;
bb885a9e 383 fCloseSE = other.fCloseSE;
0df6ccf2 384 fFriendChainName = other.fFriendChainName;
c6cb3634 385 fJobTag = other.fJobTag;
648174cf 386 fOutputSingle = other.fOutputSingle;
5fce53f4 387 fRunPrefix = other.fRunPrefix;
3bdcb562 388 fProofCluster = other.fProofCluster;
389 fProofDataSet = other.fProofDataSet;
390 fFileForTestMode = other.fFileForTestMode;
391 fRootVersionForProof = other.fRootVersionForProof;
392 fAliRootMode = other.fAliRootMode;
f47d5cb4 393 fMergeDirName = other.fMergeDirName;
c57f56b7 394 if (other.fInputFiles) {
395 fInputFiles = new TObjArray();
396 TIter next(other.fInputFiles);
397 TObject *obj;
398 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
399 fInputFiles->SetOwner();
400 }
4e5c5506 401 if (other.fPackages) {
402 fPackages = new TObjArray();
403 TIter next(other.fPackages);
404 TObject *obj;
405 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
406 fPackages->SetOwner();
407 }
77f34eae 408 if (other.fModules) {
409 fModules = new TObjArray();
410 fModules->SetOwner();
411 TIter next(other.fModules);
412 AliAnalysisTaskCfg *mod, *crt;
413 while ((crt=(AliAnalysisTaskCfg*)next())) {
414 mod = new AliAnalysisTaskCfg(*crt);
415 fModules->Add(mod);
416 }
417 }
c57f56b7 418 }
419 return *this;
420}
421
37d21c01 422//______________________________________________________________________________
423void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
424{
425// Add a single additional library to be loaded. Extension must be present.
426 TString lib(name);
427 if (!lib.Contains(".")) {
428 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
429 return;
430 }
431 if (fAdditionalLibs.Contains(name)) {
432 Warning("AddAdditionalLibrary", "Library %s already added.", name);
433 return;
434 }
435 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
436 fAdditionalLibs += lib;
437}
438
77f34eae 439//______________________________________________________________________________
440void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
441{
442// Adding a module. Checks if already existing. Becomes owned by this.
443 if (!module) return;
444 if (GetModule(module->GetName())) {
445 Error("AddModule", "A module having the same name %s already added", module->GetName());
446 return;
447 }
448 if (!fModules) {
449 fModules = new TObjArray();
450 fModules->SetOwner();
451 }
452 fModules->Add(module);
453}
454
455//______________________________________________________________________________
456void AliAnalysisAlien::AddModules(TObjArray *list)
457{
458// Adding a list of modules. Checks if already existing. Becomes owned by this.
459 TIter next(list);
460 AliAnalysisTaskCfg *module;
461 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
462}
463
464//______________________________________________________________________________
465Bool_t AliAnalysisAlien::CheckDependencies()
466{
467// Check if all dependencies are satisfied. Reorder modules if needed.
468 Int_t nmodules = GetNmodules();
469 if (!nmodules) {
470 Warning("CheckDependencies", "No modules added yet to check their dependencies");
471 return kTRUE;
472 }
473 AliAnalysisTaskCfg *mod = 0;
474 AliAnalysisTaskCfg *dep = 0;
475 TString depname;
476 Int_t i, j, k;
477 for (i=0; i<nmodules; i++) {
478 mod = (AliAnalysisTaskCfg*) fModules->At(i);
479 Int_t ndeps = mod->GetNdeps();
480 Int_t istart = i;
481 for (j=0; j<ndeps; j++) {
482 depname = mod->GetDependency(j);
483 dep = GetModule(depname);
484 if (!dep) {
485 Error("CheckDependencies","Dependency %s not added for module %s",
486 depname.Data(), mod->GetName());
487 return kFALSE;
488 }
489 if (dep->NeedsDependency(mod->GetName())) {
490 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
491 mod->GetName(), dep->GetName());
492 return kFALSE;
493 }
494 Int_t idep = fModules->IndexOf(dep);
495 // The dependency task must come first
496 if (idep>i) {
497 // Remove at idep and move all objects below up one slot
498 // down to index i included.
499 fModules->RemoveAt(idep);
3e40fd4c 500 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 501 fModules->AddAt(dep, i++);
502 }
503 //Redo from istart if dependencies were inserted
504 if (i>istart) i=istart-1;
505 }
506 }
507 return kTRUE;
508}
509
3e40fd4c 510//______________________________________________________________________________
511AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
512{
513// Create the analysis manager and optionally execute the macro in filename.
514 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
515 if (mgr) return mgr;
516 mgr = new AliAnalysisManager(name);
517 mgr->SetGridHandler((AliAnalysisGrid*)this);
518 if (strlen(filename)) {
519 TString line = gSystem->ExpandPathName(filename);
520 line.Prepend(".x ");
521 gROOT->ProcessLine(line.Data());
522 }
523 return mgr;
524}
525
77f34eae 526//______________________________________________________________________________
527Int_t AliAnalysisAlien::GetNmodules() const
528{
529// Get number of modules.
530 if (!fModules) return 0;
531 return fModules->GetEntries();
532}
533
534//______________________________________________________________________________
535AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
536{
537// Get a module by name.
538 if (!fModules) return 0;
539 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
540}
541
542//______________________________________________________________________________
543Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
544{
545// Load a given module.
546 if (mod->IsLoaded()) return kTRUE;
547 Int_t ndeps = mod->GetNdeps();
548 TString depname;
549 for (Int_t j=0; j<ndeps; j++) {
550 depname = mod->GetDependency(j);
551 AliAnalysisTaskCfg *dep = GetModule(depname);
552 if (!dep) {
553 Error("LoadModule","Dependency %s not existing for module %s",
554 depname.Data(), mod->GetName());
555 return kFALSE;
556 }
557 if (!LoadModule(dep)) {
558 Error("LoadModule","Dependency %s for module %s could not be loaded",
559 depname.Data(), mod->GetName());
560 return kFALSE;
561 }
562 }
563 // Load libraries for the module
564 if (!mod->CheckLoadLibraries()) {
565 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
566 return kFALSE;
567 }
568 // Execute the macro
569 if (mod->ExecuteMacro()<0) {
570 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
571 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
572 return kFALSE;
573 }
574 // Configure dependencies
575 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
576 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
577 mod->GetConfigMacro()->GetTitle(), mod->GetName());
578 return kFALSE;
579 }
5e8c7c22 580 // Adjust extra libraries
581 Int_t nlibs = mod->GetNlibs();
582 TString lib;
583 for (Int_t i=0; i<nlibs; i++) {
584 lib = mod->GetLibrary(i);
585 if (fAdditionalLibs.Contains(lib)) continue;
586 lib = Form("lib%s.so", lib.Data());
587 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
588 fAdditionalLibs += lib;
589 }
77f34eae 590 return kTRUE;
591}
592
139fe6ba 593//______________________________________________________________________________
594Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
595{
596// Generate the full train.
597 fAdditionalLibs = "";
598 if (!LoadModules()) return kFALSE;
599 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
600 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 601 mgr->RunLocalInit();
139fe6ba 602 mgr->PrintStatus();
603 Int_t productionMode = fProductionMode;
604 SetProductionMode();
605 TString macro = fAnalysisMacro;
606 TString executable = fExecutable;
607 TString validation = fValidationScript;
608 TString execCommand = fExecutableCommand;
609 SetAnalysisMacro(Form("%s.C", name));
610 SetExecutable(Form("%s.sh", name));
611 SetExecutableCommand("aliroot -b -q ");
612 SetValidationScript(Form("%s_validation.sh", name));
613 StartAnalysis();
614 SetProductionMode(productionMode);
615 fAnalysisMacro = macro;
616 fExecutable = executable;
617 fExecutableCommand = execCommand;
618 fValidationScript = validation;
619 return kTRUE;
620}
621
77f34eae 622//______________________________________________________________________________
21ca8e59 623Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 624{
625// Generate test macros for a single module or for the full train.
5e8c7c22 626 fAdditionalLibs = "";
77f34eae 627 if (strlen(modname)) {
21ca8e59 628 if (!CheckDependencies()) return kFALSE;
629 AliAnalysisTaskCfg *mod = GetModule(modname);
630 if (!mod) {
631 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
632 return kFALSE;
633 }
634 if (!LoadModule(mod)) return kFALSE;
635 } else if (!LoadModules()) return kFALSE;
636 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
637 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 638 mgr->RunLocalInit();
21ca8e59 639 mgr->PrintStatus();
640 SetLocalTest(kTRUE);
641 Int_t productionMode = fProductionMode;
642 SetProductionMode();
643 TString macro = fAnalysisMacro;
644 TString executable = fExecutable;
645 TString validation = fValidationScript;
646 TString execCommand = fExecutableCommand;
647 SetAnalysisMacro(Form("%s.C", name));
648 SetExecutable(Form("%s.sh", name));
649 SetExecutableCommand("aliroot -b -q ");
650 SetValidationScript(Form("%s_validation.sh", name));
651 WriteAnalysisFile();
652 WriteAnalysisMacro();
653 WriteExecutable();
654 WriteValidationScript();
655 SetLocalTest(kFALSE);
656 SetProductionMode(productionMode);
657 fAnalysisMacro = macro;
658 fExecutable = executable;
659 fExecutableCommand = execCommand;
660 fValidationScript = validation;
77f34eae 661 return kTRUE;
662}
663
664//______________________________________________________________________________
665Bool_t AliAnalysisAlien::LoadModules()
666{
667// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 668 fAdditionalLibs = "";
21ca8e59 669 Int_t nmodules = GetNmodules();
670 if (!nmodules) {
671 Warning("LoadModules", "No module to be loaded");
672 return kTRUE;
673 }
3e40fd4c 674 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
675 if (!mgr) {
676 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
677 return kFALSE;
678 }
77f34eae 679 if (!CheckDependencies()) return kFALSE;
21ca8e59 680 nmodules = GetNmodules();
77f34eae 681 AliAnalysisTaskCfg *mod;
21ca8e59 682 for (Int_t imod=0; imod<nmodules; imod++) {
683 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 684 if (!LoadModule(mod)) return kFALSE;
685 }
77f34eae 686 return kTRUE;
687}
688
74013535 689//______________________________________________________________________________
690void AliAnalysisAlien::SetRunPrefix(const char *prefix)
691{
692// Set the run number format. Can be a prefix or a format like "%09d"
693 fRunPrefix = prefix;
694 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
695}
696
f965131e 697//______________________________________________________________________________
698void AliAnalysisAlien::AddIncludePath(const char *path)
699{
700// Add include path in the remote analysis macro.
701 TString p(path);
702 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
703 else fIncludePath += Form("-I%s ", path);
704}
705
c57f56b7 706//______________________________________________________________________________
707void AliAnalysisAlien::AddRunNumber(Int_t run)
708{
709// Add a run number to the list of runs to be processed.
710 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 711 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 712}
713
98690895 714//______________________________________________________________________________
715void AliAnalysisAlien::AddRunList(const char* runList)
716{
717// Add several runs into the list of runs; they are expected to be separated by a blank character.
718 TString sList = runList;
719 TObjArray *list = sList.Tokenize(" ");
720 Int_t n = list->GetEntries();
721 for (Int_t i = 0; i < n; i++) {
722 TObjString *os = (TObjString*)list->At(i);
723 AddRunNumber(os->GetString().Atoi());
724 }
725 delete list;
726}
727
ee75cfc3 728//______________________________________________________________________________
729void AliAnalysisAlien::AddRunNumber(const char* run)
730{
731// Add a run number to the list of runs to be processed.
7aa5d539 732 TString runs = run;
733 TObjString *os;
734 TObjArray *arr = runs.Tokenize(" ");
735 TIter next(arr);
736 TString prefix;
737 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
738 while ((os=(TObjString*)next())){
739 if (fRunNumbers.Length()) fRunNumbers += " ";
740 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
741 }
742 delete arr;
ee75cfc3 743}
744
c57f56b7 745//______________________________________________________________________________
746void AliAnalysisAlien::AddDataFile(const char *lfn)
747{
748// Adds a data file to the input to be analysed. The file should be a valid LFN
749// or point to an existing file in the alien workdir.
750 if (!fInputFiles) fInputFiles = new TObjArray();
751 fInputFiles->Add(new TObjString(lfn));
752}
648174cf 753
754//______________________________________________________________________________
755void AliAnalysisAlien::AddExternalPackage(const char *package)
756{
757// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
758 if (fExternalPackages) fExternalPackages += " ";
759 fExternalPackages += package;
760}
761
c57f56b7 762//______________________________________________________________________________
763Bool_t AliAnalysisAlien::Connect()
764{
765// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
766 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 767 if (fProductionMode) return kTRUE;
c57f56b7 768 if (!gGrid) {
769 Info("Connect", "Trying to connect to AliEn ...");
770 TGrid::Connect("alien://");
771 }
772 if (!gGrid || !gGrid->IsConnected()) {
773 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
774 return kFALSE;
775 }
776 fUser = gGrid->GetUser();
777 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
778 return kTRUE;
779}
780
781//______________________________________________________________________________
782void AliAnalysisAlien::CdWork()
783{
784// Check validity of alien workspace. Create directory if possible.
785 if (!Connect()) {
786 Error("CdWork", "Alien connection required");
787 return;
788 }
789 TString homedir = gGrid->GetHomeDirectory();
790 TString workdir = homedir + fGridWorkingDir;
923e2ca5 791 if (DirectoryExists(workdir)) {
792 gGrid->Cd(workdir);
793 return;
794 }
795 // Work directory not existing - create it
796 gGrid->Cd(homedir);
b93f8109 797 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 798 gGrid->Cd(fGridWorkingDir);
d3b18c4c 799 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 800 } else {
d3b18c4c 801 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 802 workdir.Data(), homedir.Data());
803 fGridWorkingDir = "";
804 }
c57f56b7 805}
806
348be253 807//______________________________________________________________________________
808Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
809{
810// Check if file copying is possible.
d3b18c4c 811 if (fProductionMode) return kTRUE;
348be253 812 if (!Connect()) {
813 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
814 return kFALSE;
815 }
d3339be3 816 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
817 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 818 // Check if alien_CLOSE_SE is defined
819 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
820 if (!closeSE.IsNull()) {
821 Info("CheckFileCopy", "Your current close storage is pointing to: \
822 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
823 } else {
824 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
825 }
826 // Check if grid directory exists.
827 if (!DirectoryExists(alienpath)) {
828 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
829 return kFALSE;
830 }
d3339be3 831 TFile f("plugin_test_copy", "RECREATE");
348be253 832 // User may not have write permissions to current directory
833 if (f.IsZombie()) {
834 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
835 gSystem->WorkingDirectory());
836 return kFALSE;
837 }
838 f.Close();
d3339be3 839 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
840 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
841 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 842 \n# 1. Make sure you have write permissions there. If this is the case: \
843 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
844 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
845 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
846 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
847 gSystem->Unlink(f.GetName());
848 return kFALSE;
849 }
850 gSystem->Unlink(f.GetName());
d3339be3 851 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
852 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 853 return kTRUE;
854}
855
c57f56b7 856//______________________________________________________________________________
857Bool_t AliAnalysisAlien::CheckInputData()
858{
859// Check validity of input data. If necessary, create xml files.
d3b18c4c 860 if (fProductionMode) return kTRUE;
a8739e8a 861 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 862 if (!fGridDataDir.Length()) {
863 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
864 return kFALSE;
865 }
9f020952 866 if (fMergeViaJDL) {
867 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
868 return kFALSE;
869 }
d2a409b2 870 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 871 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
872 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 873 return kTRUE;
c57f56b7 874 }
875 // Process declared files
a2f5fc01 876 Bool_t isCollection = kFALSE;
877 Bool_t isXml = kFALSE;
878 Bool_t useTags = kFALSE;
c57f56b7 879 Bool_t checked = kFALSE;
d3b18c4c 880 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 881 TString file;
882 TString workdir = gGrid->GetHomeDirectory();
883 workdir += fGridWorkingDir;
884 if (fInputFiles) {
885 TObjString *objstr;
886 TIter next(fInputFiles);
887 while ((objstr=(TObjString*)next())) {
888 file = workdir;
889 file += "/";
890 file += objstr->GetString();
891 // Store full lfn path
892 if (FileExists(file)) objstr->SetString(file);
893 else {
894 file = objstr->GetName();
895 if (!FileExists(objstr->GetName())) {
896 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
897 objstr->GetName(), workdir.Data());
898 return kFALSE;
899 }
900 }
901 Bool_t iscoll, isxml, usetags;
902 CheckDataType(file, iscoll, isxml, usetags);
903 if (!checked) {
904 checked = kTRUE;
a2f5fc01 905 isCollection = iscoll;
906 isXml = isxml;
907 useTags = usetags;
908 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 909 } else {
a2f5fc01 910 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 911 Error("CheckInputData", "Some conflict was found in the types of inputs");
912 return kFALSE;
913 }
914 }
915 }
916 }
917 // Process requested run numbers
a8739e8a 918 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 919 // Check validity of alien data directory
920 if (!fGridDataDir.Length()) {
921 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
922 return kFALSE;
923 }
923e2ca5 924 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 925 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
926 return kFALSE;
927 }
a2f5fc01 928 if (isCollection) {
c57f56b7 929 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
930 return kFALSE;
931 }
932
a2f5fc01 933 if (checked && !isXml) {
c57f56b7 934 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
935 return kFALSE;
936 }
937 // Check validity of run number(s)
938 TObjArray *arr;
939 TObjString *os;
74013535 940 TString format;
319593fb 941 Int_t nruns = 0;
904f9f5f 942 TString schunk, schunk2;
c57f56b7 943 TString path;
944 if (!checked) {
945 checked = kTRUE;
a2f5fc01 946 useTags = fDataPattern.Contains("tag");
947 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 948 }
a2f5fc01 949 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 950 Error("CheckInputData", "Cannot mix input files using/not using tags");
951 return kFALSE;
952 }
953 if (fRunNumbers.Length()) {
a8739e8a 954 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 955 arr = fRunNumbers.Tokenize(" ");
956 TIter next(arr);
957 while ((os=(TObjString*)next())) {
958 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 959 if (!DirectoryExists(path)) {
960 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 961 continue;
c57f56b7 962 }
963 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
964 TString msg = "\n##### file: ";
965 msg += path;
966 msg += " type: xml_collection;";
a2f5fc01 967 if (useTags) msg += " using_tags: Yes";
c57f56b7 968 else msg += " using_tags: No";
23329835 969 Info("CheckDataType", "%s", msg.Data());
319593fb 970 if (fNrunsPerMaster<2) {
d2a409b2 971 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 972 } else {
973 nruns++;
974 if (((nruns-1)%fNrunsPerMaster) == 0) {
975 schunk = os->GetString();
976 }
977 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
978 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 979 AddDataFile(schunk);
319593fb 980 }
c57f56b7 981 }
982 delete arr;
a8739e8a 983 } else {
984 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
985 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 986 format = Form("%%s/%s ", fRunPrefix.Data());
987 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 988 if (!DirectoryExists(path)) {
a8739e8a 989 continue;
990 }
74013535 991 format = Form("%%s/%s.xml", fRunPrefix.Data());
992 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 993 TString msg = "\n##### file: ";
994 msg += path;
995 msg += " type: xml_collection;";
a2f5fc01 996 if (useTags) msg += " using_tags: Yes";
a8739e8a 997 else msg += " using_tags: No";
23329835 998 Info("CheckDataType", "%s", msg.Data());
319593fb 999 if (fNrunsPerMaster<2) {
74013535 1000 format = Form("%s.xml", fRunPrefix.Data());
1001 AddDataFile(Form(format.Data(),irun));
319593fb 1002 } else {
1003 nruns++;
1004 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1005 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1006 }
74013535 1007 format = Form("_%s.xml", fRunPrefix.Data());
1008 schunk2 = Form(format.Data(), irun);
319593fb 1009 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1010 schunk += schunk2;
d2a409b2 1011 AddDataFile(schunk);
319593fb 1012 }
a8739e8a 1013 }
904f9f5f 1014 if (!fInputFiles) {
1015 schunk += schunk2;
1016 AddDataFile(schunk);
1017 }
c57f56b7 1018 }
1019 return kTRUE;
1020}
1021
21ca8e59 1022//______________________________________________________________________________
43b807f6 1023Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1024{
1025// Copy data from the given grid directory according a pattern and make a local
1026// dataset.
1027 if (!Connect()) {
1028 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
1029 return kFALSE;
1030 }
1031 if (!DirectoryExists(griddir)) {
1032 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
1033 return kFALSE;
1034 }
1035 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1036 printf("Running command: %s\n", command.Data());
1037 TGridResult *res = gGrid->Command(command);
1038 Int_t nfound = res->GetEntries();
1039 if (!nfound) {
1040 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
1041 return kFALSE;
1042 }
1043 printf("... found %d files. Copying locally ...\n", nfound);
1044 // Copy files locally
1045 ofstream out;
1046 out.open(output, ios::out);
1047 TMap *map;
1048 TString turl, dirname, filename, temp;
1049 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1050 gSystem->MakeDirectory(outputdir);
1051 gSystem->ChangeDirectory(outputdir);
21ca8e59 1052 for (Int_t i=0; i<nfound; i++) {
1053 map = (TMap*)res->At(i);
1054 turl = map->GetValue("turl")->GetName();
1055 filename = gSystem->BaseName(turl.Data());
1056 dirname = gSystem->DirName(turl.Data());
1057 dirname = gSystem->BaseName(dirname.Data());
1058 gSystem->MakeDirectory(dirname);
43b807f6 1059
1060 TString source(turl);
1061 TString targetFileName(filename);
1062
1063 if (strlen(archivefile) > 0) {
1064 // TODO here the archive in which the file resides should be determined
1065 // however whereis returns only a guid, and guid2lfn does not work
1066 // Therefore we use the one provided as argument for now
1067 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1068 targetFileName = archivefile;
1069 }
1070 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
1071 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1072 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
21ca8e59 1073 }
1074 }
1075 gSystem->ChangeDirectory(cdir);
1076 delete res;
1077 return kTRUE;
1078}
1079
c57f56b7 1080//______________________________________________________________________________
1081Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1082{
1083// Create dataset for the grid data directory + run number.
3af861ff 1084 const Int_t gMaxEntries = 15000;
d3b18c4c 1085 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1086 if (!Connect()) {
1087 Error("CreateDataset", "Cannot create dataset with no grid connection");
1088 return kFALSE;
1089 }
1090
1091 // Cd workspace
d3b18c4c 1092 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1093 TString workdir = gGrid->GetHomeDirectory();
1094 workdir += fGridWorkingDir;
1095
1096 // Compose the 'find' command arguments
74013535 1097 TString format;
c57f56b7 1098 TString command;
1099 TString options = "-x collection ";
bb885a9e 1100 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1101 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1102 TString conditions = "";
3af861ff 1103 Int_t nstart = 0;
1104 Int_t ncount = 0;
1105 Int_t stage = 0;
c57f56b7 1106 TString file;
1107 TString path;
319593fb 1108 Int_t nruns = 0;
904f9f5f 1109 TString schunk, schunk2;
ab254fd1 1110 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1111 if (!fRunNumbers.Length() && !fRunRange[0]) {
1112 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1113 // Make a single data collection from data directory.
1114 path = fGridDataDir;
923e2ca5 1115 if (!DirectoryExists(path)) {
d2a409b2 1116 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1117 return kFALSE;
1118 }
923e2ca5 1119// CdWork();
d2a409b2 1120 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1121 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1122 while (1) {
1123 ncount = 0;
1124 stage++;
1125 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1126 command = "find ";
1127 command += Form("%s -o %d ",options.Data(), nstart);
1128 command += path;
1129 command += " ";
1130 command += pattern;
1131 command += conditions;
1132 printf("command: %s\n", command.Data());
1133 TGridResult *res = gGrid->Command(command);
1134 if (res) delete res;
1135 // Write standard output to file
1136 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1137 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1138 Bool_t nullFile = kFALSE;
1139 if (!hasGrep) {
1140 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1141 } else {
1142 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1143 if (nullFile) {
1144 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1145 gSystem->Exec("rm -f __tmp*");
1146 return kFALSE;
1147 }
1148 TString line;
1149 ifstream in;
1150 in.open("__tmp__");
1151 in >> line;
1152 in.close();
1153 gSystem->Exec("rm -f __tmp__");
1154 ncount = line.Atoi();
1155 }
1156 }
1157 if (ncount == gMaxEntries) {
1158 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1159 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1160 if (!cbase) cbase = cadd;
1161 else {
1162 cbase->Add(cadd);
1163 delete cadd;
defd7a3a 1164 }
3af861ff 1165 nstart += ncount;
1166 } else {
1167 if (cbase) {
1168 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1169 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1170 cbase->Add(cadd);
1171 delete cadd;
1172 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1173 delete cbase; cbase = 0;
1174 } else {
1175 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1176 }
1177 gSystem->Exec("rm -f __tmp*");
1178 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1179 break;
1180 }
0f389141 1181 }
1182 Bool_t fileExists = FileExists(file);
1183 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1184 // Copy xml file to alien space
0f389141 1185 if (fileExists) gGrid->Rm(file);
d2a409b2 1186 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1187 if (!FileExists(file)) {
1188 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1189 return kFALSE;
1190 }
1191 // Update list of files to be processed.
1192 }
1193 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1194 return kTRUE;
1195 }
c57f56b7 1196 // Several runs
a2f5fc01 1197 Bool_t nullResult = kTRUE;
a8739e8a 1198 if (fRunNumbers.Length()) {
1199 TObjArray *arr = fRunNumbers.Tokenize(" ");
1200 TObjString *os;
1201 TIter next(arr);
1202 while ((os=(TObjString*)next())) {
3af861ff 1203 nstart = 0;
1204 stage = 0;
dc030a24 1205 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1206 if (!DirectoryExists(path)) continue;
1207// CdWork();
a8739e8a 1208 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1209 else file = Form("%s.xml", os->GetString().Data());
319593fb 1210 // If local collection file does not exist, create it via 'find' command.
3af861ff 1211 while (1) {
1212 ncount = 0;
1213 stage++;
1214 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1215 command = "find ";
1216 command += Form("%s -o %d ",options.Data(), nstart);
1217 command += path;
1218 command += pattern;
1219 command += conditions;
1220 TGridResult *res = gGrid->Command(command);
1221 if (res) delete res;
1222 // Write standard output to file
1223 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1224 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1225 Bool_t nullFile = kFALSE;
1226 if (!hasGrep) {
1227 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1228 } else {
1229 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1230 if (nullFile) {
1231 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1232 gSystem->Exec("rm -f __tmp*");
1233 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1234 break;
3af861ff 1235 }
1236 TString line;
1237 ifstream in;
1238 in.open("__tmp__");
1239 in >> line;
1240 in.close();
1241 gSystem->Exec("rm -f __tmp__");
1242 ncount = line.Atoi();
1243 }
1244 nullResult = kFALSE;
1245 }
1246 if (ncount == gMaxEntries) {
1247 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1248 if (fNrunsPerMaster > 1) {
1249 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1250 file.Data(),gMaxEntries);
1251 return kFALSE;
1252 }
1253 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1254 if (!cbase) cbase = cadd;
1255 else {
1256 cbase->Add(cadd);
1257 delete cadd;
defd7a3a 1258 }
3af861ff 1259 nstart += ncount;
1260 } else {
c89a84bc 1261 if (cbase && fNrunsPerMaster<2) {
3af861ff 1262 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1263 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1264 cbase->Add(cadd);
1265 delete cadd;
1266 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1267 delete cbase; cbase = 0;
1268 } else {
1269 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1270 }
1271 gSystem->Exec("rm -f __tmp*");
1272 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1273 break;
d3339be3 1274 }
3af861ff 1275 }
a8739e8a 1276 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1277 // Check if there is one run per master job.
1278 if (fNrunsPerMaster<2) {
1279 if (FileExists(file)) {
0f389141 1280 if (fOverwriteMode) gGrid->Rm(file);
1281 else {
1282 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1283 continue;
1284 }
319593fb 1285 }
1286 // Copy xml file to alien space
1287 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1288 if (!FileExists(file)) {
1289 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1290 delete arr;
1291 return kFALSE;
1292 }
1293 } else {
1294 nruns++;
1295 if (((nruns-1)%fNrunsPerMaster) == 0) {
1296 schunk = os->GetString();
1297 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1298 } else {
1299 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1300 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1301 cbase->Add(cadd);
1302 delete cadd;
1303 }
1304 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1305 continue;
1306 }
1307 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1308 if (FileExists(schunk)) {
1309 if (fOverwriteMode) gGrid->Rm(file);
1310 else {
1311 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1312 continue;
1313 }
319593fb 1314 }
84fcd93f 1315 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1316 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1317 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1318 if (!FileExists(schunk)) {
1319 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1320 delete arr;
1321 return kFALSE;
1322 }
d3339be3 1323 }
a8739e8a 1324 }
1325 delete arr;
a2f5fc01 1326 if (nullResult) {
d3339be3 1327 Error("CreateDataset", "No valid dataset corresponding to the query!");
1328 return kFALSE;
1329 }
a8739e8a 1330 } else {
1331 // Process a full run range.
1332 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1333 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1334 nstart = 0;
1335 stage = 0;
74013535 1336 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1337 if (!DirectoryExists(path)) continue;
1338// CdWork();
74013535 1339 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1340 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1341 else file = Form(format.Data(), irun);
0f389141 1342 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1343 if (fOverwriteMode) gGrid->Rm(file);
1344 else {
1345 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1346 continue;
1347 }
a8739e8a 1348 }
319593fb 1349 // If local collection file does not exist, create it via 'find' command.
3af861ff 1350 while (1) {
1351 ncount = 0;
1352 stage++;
1353 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1354 command = "find ";
1355 command += Form("%s -o %d ",options.Data(), nstart);
1356 command += path;
1357 command += pattern;
1358 command += conditions;
1359 TGridResult *res = gGrid->Command(command);
1360 if (res) delete res;
1361 // Write standard output to file
1362 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1363 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1364 Bool_t nullFile = kFALSE;
1365 if (!hasGrep) {
1366 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1367 } else {
1368 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1369 if (nullFile) {
1370 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1371 gSystem->Exec("rm -f __tmp*");
1604c9ed 1372 break;
3af861ff 1373 }
1374 TString line;
1375 ifstream in;
1376 in.open("__tmp__");
1377 in >> line;
1378 in.close();
1379 gSystem->Exec("rm -f __tmp__");
1380 ncount = line.Atoi();
1381 }
1382 nullResult = kFALSE;
1383 }
1384 if (ncount == gMaxEntries) {
1385 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1386 if (fNrunsPerMaster > 1) {
1387 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1388 file.Data(),gMaxEntries);
1389 return kFALSE;
1390 }
1391 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1392 if (!cbase) cbase = cadd;
1393 else {
1394 cbase->Add(cadd);
1395 delete cadd;
defd7a3a 1396 }
3af861ff 1397 nstart += ncount;
1398 } else {
c89a84bc 1399 if (cbase && fNrunsPerMaster<2) {
3af861ff 1400 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1401 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1402 cbase->Add(cadd);
1403 delete cadd;
1404 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1405 delete cbase; cbase = 0;
1406 } else {
1407 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1408 }
1409 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1410 break;
d3339be3 1411 }
319593fb 1412 }
a8739e8a 1413 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1414 // Check if there is one run per master job.
1415 if (fNrunsPerMaster<2) {
1416 if (FileExists(file)) {
0f389141 1417 if (fOverwriteMode) gGrid->Rm(file);
1418 else {
1419 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1420 continue;
1421 }
319593fb 1422 }
1423 // Copy xml file to alien space
1424 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1425 if (!FileExists(file)) {
1426 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1427 return kFALSE;
1428 }
1429 } else {
1430 nruns++;
95e5b448 1431 // Check if the collection for the chunk exist locally.
1432 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1433 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1434 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1435 else continue;
1436 }
84fcd93f 1437 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1438 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1439 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1440 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1441 } else {
1442 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1443 cbase->Add(cadd);
1444 delete cadd;
1445 }
74013535 1446 format = Form("%%s_%s.xml", fRunPrefix.Data());
1447 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1448 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1449 continue;
1450 }
904f9f5f 1451 schunk = schunk2;
319593fb 1452 if (FileExists(schunk)) {
0f389141 1453 if (fOverwriteMode) gGrid->Rm(schunk);
1454 else {
1455 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1456 continue;
1457 }
319593fb 1458 }
84fcd93f 1459 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1460 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1461 if (FileExists(schunk)) {
0f389141 1462 if (fOverwriteMode) gGrid->Rm(schunk);
1463 else {
1464 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1465 continue;
1466 }
95e5b448 1467 }
319593fb 1468 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1469 if (!FileExists(schunk)) {
1470 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1471 return kFALSE;
1472 }
1473 }
c57f56b7 1474 }
a2f5fc01 1475 if (nullResult) {
d3339be3 1476 Error("CreateDataset", "No valid dataset corresponding to the query!");
1477 return kFALSE;
1478 }
a8739e8a 1479 }
c57f56b7 1480 return kTRUE;
1481}
1482
1483//______________________________________________________________________________
1484Bool_t AliAnalysisAlien::CreateJDL()
1485{
1486// Generate a JDL file according to current settings. The name of the file is
1487// specified by fJDLName.
1488 Bool_t error = kFALSE;
1489 TObjArray *arr = 0;
1490 Bool_t copy = kTRUE;
d3b18c4c 1491 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1492 Bool_t generate = kTRUE;
1493 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1494 if (!Connect()) {
1495 Error("CreateJDL", "Alien connection required");
1496 return kFALSE;
1497 }
1498 // Check validity of alien workspace
d3b18c4c 1499 TString workdir;
1500 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1501 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1502 workdir += fGridWorkingDir;
1503 if (generate) {
1504 TObjString *os;
1505 if (!fInputFiles) {
1506 Error("CreateJDL()", "Define some input files for your analysis.");
1507 error = kTRUE;
1508 }
1509 // Compose list of input files
1510 // Check if output files were defined
1511 if (!fOutputFiles.Length()) {
1512 Error("CreateJDL", "You must define at least one output file");
1513 error = kTRUE;
1514 }
1515 // Check if an output directory was defined and valid
1516 if (!fGridOutputDir.Length()) {
1517 Error("CreateJDL", "You must define AliEn output directory");
1518 error = kTRUE;
1519 } else {
d3b18c4c 1520 if (!fProductionMode) {
1521 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1522 if (!DirectoryExists(fGridOutputDir)) {
1523 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1524 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1525 } else {
1526 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1527 // error = kTRUE;
1528 }
67a3b254 1529 } else {
1530 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1531 }
1532 gGrid->Cd(workdir);
1533 }
c57f56b7 1534 }
1535 // Exit if any error up to now
1536 if (error) return kFALSE;
1537 // Set JDL fields
0f389141 1538 if (!fUser.IsNull()) {
1539 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1540 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1541 }
1542 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1543 TString mergeExec = fExecutable;
1544 mergeExec.ReplaceAll(".sh", "_merge.sh");
1545 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1546 mergeExec.ReplaceAll(".sh", ".C");
1547 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1548 if (!fArguments.IsNull())
1549 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1550 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1551 else {
1552 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1553 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1554 }
b3e07543 1555
1f0d1ca2 1556 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1557 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1558 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1559 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1560
0f389141 1561 if (fMaxInitFailed > 0) {
c57f56b7 1562 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1563 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1564 }
1565 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1566 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1567 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1568 }
1569 if (!IsOneStageMerging()) {
f47d5cb4 1570 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1571 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1572 }
1573 if (fSplitMode.Length()) {
c57f56b7 1574 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1575 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1576 }
1577 fMergingJDL->SetValue("Split", "\"se\"");
1578 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1579 if (!fAliROOTVersion.IsNull()) {
1580 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1581 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1582 }
1583 if (!fROOTVersion.IsNull()) {
c57f56b7 1584 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1585 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1586 }
1587 if (!fAPIVersion.IsNull()) {
c57f56b7 1588 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1589 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1590 }
648174cf 1591 if (!fExternalPackages.IsNull()) {
1592 arr = fExternalPackages.Tokenize(" ");
1593 TIter next(arr);
1594 while ((os=(TObjString*)next())) {
1595 TString pkgname = os->GetString();
1596 Int_t index = pkgname.Index("::");
1597 TString pkgversion = pkgname(index+2, pkgname.Length());
1598 pkgname.Remove(index);
1599 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1600 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1601 }
1602 delete arr;
1603 }
0f389141 1604 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1605 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1606 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1607 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1608 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1609 TString analysisFile = fExecutable;
1610 analysisFile.ReplaceAll(".sh", ".root");
1611 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1612 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1613 if (fAdditionalLibs.Length()) {
1614 arr = fAdditionalLibs.Tokenize(" ");
1615 TIter next(arr);
1616 while ((os=(TObjString*)next())) {
1617 if (os->GetString().Contains(".so")) continue;
1618 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1619 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1620 }
1621 delete arr;
1622 }
4e5c5506 1623 if (fPackages) {
1624 TIter next(fPackages);
1625 TObject *obj;
0f389141 1626 while ((obj=next())) {
4e5c5506 1627 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1628 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1629 }
4e5c5506 1630 }
c57f56b7 1631 if (fOutputArchive.Length()) {
1632 arr = fOutputArchive.Tokenize(" ");
1633 TIter next(arr);
0f389141 1634 Bool_t first = kTRUE;
1635 const char *comment = "Files to be archived";
1636 const char *comment1 = comment;
1637 while ((os=(TObjString*)next())) {
1638 if (!first) comment = NULL;
1639 if (!os->GetString().Contains("@") && fCloseSE.Length())
1640 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1641 else
1642 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1643 first = kFALSE;
1644 }
c57f56b7 1645 delete arr;
f790bc1b 1646 // Output archive for the merging jdl
1647 TString outputArchive;
1648 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1649 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1650 // Add normal output files, extra files + terminate files
1651 TString files = GetListOfFiles("outextter");
37d21c01 1652 // Do not register files in fRegisterExcludes
1653 if (!fRegisterExcludes.IsNull()) {
1654 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1655 TIter next1(arr);
1656 while ((os=(TObjString*)next1())) {
1657 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1658 files.ReplaceAll(os->GetString(),"");
1659 }
66d71516 1660 delete arr;
0f389141 1661 }
f790bc1b 1662 files.ReplaceAll(".root", "*.root");
b83f84f0 1663 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1664 } else {
95f84779 1665 TString files = fOutputArchive;
1666 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1667 outputArchive = files;
f790bc1b 1668 }
0f389141 1669 arr = outputArchive.Tokenize(" ");
705adb3e 1670 TIter next2(arr);
0f389141 1671 comment = comment1;
1672 first = kTRUE;
705adb3e 1673 while ((os=(TObjString*)next2())) {
0f389141 1674 if (!first) comment = NULL;
0cdf65a8 1675 TString currentfile = os->GetString();
0cdf65a8 1676 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1677 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1678 else
0cdf65a8 1679 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1680 first = kFALSE;
1681 }
1682 delete arr;
c57f56b7 1683 }
149d288c 1684 arr = fOutputFiles.Tokenize(",");
c57f56b7 1685 TIter next(arr);
0f389141 1686 Bool_t first = kTRUE;
f790bc1b 1687 const char *comment = "Files to be saved";
43da816a 1688 while ((os=(TObjString*)next())) {
e1eaf596 1689 // Ignore ouputs in jdl that are also in outputarchive
1690 TString sout = os->GetString();
f790bc1b 1691 sout.ReplaceAll("*", "");
1692 sout.ReplaceAll(".root", "");
e1eaf596 1693 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1694 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1695 // Ignore fRegisterExcludes
1696 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1697 if (!first) comment = NULL;
43da816a 1698 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1699 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1700 else
0f389141 1701 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1702 first = kFALSE;
f790bc1b 1703 if (fMergeExcludes.Contains(sout)) continue;
1704 if (!os->GetString().Contains("@") && fCloseSE.Length())
1705 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1706 else
1707 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1708 }
c57f56b7 1709 delete arr;
0f389141 1710 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1711 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1712 TString validationScript = fValidationScript;
0f389141 1713 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1714 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1715 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1716 if (fMasterResubmitThreshold) {
1717 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1718 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1719 }
d2a409b2 1720 // Write a jdl with 2 input parameters: collection name and output dir name.
1721 WriteJDL(copy);
c57f56b7 1722 }
1723 // Copy jdl to grid workspace
a8739e8a 1724 if (copy) {
b5e4aaa7 1725 // Check if an output directory was defined and valid
1726 if (!fGridOutputDir.Length()) {
1727 Error("CreateJDL", "You must define AliEn output directory");
1728 return kFALSE;
1729 } else {
1730 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1731 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1732 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1733 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1734 } else {
1735 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1736 return kFALSE;
1737 }
1738 }
1739 gGrid->Cd(workdir);
1740 }
648174cf 1741 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1742 TString mergeJDLName = fExecutable;
1743 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1744 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1745 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1746 if (fProductionMode) {
648174cf 1747 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1748 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1749 }
648174cf 1750 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1751 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1752 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1753 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1754 Fatal("","Terminating");
1755// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1756 if (fMergeViaJDL) {
1757 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1758// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1759 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1760 Fatal("","Terminating");
0f389141 1761 }
648174cf 1762 }
c57f56b7 1763 if (fAdditionalLibs.Length()) {
e7c71df0 1764 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1765 TObjString *os;
1766 TIter next(arr);
1767 while ((os=(TObjString*)next())) {
c57f56b7 1768 if (os->GetString().Contains(".so")) continue;
4e5c5506 1769 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1770 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1771// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1772 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1773 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1774 Fatal("","Terminating");
c57f56b7 1775 }
1776 delete arr;
1777 }
4e5c5506 1778 if (fPackages) {
1779 TIter next(fPackages);
1780 TObject *obj;
1781 while ((obj=next())) {
fdbbc7be 1782 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1783 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1784// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1785 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1786 Form("%s/%s", workdir.Data(), obj->GetName())))
1787 Fatal("","Terminating");
4e5c5506 1788 }
1789 }
c57f56b7 1790 }
1791 return kTRUE;
1792}
1793
a8739e8a 1794//______________________________________________________________________________
d2a409b2 1795Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1796{
1797// Writes one or more JDL's corresponding to findex. If findex is negative,
1798// all run numbers are considered in one go (jdl). For non-negative indices
1799// they correspond to the indices in the array fInputFiles.
1800 if (!fInputFiles) return kFALSE;
d3b18c4c 1801 TObject *os;
1802 TString workdir;
1803 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1804 workdir += fGridWorkingDir;
f47d5cb4 1805 TString stageName = "$2";
1806 if (fProductionMode) stageName = "$4";
1807 if (!fMergeDirName.IsNull()) {
1808 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1809 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1810 } else {
1811 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1812 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1813 }
d3b18c4c 1814 if (fProductionMode) {
a8739e8a 1815 TIter next(fInputFiles);
b3e07543 1816 while ((os=next())) {
d3b18c4c 1817 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1818 }
56754bf6 1819 if (!fOutputToRunNo)
1820 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1821 else
1822 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1823 } else {
1824 if (!fRunNumbers.Length() && !fRunRange[0]) {
1825 // One jdl with no parameters in case input data is specified by name.
1826 TIter next(fInputFiles);
1827 while ((os=next()))
1828 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1829 if (!fOutputSingle.IsNull())
1830 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1831 else {
1832 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1833 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1834 }
1835 } else {
1836 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1837 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1838 if (!fOutputSingle.IsNull()) {
1839 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1840 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1841 } else {
1842 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1843 }
1844 }
a8739e8a 1845 }
1846
a8739e8a 1847 // Generate the JDL as a string
1848 TString sjdl = fGridJDL->Generate();
0f389141 1849 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1850 // Final merge jdl
1851 if (!fMergeDirName.IsNull()) {
1852 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1853 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1854 } else {
1855 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1856 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1857 }
b3e07543 1858 TString sjdl2 = fMergingJDL->Generate();
1859 Int_t index, index1;
a8739e8a 1860 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1861 sjdl.ReplaceAll("(member", "\n (member");
1862 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1863 sjdl.ReplaceAll("{", "{\n ");
1864 sjdl.ReplaceAll("};", "\n};");
1865 sjdl.ReplaceAll("{\n \n", "{\n");
1866 sjdl.ReplaceAll("\n\n", "\n");
1867 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1868 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1869 sjdl1.ReplaceAll("(member", "\n (member");
1870 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1871 sjdl1.ReplaceAll("{", "{\n ");
1872 sjdl1.ReplaceAll("};", "\n};");
1873 sjdl1.ReplaceAll("{\n \n", "{\n");
1874 sjdl1.ReplaceAll("\n\n", "\n");
1875 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1876 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1877 sjdl2.ReplaceAll("(member", "\n (member");
1878 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1879 sjdl2.ReplaceAll("{", "{\n ");
1880 sjdl2.ReplaceAll("};", "\n};");
1881 sjdl2.ReplaceAll("{\n \n", "{\n");
1882 sjdl2.ReplaceAll("\n\n", "\n");
1883 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1884 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1885 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1886 index = sjdl.Index("JDLVariables");
1887 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1888 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1889 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1890 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1891 index = fJobTag.Index(":");
1892 if (index < 0) index = fJobTag.Length();
1893 TString jobTag = fJobTag;
f47d5cb4 1894 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1895 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1896 if (fProductionMode) {
1897 sjdl1.Prepend("# Generated merging jdl (production mode) \
1898 \n# $1 = full alien path to output directory to be merged \
1899 \n# $2 = train number \
1900 \n# $3 = production (like LHC10b) \
1901 \n# $4 = merging stage \
1902 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1903 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1904 sjdl2.Prepend("# Generated merging jdl \
1905 \n# $1 = full alien path to output directory to be merged \
1906 \n# $2 = train number \
1907 \n# $3 = production (like LHC10b) \
1908 \n# $4 = merging stage \
1909 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1910 } else {
1911 sjdl1.Prepend("# Generated merging jdl \
1912 \n# $1 = full alien path to output directory to be merged \
1913 \n# $2 = merging stage \
03335324 1914 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1915 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1916 sjdl2.Prepend("# Generated merging jdl \
1917 \n# $1 = full alien path to output directory to be merged \
1918 \n# $2 = merging stage \
03335324 1919 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1920 }
0f389141 1921 index = sjdl1.Index("JDLVariables");
1922 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1923 index = sjdl2.Index("JDLVariables");
1924 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1925 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1926 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1927 index = sjdl2.Index("Split =");
1928 if (index>=0) {
1929 index1 = sjdl2.Index("\n", index);
1930 sjdl2.Remove(index, index1-index+1);
1931 }
1932 index = sjdl2.Index("SplitMaxInputFileNumber");
1933 if (index>=0) {
1934 index1 = sjdl2.Index("\n", index);
1935 sjdl2.Remove(index, index1-index+1);
1936 }
1937 index = sjdl2.Index("InputDataCollection");
1938 if (index>=0) {
1939 index1 = sjdl2.Index(";", index);
1940 sjdl2.Remove(index, index1-index+1);
1941 }
1942 index = sjdl2.Index("InputDataListFormat");
1943 if (index>=0) {
1944 index1 = sjdl2.Index("\n", index);
1945 sjdl2.Remove(index, index1-index+1);
1946 }
1947 index = sjdl2.Index("InputDataList");
1948 if (index>=0) {
1949 index1 = sjdl2.Index("\n", index);
1950 sjdl2.Remove(index, index1-index+1);
1951 }
f47d5cb4 1952 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1953 // Write jdl to file
a8739e8a 1954 ofstream out;
d2a409b2 1955 out.open(fJDLName.Data(), ios::out);
a8739e8a 1956 if (out.bad()) {
d3b18c4c 1957 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1958 return kFALSE;
1959 }
1960 out << sjdl << endl;
b3e07543 1961 out.close();
0f389141 1962 TString mergeJDLName = fExecutable;
1963 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1964 if (fMergeViaJDL) {
1965 ofstream out1;
1966 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1967 if (out1.bad()) {
d3b18c4c 1968 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1969 return kFALSE;
1970 }
1971 out1 << sjdl1 << endl;
b3e07543 1972 out1.close();
1973 ofstream out2;
1974 TString finalJDL = mergeJDLName;
1975 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1976 out2.open(finalJDL.Data(), ios::out);
1977 if (out2.bad()) {
1978 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1979 return kFALSE;
1980 }
1981 out2 << sjdl2 << endl;
1982 out2.close();
0f389141 1983 }
a8739e8a 1984
1985 // Copy jdl to grid workspace
1986 if (!copy) {
d3b18c4c 1987 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1988 } else {
d2a409b2 1989 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1990 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1991 TString finalJDL = mergeJDLName;
1992 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1993 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1994 if (fProductionMode) {
b5fe9cba 1995 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1996 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1997 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1998 }
d2a409b2 1999 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2000 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2001 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2002 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2003// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2004 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2005 Fatal("","Terminating");
0f389141 2006 if (fMergeViaJDL) {
b3e07543 2007 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2008// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2009// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2010 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2011 Fatal("","Terminating");
2012 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2013 Fatal("","Terminating");
0f389141 2014 }
a8739e8a 2015 }
2016 return kTRUE;
2017}
2018
c57f56b7 2019//______________________________________________________________________________
5513444a 2020Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2021{
2022// Returns true if file exists.
5513444a 2023 if (!gGrid) return kFALSE;
2ea55496 2024 TString slfn = lfn;
2025 slfn.ReplaceAll("alien://","");
2026 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2027 if (!res) return kFALSE;
2028 TMap *map = dynamic_cast<TMap*>(res->At(0));
2029 if (!map) {
2030 delete res;
2031 return kFALSE;
2032 }
2033 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2034 if (!objs || !objs->GetString().Length()) {
2035 delete res;
2036 return kFALSE;
2037 }
2038 delete res;
2039 return kTRUE;
2040}
2041
923e2ca5 2042//______________________________________________________________________________
2043Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2044{
2045// Returns true if directory exists. Can be also a path.
2046 if (!gGrid) return kFALSE;
2047 // Check if dirname is a path
2048 TString dirstripped = dirname;
2049 dirstripped = dirstripped.Strip();
2050 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2051 TString dir = gSystem->BaseName(dirstripped);
2052 dir += "/";
2053 TString path = gSystem->DirName(dirstripped);
2054 TGridResult *res = gGrid->Ls(path, "-F");
2055 if (!res) return kFALSE;
2056 TIter next(res);
2057 TMap *map;
2058 TObject *obj;
2059 while ((map=dynamic_cast<TMap*>(next()))) {
2060 obj = map->GetValue("name");
2061 if (!obj) break;
2062 if (dir == obj->GetName()) {
2063 delete res;
2064 return kTRUE;
2065 }
2066 }
2067 delete res;
2068 return kFALSE;
2069}
2070
c57f56b7 2071//______________________________________________________________________________
a2f5fc01 2072void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2073{
2074// Check input data type.
a2f5fc01 2075 isCollection = kFALSE;
2076 isXml = kFALSE;
2077 useTags = kFALSE;
c57f56b7 2078 if (!gGrid) {
2079 Error("CheckDataType", "No connection to grid");
2080 return;
2081 }
a2f5fc01 2082 isCollection = IsCollection(lfn);
c57f56b7 2083 TString msg = "\n##### file: ";
2084 msg += lfn;
a2f5fc01 2085 if (isCollection) {
c57f56b7 2086 msg += " type: raw_collection;";
2087 // special treatment for collections
a2f5fc01 2088 isXml = kFALSE;
c57f56b7 2089 // check for tag files in the collection
2090 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2091 if (!res) {
2092 msg += " using_tags: No (unknown)";
23329835 2093 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2094 return;
2095 }
2096 const char* typeStr = res->GetKey(0, "origLFN");
2097 if (!typeStr || !strlen(typeStr)) {
2098 msg += " using_tags: No (unknown)";
23329835 2099 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2100 return;
2101 }
2102 TString file = typeStr;
a2f5fc01 2103 useTags = file.Contains(".tag");
2104 if (useTags) msg += " using_tags: Yes";
c57f56b7 2105 else msg += " using_tags: No";
23329835 2106 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2107 return;
2108 }
2109 TString slfn(lfn);
2110 slfn.ToLower();
a2f5fc01 2111 isXml = slfn.Contains(".xml");
2112 if (isXml) {
c57f56b7 2113 // Open xml collection and check if there are tag files inside
2114 msg += " type: xml_collection;";
2115 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2116 if (!coll) {
2117 msg += " using_tags: No (unknown)";
23329835 2118 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2119 return;
2120 }
2121 TMap *map = coll->Next();
2122 if (!map) {
2123 msg += " using_tags: No (unknown)";
23329835 2124 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2125 return;
2126 }
2127 map = (TMap*)map->GetValue("");
2128 TString file;
2129 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2130 useTags = file.Contains(".tag");
c57f56b7 2131 delete coll;
a2f5fc01 2132 if (useTags) msg += " using_tags: Yes";
c57f56b7 2133 else msg += " using_tags: No";
23329835 2134 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2135 return;
2136 }
a2f5fc01 2137 useTags = slfn.Contains(".tag");
c57f56b7 2138 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2139 else msg += " type: unknown file;";
a2f5fc01 2140 if (useTags) msg += " using_tags: Yes";
c57f56b7 2141 else msg += " using_tags: No";
23329835 2142 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2143}
2144
4e5c5506 2145//______________________________________________________________________________
2146void AliAnalysisAlien::EnablePackage(const char *package)
2147{
2148// Enables a par file supposed to exist in the current directory.
2149 TString pkg(package);
2150 pkg.ReplaceAll(".par", "");
2151 pkg += ".par";
2152 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2153 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2154 return;
2155 }
2156 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2157 Info("EnablePackage", "AliEn plugin will use .par packages");
2158 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2159 if (!fPackages) {
2160 fPackages = new TObjArray();
2161 fPackages->SetOwner();
2162 }
2163 fPackages->Add(new TObjString(pkg));
2164}
2165
3bdcb562 2166//______________________________________________________________________________
2167TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2168{
2169// Make a tree from files having the location specified in fFileForTestMode.
2170// Inspired from JF's CreateESDChain.
2171 if (fFileForTestMode.IsNull()) {
2172 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2173 return NULL;
2174 }
2175 if (gSystem->AccessPathName(fFileForTestMode)) {
2176 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2177 return NULL;
2178 }
2179 // Open the file
2180 ifstream in;
2181 in.open(fFileForTestMode);
2182 Int_t count = 0;
2183 // Read the input list of files and add them to the chain
b505f1f8 2184 TString line;
2185 TChain *chain = new TChain(treeName);
2186 TChain *chainFriend = 0;
2187 if (!fFriendChainName.IsNull()) chainFriend = new TChain(treeName);
2188 while (in.good())
2189 {
3bdcb562 2190 in >> line;
2191 if (line.IsNull()) continue;
2192 if (count++ == fNtestFiles) break;
2193 TString esdFile(line);
2194 TFile *file = TFile::Open(esdFile);
b505f1f8 2195 if (file && !file->IsZombie()) {
2196 chain->Add(esdFile);
3bdcb562 2197 file->Close();
b505f1f8 2198 if (!fFriendChainName.IsNull()) {
2199 esdFile.ReplaceAll("AliAOD.root", fFriendChainName.Data());
2200 esdFile.ReplaceAll("AliAODs.root", fFriendChainName.Data());
2201 file = TFile::Open(esdFile);
2202 if (file && !file->IsZombie()) {
2203 file->Close();
2204 chainFriend->Add(esdFile);
2205 } else {
2206 Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data());
2207 return 0;
2208 }
2209 }
3bdcb562 2210 } else {
2211 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2212 }
b505f1f8 2213 }
2214 in.close();
2215 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2216 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2217 delete chain;
b505f1f8 2218 delete chainFriend;
3bdcb562 2219 return NULL;
b505f1f8 2220 }
3bdcb562 2221// chain->ls();
b505f1f8 2222 if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend);
2223 return chain;
3bdcb562 2224}
2225
d2a409b2 2226//______________________________________________________________________________
2227const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2228{
2229// Get job status for all jobs with jobid>jobidstart.
2230 static char mstatus[20];
2231 mstatus[0] = '\0';
2232 nrunning = 0;
2233 nwaiting = 0;
2234 nerror = 0;
2235 ndone = 0;
2236 TGridJobStatusList *list = gGrid->Ps("");
2237 if (!list) return mstatus;
2238 Int_t nentries = list->GetSize();
2239 TGridJobStatus *status;
2240 Int_t pid;
2241 for (Int_t ijob=0; ijob<nentries; ijob++) {
2242 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2243 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2244 if (pid<jobidstart) continue;
2245 if (pid == lastid) {
8ddad121 2246 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2247 }
2248 switch (status->GetStatus()) {
2249 case TGridJobStatus::kWAITING:
2250 nwaiting++; break;
2251 case TGridJobStatus::kRUNNING:
2252 nrunning++; break;
2253 case TGridJobStatus::kABORTED:
2254 case TGridJobStatus::kFAIL:
2255 case TGridJobStatus::kUNKNOWN:
2256 nerror++; break;
2257 case TGridJobStatus::kDONE:
2258 ndone++;
2259 }
2260 }
2261 list->Delete();
2262 delete list;
2263 return mstatus;
2264}
2265
c57f56b7 2266//______________________________________________________________________________
2267Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2268{
2269// Returns true if file is a collection. Functionality duplicated from
2270// TAlien::Type() because we don't want to directly depend on TAlien.
2271 if (!gGrid) {
2272 Error("IsCollection", "No connection to grid");
2273 return kFALSE;
2274 }
2275 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2276 if (!res) return kFALSE;
2277 const char* typeStr = res->GetKey(0, "type");
2278 if (!typeStr || !strlen(typeStr)) return kFALSE;
2279 if (!strcmp(typeStr, "collection")) return kTRUE;
2280 delete res;
2281 return kFALSE;
2282}
2283
fe2d7fc2 2284//______________________________________________________________________________
2285Bool_t AliAnalysisAlien::IsSingleOutput() const
2286{
2287// Check if single-ouput option is on.
2288 return (!fOutputSingle.IsNull());
2289}
2290
16a4353c 2291//______________________________________________________________________________
2292void AliAnalysisAlien::Print(Option_t *) const
2293{
2294// Print current plugin settings.
84fcd93f 2295 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2296 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2297 if (mgr && mgr->IsProofMode()) {
2298 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2299 if (TestBit(AliAnalysisGrid::kTest))
2300 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2301 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2302 if (!fProofDataSet.IsNull())
2303 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2304 if (fProofReset==1)
2305 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2306 if (fProofReset>1)
2307 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2308 if (!fRootVersionForProof.IsNull())
2309 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2310 else
2311 printf("= ROOT version requested________________________ default\n");
2312 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2313 if (!fAliRootMode.IsNull())
2314 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2315 if (fNproofWorkers)
2316 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2317 if (fNproofWorkersPerSlave)
2318 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2319 if (TestSpecialBit(kClearPackages))
2320 printf("= ClearPackages requested...\n");
2321 if (fIncludePath.Data())
2322 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2323 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2324 if (fPackages && fPackages->GetEntries()) {
2325 TIter next(fPackages);
2326 TObject *obj;
2327 TString list;
2328 while ((obj=next())) list += obj->GetName();
2329 printf("= Par files to be used: ________________________ %s\n", list.Data());
2330 }
2331 if (TestSpecialBit(kProofConnectGrid))
2332 printf("= Requested PROOF connection to grid\n");
2333 return;
2334 }
e1c22e21 2335 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2336 if (fOverwriteMode) {
2337 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2338 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2339 }
348be253 2340 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2341 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2342 printf("= Production mode:______________________________ %d\n", fProductionMode);
2343 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2344 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2345 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2346 if (fUser.Length())
84fcd93f 2347 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2348 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2349 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2350 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2351 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2352 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2353 if (fRunNumbers.Length())
84fcd93f 2354 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2355 if (fRunRange[0])
74013535 2356 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2357 if (!fRunRange[0] && !fRunNumbers.Length()) {
2358 TIter next(fInputFiles);
2359 TObject *obj;
2360 TString list;
2361 while ((obj=next())) list += obj->GetName();
84fcd93f 2362 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2363 }
2364 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2365 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2366 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2367 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2368 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2369 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2370 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2371 printf("=====================================================================\n");
2372 printf("= Job price: ___________________________________ %d\n", fPrice);
2373 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2374 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2375 if (fMaxInitFailed>0)
84fcd93f 2376 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2377 if (fMasterResubmitThreshold>0)
84fcd93f 2378 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2379 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2380 if (fNrunsPerMaster>0)
84fcd93f 2381 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2382 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2383 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2384 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2385 if (fArguments.Length())
84fcd93f 2386 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2387 if (fExecutableArgs.Length())
2388 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2389 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2390 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2391 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2392 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2393 if (fDatasetName)
84fcd93f 2394 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2395 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2396 if (fIncludePath.Data())
84fcd93f 2397 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2398 if (fCloseSE.Length())
84fcd93f 2399 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2400 if (fFriendChainName.Length())
84fcd93f 2401 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2402 if (fPackages && fPackages->GetEntries()) {
16a4353c 2403 TIter next(fPackages);
2404 TObject *obj;
2405 TString list;
2406 while ((obj=next())) list += obj->GetName();
84fcd93f 2407 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2408 }
2409}
2410
c57f56b7 2411//______________________________________________________________________________
2412void AliAnalysisAlien::SetDefaults()
2413{
2414// Set default values for everything. What cannot be filled will be left empty.
2415 if (fGridJDL) delete fGridJDL;
2416 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2417 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2418 fPrice = 1;
2419 fTTL = 30000;
2420 fSplitMaxInputFileNumber = 100;
2421 fMaxInitFailed = 0;
2422 fMasterResubmitThreshold = 0;
bb885a9e 2423 fNtestFiles = 10;
149d288c 2424 fNreplicas = 2;
a8739e8a 2425 fRunRange[0] = 0;
2426 fRunRange[1] = 0;
8e3bd06a 2427 fRunPrefix = "%d";
319593fb 2428 fNrunsPerMaster = 1;
16a4353c 2429 fMaxMergeFiles = 100;
c57f56b7 2430 fRunNumbers = "";
2431 fExecutable = "analysis.sh";
0a1c1f7f 2432 fExecutableCommand = "root -b -q";
c57f56b7 2433 fArguments = "";
631c0b05 2434 fExecutableArgs = "";
c57f56b7 2435 fAnalysisMacro = "myAnalysis.C";
2436 fAnalysisSource = "";
2437 fAdditionalLibs = "";
2438 fSplitMode = "se";
2439 fAPIVersion = "";
2440 fROOTVersion = "";
2441 fAliROOTVersion = "";
2442 fUser = ""; // Your alien user name
2443 fGridWorkingDir = "";
2444 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2445 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2446 fFriendChainName = "";
c57f56b7 2447 fGridOutputDir = "output";
b83f84f0 2448 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2449 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2450 fInputFormat = "xml-single";
2451 fJDLName = "analysis.jdl";
c6cb3634 2452 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2453 fMergeExcludes = "";
0f389141 2454 fMergeViaJDL = 0;
348be253 2455 SetUseCopy(kTRUE);
2456 SetCheckCopy(kTRUE);
149d288c 2457 SetDefaultOutputs(kTRUE);
e1c22e21 2458 fOverwriteMode = 1;
c57f56b7 2459}
2460
0f389141 2461//______________________________________________________________________________
b3e07543 2462Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2463{
b3e07543 2464// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2465 // First check if the result is already in the output directory.
2466 if (FileExists(Form("%s/%s",aliendir,filename))) {
2467 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2468 return kFALSE;
2469 }
b3e07543 2470 // Now check the last stage done.
2471 Int_t stage = 0;
2472 while (1) {
2473 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2474 stage++;
2475 }
2476 // Next stage of merging
2477 stage++;
2478 TString pattern = "*root_archive.zip";
2479 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2480 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2481 if (res) delete res;
2482 // Write standard output to file
2483 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2484 // Count the number of files inside
2485 ifstream ifile;
2486 ifile.open(Form("Stage_%d.xml",stage));
2487 if (!ifile.good()) {
2488 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2489 return kFALSE;
b3e07543 2490 }
2491 TString line;
2492 Int_t nfiles = 0;
2493 while (!ifile.eof()) {
2494 ifile >> line;
2495 if (line.Contains("/event")) nfiles++;
7c2cd90a 2496 }
b3e07543 2497 ifile.close();
2498 if (!nfiles) {
2499 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2500 return kFALSE;
2501 } else {
2502 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2503 }
b3e07543 2504 // Copy the file in the output directory
2505 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2506// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2507 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2508 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2509 // Check if this is the last stage to be done.
2510 Bool_t laststage = (nfiles<nperchunk);
2511 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2512 if (laststage) {
2513 printf("### Submiting final merging stage %d\n", stage);
2514 TString finalJDL = jdl;
2515 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2516 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2517 Int_t jobId = SubmitSingleJob(query);
2518 if (!jobId) return kFALSE;
2519 } else {
2520 printf("### Submiting merging stage %d\n", stage);
03335324 2521 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2522 Int_t jobId = SubmitSingleJob(query);
2523 if (!jobId) return kFALSE;
7c2cd90a 2524 }
b3e07543 2525 return kTRUE;
2526}
7c2cd90a 2527
105631b5 2528//______________________________________________________________________________
2529AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2530{
2531// Loat the analysis manager from a file.
2532 TFile *file = TFile::Open(fname);
2533 if (!file) {
2534 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2535 return 0;
2536 }
2537 TIter nextkey(file->GetListOfKeys());
2538 AliAnalysisManager *mgr = 0;
2539 TKey *key;
2540 while ((key=(TKey*)nextkey())) {
2541 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2542 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2543 }
2544 if (!mgr)
2545 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2546 return mgr;
2547}
2548
7c2cd90a 2549//______________________________________________________________________________
2550Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2551{
2552// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2553 if (!gGrid) return 0;
2554 printf("=> %s ------> ",query);
2555 TGridResult *res = gGrid->Command(query);
2556 if (!res) return 0;
2557 TString jobId = res->GetKey(0,"jobId");
2558 delete res;
2559 if (jobId.IsNull()) {
2560 printf("submission failed. Reason:\n");
2561 gGrid->Stdout();
2562 gGrid->Stderr();
2563 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2564 return 0;
2565 }
2566 printf(" Job id: %s\n", jobId.Data());
2567 return atoi(jobId);
2568}
2569
2570//______________________________________________________________________________
b3e07543 2571Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2572{
b3e07543 2573// Merge given output files from basedir. Basedir can be an alien output directory
2574// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2575// files in a group (ignored for xml input). Merging can be done in stages:
2576// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2577// stage=1 : works with an xml of all root_archive.zip in the output directory
2578// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2579 TString outputFile = output;
0f389141 2580 TString command;
a2f5fc01 2581 TString outputChunk;
2582 TString previousChunk = "";
b3e07543 2583 TObjArray *listoffiles = new TObjArray();
2584// listoffiles->SetOwner();
a2f5fc01 2585 Int_t countChunk = 0;
2586 Int_t countZero = nmaxmerge;
0f389141 2587 Bool_t merged = kTRUE;
a2f5fc01 2588 Int_t index = outputFile.Index("@");
2589 if (index > 0) outputFile.Remove(index);
7c2cd90a 2590 TString inputFile = outputFile;
b3e07543 2591 TString sbasedir = basedir;
2592 if (sbasedir.Contains(".xml")) {
2593 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2594 nmaxmerge = 9999999;
2595 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2596 if (!coll) {
2597 ::Error("MergeOutput", "Input XML collection empty.");
2598 return kFALSE;
2599 }
2600 // Iterate grid collection
2601 while (coll->Next()) {
2602 TString fname = gSystem->DirName(coll->GetTURL());
2603 fname += "/";
2604 fname += inputFile;
2605 listoffiles->Add(new TNamed(fname.Data(),""));
2606 }
2607 } else {
2608 command = Form("find %s/ *%s", basedir, inputFile.Data());
2609 printf("command: %s\n", command.Data());
2610 TGridResult *res = gGrid->Command(command);
2611 if (!res) {
2612 ::Error("MergeOutput","No result for the find command\n");
2613 delete listoffiles;
2614 return kFALSE;
2615 }
2616 TIter nextmap(res);
2617 TMap *map = 0;
2618 while ((map=(TMap*)nextmap())) {
2619 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2620 if (!objs || !objs->GetString().Length()) {
2621 // Nothing found - skip this output
2622 delete res;
2623 delete listoffiles;
2624 return kFALSE;
2625 }
2626 listoffiles->Add(new TNamed(objs->GetName(),""));
2627 }
2628 delete res;
2629 }
2630 if (!listoffiles->GetEntries()) {
7c2cd90a 2631 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2632 delete listoffiles;
0f389141 2633 return kFALSE;
2634 }
2635
2636 TFileMerger *fm = 0;
b3e07543 2637 TIter next0(listoffiles);
2638 TObjArray *listoffilestmp = new TObjArray();
2639 listoffilestmp->SetOwner();
2640 TObject *nextfile;
2641 TString snextfile;
2642 // Keep only the files at upper level
2643 Int_t countChar = 0;
2644 while ((nextfile=next0())) {
2645 snextfile = nextfile->GetName();
2646 Int_t crtCount = snextfile.CountChar('/');
2647 if (nextfile == listoffiles->First()) countChar = crtCount;
2648 if (crtCount < countChar) countChar = crtCount;
2649 }
2650 next0.Reset();
2651 while ((nextfile=next0())) {
2652 snextfile = nextfile->GetName();
2653 Int_t crtCount = snextfile.CountChar('/');
2654 if (crtCount > countChar) {
2655 delete nextfile;
2656 continue;
2657 }
2658 listoffilestmp->Add(nextfile);
2659 }
2660 delete listoffiles;
2661 listoffiles = listoffilestmp; // Now contains 'good' files
2662 listoffiles->Print();
2663 TIter next(listoffiles);
7c2cd90a 2664 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2665 outputChunk = outputFile;
2666 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2667 // Check for existent temporary merge files
e1c22e21 2668 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2669 // Preserve old merging functionality for stage 0.
2670 if (stage==0) {
2671 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2672 while (1) {
2673 // Skip as many input files as in a chunk
2ea55496 2674 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2675 nextfile = next();
2676 if (!nextfile) {
2ea55496 2677 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2678 delete listoffiles;
2ea55496 2679 return kFALSE;
2680 }
b3e07543 2681 snextfile = nextfile->GetName();
7c2cd90a 2682 }
2683 outputChunk = outputFile;
2684 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2685 countChunk++;
2686 if (gSystem->AccessPathName(outputChunk)) continue;
2687 // Merged file with chunks up to <countChunk> found
2688 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2689 previousChunk = outputChunk;
2690 break;
2691 }
2692 }
2693 countZero = nmaxmerge;
2694
b3e07543 2695 while ((nextfile=next())) {
2696 snextfile = nextfile->GetName();
2ea55496 2697 // Loop 'find' results and get next LFN
7c2cd90a 2698 if (countZero == nmaxmerge) {
2699 // First file in chunk - create file merger and add previous chunk if any.
2b895f34 2700 fm = new TFileMerger(kTRUE);
7c2cd90a 2701 fm->SetFastMethod(kTRUE);
2702 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2703 outputChunk = outputFile;
2704 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2705 }
2706 // If last file found, put merged results in the output file
b3e07543 2707 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2708 // Add file to be merged and decrement chunk counter.
b3e07543 2709 fm->AddFile(snextfile);
7c2cd90a 2710 countZero--;
b3e07543 2711 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2712 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2713 // Nothing found - skip this output
2714 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2715 merged = kFALSE;
2716 break;
7c2cd90a 2717 }
2718 fm->OutputFile(outputChunk);
2719 // Merge the outputs, then go to next chunk
2720 if (!fm->Merge()) {
2721 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2722 merged = kFALSE;
2723 break;
7c2cd90a 2724 } else {
2725 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2726 gSystem->Unlink(previousChunk);
2727 }
b3e07543 2728 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2729 countChunk++;
2730 countZero = nmaxmerge;
2731 previousChunk = outputChunk;
0f389141 2732 }
0f389141 2733 }
b3e07543 2734 delete listoffiles;
2735 delete fm;
7c2cd90a 2736 return merged;
2737 }
2738 // Merging stage different than 0.
2739 // Move to the begining of the requested chunk.
2b895f34 2740 fm = new TFileMerger(kTRUE);
7c2cd90a 2741 fm->SetFastMethod(kTRUE);
b3e07543 2742 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2743 delete listoffiles;
7c2cd90a 2744 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2745 // Nothing found - skip this output
2746 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2747 delete fm;
2748 return kFALSE;
2749 }
b3e07543 2750 fm->OutputFile(outputFile);
7c2cd90a 2751 // Merge the outputs
2752 if (!fm->Merge()) {
2753 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2754 delete fm;
2755 return kFALSE;
2756 } else {
b3e07543 2757 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2758 }
7c2cd90a 2759 delete fm;
2760 return kTRUE;
0f389141 2761}
2762
c57f56b7 2763//______________________________________________________________________________
2764Bool_t AliAnalysisAlien::MergeOutputs()
2765{
2766// Merge analysis outputs existing in the AliEn space.
2767 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2768 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2769 if (!Connect()) {
2770 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2771 return kFALSE;
0f389141 2772 }
c9e8f7fd 2773 if (fMergeViaJDL) {
2774 if (!TestBit(AliAnalysisGrid::kMerge)) {
2775 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2776 return kFALSE;
2777 }
2778 if (fProductionMode) {
2779 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2780 return kFALSE;
2781 }
0f389141 2782 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2783 if (!SubmitMerging()) return kFALSE;
0f389141 2784 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2785 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2786 return kFALSE;
c57f56b7 2787 }
2788 // Get the output path
dc030a24 2789 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2790 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2791 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2792 return kFALSE;
2793 }
2794 if (!fOutputFiles.Length()) {
2795 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2796 return kFALSE;
0f389141 2797 }
2798 // Check if fast read option was requested
e1c22e21 2799 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2800 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2801 if (fFastReadOption) {
d3339be3 2802 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2803 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2804 gEnv->SetValue("XNet.ConnectTimeout",50);
2805 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2806 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2807 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2808 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2809 }
e8b839ab 2810 // Make sure we change the temporary directory
2811 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2812 // Set temporary compilation directory to current one
2813 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2814 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2815 TIter next(list);
2816 TObjString *str;
a2f5fc01 2817 TString outputFile;
c57f56b7 2818 Bool_t merged = kTRUE;
2819 while((str=(TObjString*)next())) {
a2f5fc01 2820 outputFile = str->GetString();
2821 Int_t index = outputFile.Index("@");
2822 if (index > 0) outputFile.Remove(index);
2823 TString outputChunk = outputFile;
2824 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2825 // Skip already merged outputs
a2f5fc01 2826 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2827 if (fOverwriteMode) {
a2f5fc01 2828 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2829 gSystem->Unlink(outputFile);
2830 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2831 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2832 outputChunk.Data());
2833 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2834 }
2835 } else {
a2f5fc01 2836 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2837 continue;
2838 }
2839 } else {
a2f5fc01 2840 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2841 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2842 outputChunk.Data());
2843 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2844 }
2845 }
37d21c01 2846 if (fMergeExcludes.Contains(outputFile.Data()) ||
2847 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 2848 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2849 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2850 if (!merged) {
2851 Error("MergeOutputs", "Terminate() will NOT be executed");
2852 return kFALSE;
ff07ec61 2853 }
2854 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2855 if (fileOpened) fileOpened->Close();
c57f56b7 2856 }
0f389141 2857 return kTRUE;
c57f56b7 2858}
2859
bb885a9e 2860//______________________________________________________________________________
2861void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2862{
2863// Use the output files connected to output containers from the analysis manager
2864// rather than the files defined by SetOutputFiles
2865 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2866 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2867 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2868}
2869
149d288c 2870//______________________________________________________________________________
2871void AliAnalysisAlien::SetOutputFiles(const char *list)
2872{
2873// Manually set the output files list.
2874// Removes duplicates. Not allowed if default outputs are not disabled.
2875 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2876 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2877 return;
2878 }
2879 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2880 fOutputFiles = "";
2881 TString slist = list;
2882 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2883 TObjArray *arr = slist.Tokenize(" ");
2884 TObjString *os;
2885 TIter next(arr);
2886 TString sout;
2887 while ((os=(TObjString*)next())) {
2888 sout = os->GetString();
2889 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2890 if (fOutputFiles.Contains(sout)) continue;
2891 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2892 fOutputFiles += sout;
2893 }
2894 delete arr;
f790bc1b 2895}
149d288c 2896
2897//______________________________________________________________________________
2898void AliAnalysisAlien::SetOutputArchive(const char *list)
2899{
2900// Manually set the output archive list. Free text - you are on your own...
2901// Not allowed if default outputs are not disabled.
2902 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2903 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2904 return;
2905 }
2906 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2907 fOutputArchive = list;
2908}
2909
2910//______________________________________________________________________________
2911void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2912{
2913// Setting a prefered output SE is not allowed anymore.
2914 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2915}
2916
28ce6c9c 2917//______________________________________________________________________________
2918void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
2919{
2920// Set some PROOF special parameter.
2921 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2922 if (pair) {
2923 TObject *old = pair->Key();
2924 TObject *val = pair->Value();
2925 fProofParam.Remove(old);
2926 delete old;
2927 delete val;
2928 }
2929 fProofParam.Add(new TObjString(pname), new TObjString(value));
2930}
2931
2932//______________________________________________________________________________
2933const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
2934{
2935// Returns a special PROOF parameter.
2936 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2937 if (!pair) return 0;
2938 return pair->Value()->GetName();
2939}
2940
5513444a 2941//______________________________________________________________________________
2942Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2943{
2944// Start remote grid analysis.
3bdcb562 2945 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2946 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2947 if (!mgr || !mgr->IsInitialized()) {
2948 Error("StartAnalysis", "You need an initialized analysis manager for this");
2949 return kFALSE;
2950 }
2951 // Are we in PROOF mode ?
2952 if (mgr->IsProofMode()) {
baa34407 2953 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
2954 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2955 if (fProofCluster.IsNull()) {
2956 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2957 return kFALSE;
2958 }
2959 if (fProofDataSet.IsNull() && !testMode) {
2960 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2961 return kFALSE;
2962 }
2963 // Set the needed environment
2964 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2965 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2966 if (fProofReset && !testMode) {
2967 if (fProofReset==1) {
2968 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2969 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2970 } else {
2971 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2972 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2973 }
2974 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2975 return kFALSE;
2976 }
baa34407 2977
2978 if (!testMode) {
2979 // Check if there is an old active session
2980 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
2981 if (nsessions) {
2982 Error("StartAnalysis","You have to reset your old session first\n");
2983 return kFALSE;
2984 }
53911ca6 2985 }
3bdcb562 2986 // Do we need to change the ROOT version ? The success of this cannot be checked.
2987 if (!fRootVersionForProof.IsNull() && !testMode) {
2988 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2989 fProofCluster.Data(), fRootVersionForProof.Data()));
2990 }
2991 // Connect to PROOF and check the status
2992 Long_t proof = 0;
27734f0e 2993 TString sworkers;
2994 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2995 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2996 if (!testMode) {
27734f0e 2997 if (!sworkers.IsNull())
2998 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2999 else
3000 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3001 } else {
3002 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3003 if (!proof) {
3004 Error("StartAnalysis", "Could not start PROOF in test mode");
3005 return kFALSE;
3006 }
3007 }
3008 if (!proof) {
3009 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3010 return kFALSE;
3011 }
27734f0e 3012 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3013 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3014 // Set proof special parameters if any
3015 TIter nextpp(&fProofParam);
3016 TObject *proofparam;
3017 while ((proofparam=nextpp())) {
3018 TString svalue = GetProofParameter(proofparam->GetName());
3019 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3020 }
3bdcb562 3021 // Is dataset existing ?
3022 if (!testMode) {
3023 TString dataset = fProofDataSet;
3024 Int_t index = dataset.Index("#");
3025 if (index>=0) dataset.Remove(index);
830acc4c 3026// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3027// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3028// return kFALSE;
3029// }
3030// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3031 }
3032 // Is ClearPackages() needed ?
3033 if (TestSpecialBit(kClearPackages)) {
3034 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3035 gROOT->ProcessLine("gProof->ClearPackages();");
3036 }
3037 // Is a given aliroot mode requested ?
3038 TList optionsList;
1ac846e9 3039 TString parLibs;
830acc4c 3040 if (!fAliRootMode.IsNull()) {
3bdcb562 3041 TString alirootMode = fAliRootMode;
3042 if (alirootMode == "default") alirootMode = "";
3043 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3044 optionsList.SetOwner();
3045 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3046 // Check the additional libs to be loaded
3047 TString extraLibs;
1ac846e9 3048 Bool_t parMode = kFALSE;
dc1ffbcb 3049 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3050 // Parse the extra libs for .so
3051 if (fAdditionalLibs.Length()) {
3052 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3053 TIter next(list);
3054 TObjString *str;
1ac846e9 3055 while((str=(TObjString*)next())) {
3056 if (str->GetString().Contains(".so")) {
3057 if (parMode) {
3058 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3059 break;
3060 }
3061 TString stmp = str->GetName();
3062 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3063 stmp.ReplaceAll(".so","");
3064 if (!extraLibs.IsNull()) extraLibs += ":";
3065 extraLibs += stmp;
3066 continue;
3067 }
3068 if (str->GetString().Contains(".par")) {
3069 // The first par file found in the list will not allow any further .so
3070 parMode = kTRUE;
3071 if (!parLibs.IsNull()) parLibs += ":";
3072 parLibs += str->GetName();
3073 continue;
3074 }
3bdcb562 3075 }
3076 if (list) delete list;
3077 }
dc030a24 3078 if (!extraLibs.IsNull()) {
3079 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3080 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3081 }
830acc4c 3082 // Check extra includes
3083 if (!fIncludePath.IsNull()) {
3084 TString includePath = fIncludePath;
3085 includePath.ReplaceAll(" ",":");
dc030a24 3086 includePath.ReplaceAll("$ALICE_ROOT/","");
3087 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3088 includePath.ReplaceAll("-I","");
dc030a24 3089 includePath.Remove(TString::kTrailing, ':');
830acc4c 3090 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3091 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3092 }
27734f0e 3093 // Check if connection to grid is requested
3094 if (TestSpecialBit(kProofConnectGrid))
3095 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3096 // Enable AliRoot par
3097 if (testMode) {
3098 // Enable proof lite package
3099 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3100 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3101 TNamed *obj = (TNamed*)optionsList.At(i);
3102 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3103 }
3104 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3105 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3106 Info("StartAnalysis", "AliRootProofLite enabled");
3107 } else {
3108 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3109 return kFALSE;
3110 }
3111 } else {
dc030a24 3112 if ( ! fAliROOTVersion.IsNull() ) {
3113 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3114 fAliROOTVersion.Data(), &optionsList))) {
3115 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3116 return kFALSE;
3117 }
3118 }
27734f0e 3119 }
1ac846e9 3120 // Enable first par files from fAdditionalLibs
3121 if (!parLibs.IsNull()) {
3122 TObjArray *list = parLibs.Tokenize(":");
3123 TIter next(list);
3124 TObjString *package;
3125 while((package=(TObjString*)next())) {
40501bb7 3126 TString spkg = package->GetName();
3127 spkg.ReplaceAll(".par", "");
3128 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3129 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3130 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3131 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3132 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3133 return kFALSE;
3134 }
3135 } else {
3136 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3137 return kFALSE;
3138 }
3139 }
3140 if (list) delete list;
3141 }
3bdcb562 3142 } else {
3143 if (fAdditionalLibs.Contains(".so") && !testMode) {
3144 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3145 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3146 return kFALSE;
3147 }
3148 }
3149 // Enable par files if requested
3150 if (fPackages && fPackages->GetEntries()) {
3151 TIter next(fPackages);
3152 TObject *package;
3153 while ((package=next())) {
1ac846e9 3154 // Skip packages already enabled
3155 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3156 TString spkg = package->GetName();
3157 spkg.ReplaceAll(".par", "");
3158 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3159 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3160 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3161 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3162 return kFALSE;
3163 }
3164 } else {
3165 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3166 return kFALSE;
3167 }
3168 }
3169 }
3170 // Do we need to load analysis source files ?
3171 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3172 if (fAnalysisSource.Length()) {
3173 TObjArray *list = fAnalysisSource.Tokenize(" ");
3174 TIter next(list);
3175 TObjString *str;
3176 while((str=(TObjString*)next())) {
3177 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3178 }
3bdcb562 3179 if (list) delete list;
3180 }
830acc4c 3181 if (testMode) {
3182 // Register dataset to proof lite.
3183 if (fFileForTestMode.IsNull()) {
3184 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3185 return kFALSE;
3186 }
3187 if (gSystem->AccessPathName(fFileForTestMode)) {
3188 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3189 return kFALSE;
3190 }
3191 TFileCollection *coll = new TFileCollection();
3192 coll->AddFromFile(fFileForTestMode);
8ddad121 3193 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3194 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3195 }
3bdcb562 3196 return kTRUE;
3197 }
c57f56b7 3198
43da816a 3199 // Check if output files have to be taken from the analysis manager
3200 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3201 // Add output files and AOD files
3202 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3203 // Add extra files registered to the analysis manager
f790bc1b 3204 TString extra = GetListOfFiles("ext");
3205 if (!extra.IsNull()) {
149d288c 3206 extra.ReplaceAll(".root", "*.root");
f790bc1b 3207 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3208 fOutputFiles += extra;
84fcd93f 3209 }
149d288c 3210 // Compose the output archive.
b83f84f0 3211 fOutputArchive = "log_archive.zip:std*@disk=1 ";
3212 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3213 }
f7b1cbc2 3214// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3215 if (TestBit(AliAnalysisGrid::kOffline)) {
3216 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3217 \n there nor any job run. You can revise the JDL and analysis \
3218 \n macro then run the same in \"submit\" mode.");
3219 } else if (TestBit(AliAnalysisGrid::kTest)) {
3220 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3221 \n dataset.");
3222 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3223 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3224 \n space and job submitted.");
3225 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3226 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3227 if (fMergeViaJDL) CheckInputData();
5513444a 3228 return kTRUE;
c57f56b7 3229 } else {
3230 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3231 }
3232
348be253 3233 Print();
c57f56b7 3234 if (!Connect()) {
3235 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3236 return kFALSE;
16a4353c 3237 }
d3b18c4c 3238 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3239 if (!CheckInputData()) {
3240 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3241 return kFALSE;
c57f56b7 3242 }
d3339be3 3243 if (!CreateDataset(fDataPattern)) {
3244 TString serror;
3245 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3246 if (fRunNumbers.Length()) serror = "run numbers";
3247 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3248 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3249 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3250 return kFALSE;
3251 }
21ca8e59 3252 WriteAnalysisFile();
c57f56b7 3253 WriteAnalysisMacro();
3254 WriteExecutable();
3255 WriteValidationScript();
0f389141 3256 if (fMergeViaJDL) {
3257 WriteMergingMacro();
3258 WriteMergeExecutable();
3259 WriteValidationScript(kTRUE);
3260 }
5513444a 3261 if (!CreateJDL()) return kFALSE;
3262 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3263 if (testMode) {
c57f56b7 3264 // Locally testing the analysis
3265 Info("StartAnalysis", "\n_______________________________________________________________________ \
3266 \n Running analysis script in a daughter shell as on a worker node \
3267 \n_______________________________________________________________________");
149d288c 3268 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3269 TIter next(list);
3270 TObjString *str;
a2f5fc01 3271 TString outputFile;
c57f56b7 3272 while((str=(TObjString*)next())) {
a2f5fc01 3273 outputFile = str->GetString();
3274 Int_t index = outputFile.Index("@");
3275 if (index > 0) outputFile.Remove(index);
3276 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3277 }
3278 delete list;
3279 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3280 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3281// gSystem->Exec("cat stdout");
5513444a 3282 return kFALSE;
c57f56b7 3283 }
5513444a 3284 // Check if submitting is managed by LPM manager
a3e84053 3285 if (fProductionMode) {
5513444a 3286 TString prodfile = fJDLName;
3287 prodfile.ReplaceAll(".jdl", ".prod");
3288 WriteProductionFile(prodfile);
3289 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3290 return kFALSE;
3291 }
a8739e8a 3292 // Submit AliEn job(s)
d2a409b2 3293 gGrid->Cd(fGridOutputDir);
a8739e8a 3294 TGridResult *res;
c57f56b7 3295 TString jobID = "";
d2a409b2 3296 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3297 // Submit a given xml or a set of runs
a8739e8a 3298 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3299 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3300 if (res) {
3301 const char *cjobId = res->GetKey(0,"jobId");
3302 if (!cjobId) {
a03be957 3303 gGrid->Stdout();
3304 gGrid->Stderr();
a8739e8a 3305 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3306 return kFALSE;
a8739e8a 3307 } else {
3308 Info("StartAnalysis", "\n_______________________________________________________________________ \
3309 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3310 \n_______________________________________________________________________",
3311 fJDLName.Data(), cjobId);
3312 jobID = cjobId;
3313 }
3314 delete res;
a03be957 3315 } else {
3316 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3317 return kFALSE;
a8739e8a 3318 }
3319 } else {
d2a409b2 3320 // Submit for a range of enumeration of runs.
a03be957 3321 if (!Submit()) return kFALSE;
c57f56b7 3322 }
a8739e8a 3323
c57f56b7 3324 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3325 \n You may exit at any time and terminate the job later using the option <terminate> \
3326 \n ##################################################################################", jobID.Data());
bb885a9e 3327 gSystem->Exec("aliensh");
5513444a 3328 return kTRUE;
c57f56b7 3329}
3330
f790bc1b 3331//______________________________________________________________________________
3332const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3333{
3334// Get a comma-separated list of output files of the requested type.
3335// Type can be (case unsensitive):
3336// aod - list of aod files (std, extensions and filters)
3337// out - list of output files connected to containers (but not aod's or extras)
3338// ext - list of extra files registered to the manager
3339// ter - list of files produced in terminate
3340 static TString files;
3341 files = "";
3342 TString stype = type;
3343 stype.ToLower();
3344 TString aodfiles, extra;
3345 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3346 if (!mgr) {
3347 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3348 return files.Data();
3349 }
3350 if (mgr->GetOutputEventHandler()) {
3351 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3352 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3353 if (!extraaod.IsNull()) {
3354 aodfiles += ",";
3355 aodfiles += extraaod;
3356 }
3357 }
3358 if (stype.Contains("aod")) {
3359 files = aodfiles;
3360 if (stype == "aod") return files.Data();
3361 }
3362 // Add output files that are not in the list of AOD files
3363 TString outputfiles = "";
3364 TIter next(mgr->GetOutputs());
3365 AliAnalysisDataContainer *output;
3366 const char *filename = 0;
3367 while ((output=(AliAnalysisDataContainer*)next())) {
3368 filename = output->GetFileName();
3369 if (!(strcmp(filename, "default"))) continue;
3370 if (outputfiles.Contains(filename)) continue;
3371 if (aodfiles.Contains(filename)) continue;
3372 if (!outputfiles.IsNull()) outputfiles += ",";
3373 outputfiles += filename;
3374 }
3375 if (stype.Contains("out")) {
3376 if (!files.IsNull()) files += ",";
3377 files += outputfiles;
3378 if (stype == "out") return files.Data();
3379 }
3380 // Add extra files registered to the analysis manager
3381 TString sextra;
3382 extra = mgr->GetExtraFiles();
3383 if (!extra.IsNull()) {
3384 extra.Strip();
3385 extra.ReplaceAll(" ", ",");
3386 TObjArray *fextra = extra.Tokenize(",");
3387 TIter nextx(fextra);
3388 TObject *obj;
3389 while ((obj=nextx())) {
3390 if (aodfiles.Contains(obj->GetName())) continue;
3391 if (outputfiles.Contains(obj->GetName())) continue;
3392 if (sextra.Contains(obj->GetName())) continue;
3393 if (!sextra.IsNull()) sextra += ",";
3394 sextra += obj->GetName();
3395 }
3396 delete fextra;
3397 if (stype.Contains("ext")) {
3398 if (!files.IsNull()) files += ",";
3399 files += sextra;
3400 }
3401 }
3402 if (stype == "ext") return files.Data();
3403 TString termfiles;
3404 if (!fTerminateFiles.IsNull()) {
3405 fTerminateFiles.Strip();
3406 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3407 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3408 TIter nextx(fextra);
3409 TObject *obj;
3410 while ((obj=nextx())) {
3411 if (aodfiles.Contains(obj->GetName())) continue;
3412 if (outputfiles.Contains(obj->GetName())) continue;
3413 if (termfiles.Contains(obj->GetName())) continue;
3414 if (sextra.Contains(obj->GetName())) continue;
3415 if (!termfiles.IsNull()) termfiles += ",";
3416 termfiles += obj->GetName();
3417 }
3418 delete fextra;
3419 }
3420 if (stype.Contains("ter")) {
3421 if (!files.IsNull() && !termfiles.IsNull()) {
3422 files += ",";
3423 files += termfiles;
3424 }
3425 }
3426 return files.Data();
3427}
3428
d2a409b2 3429//______________________________________________________________________________
a03be957 3430Bool_t AliAnalysisAlien::Submit()
d2a409b2 3431{
3432// Submit all master jobs.
3433 Int_t nmasterjobs = fInputFiles->GetEntries();
3434 Long_t tshoot = gSystem->Now();
a03be957 3435 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3436 while (fNsubmitted < nmasterjobs) {
3437 Long_t now = gSystem->Now();
3438 if ((now-tshoot)>30000) {
3439 tshoot = now;
a03be957 3440 if (!SubmitNext()) return kFALSE;
d2a409b2 3441 }
3442 }
a03be957 3443 return kTRUE;
d2a409b2 3444}
3445
0f389141 3446//______________________________________________________________________________
a03be957 3447Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3448{
3449// Submit all merging jobs.
dc030a24 3450 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3451 gGrid->Cd(fGridOutputDir);
3452 TString mergeJDLName = fExecutable;
3453 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3454 if (!fInputFiles) {
3455 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3456 return kFALSE;
3457 }
0f389141 3458 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3459 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3460 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3461 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3462 if (fOutputToRunNo) {
3463 // The output directory is the run number
3464 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3465 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3466 } else {
17b4e9a4 3467 if (!fRunNumbers.Length() && !fRunRange[0]) {
3468 // The output directory is the grid outdir
3469 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3470 runOutDir = fGridOutputDir;
3471 } else {
3472 // The output directory is the master number in 3 digits format
3473 printf("### Submitting merging job for master <%03d>\n", i);
3474 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3475 }
7c2cd90a 3476 }
3477 // Check now the number of merging stages.
2b222f7c 3478 TObjArray *list = fOutputFiles.Tokenize(",");
3479 TIter next(list);
3480 TObjString *str;
3481 TString outputFile;
3482 while((str=(TObjString*)next())) {
3483 outputFile = str->GetString();
3484 Int_t index = outputFile.Index("@");
3485 if (index > 0) outputFile.Remove(index);
37d21c01 3486 if (!fMergeExcludes.Contains(outputFile) &&
3487 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3488 }
3489 delete list;
b3e07543 3490 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3491 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3492 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3493 }
a03be957 3494 if (!ntosubmit) return kTRUE;
9f020952 3495 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3496 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3497 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3498 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3499 \n ################################################################################################################");
0f389141 3500 gSystem->Exec("aliensh");
a03be957 3501 return kTRUE;
0f389141 3502}
3503
d2a409b2 3504//______________________________________________________________________________
a03be957 3505Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3506{
08d5b699 3507// Submit next bunch of master jobs if the queue is free. The first master job is
3508// submitted right away, while the next will not be unless the previous was split.
3509// The plugin will not submit new master jobs if there are more that 500 jobs in
3510// waiting phase.
d2a409b2 3511 static Bool_t iscalled = kFALSE;
3512 static Int_t firstmaster = 0;
3513 static Int_t lastmaster = 0;
3514 static Int_t npermaster = 0;
a03be957 3515 if (iscalled) return kTRUE;
d2a409b2 3516 iscalled = kTRUE;
3517 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3518 Int_t ntosubmit = 0;
3519 TGridResult *res;
3520 TString jobID = "";
3bdcb562 3521 Int_t nmasterjobs = fInputFiles->GetEntries();
3522 if (!fNsubmitted) {
3523 ntosubmit = 1;
3524 if (!IsUseSubmitPolicy()) {
90d50a8c 3525 if (nmasterjobs>5)
3bdcb562 3526 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3527 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3528 ntosubmit = nmasterjobs;
3529 }
3530 } else {
d2a409b2 3531 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3532 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3533 // If last master not split, just return
a03be957 3534 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3535 // No more than 100 waiting jobs
08d5b699 3536 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3537 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3538 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3539 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3540 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3541 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3542 }
d2a409b2 3543 for (Int_t i=0; i<ntosubmit; i++) {
3544 // Submit for a range of enumeration of runs.
a03be957 3545 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3546 TString query;
cd11251e 3547 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3548 runOutDir.ReplaceAll(".xml", "");
3549 if (fOutputToRunNo)
3550 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3551 else
3552 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3553 printf("********* %s\n",query.Data());
d2a409b2 3554 res = gGrid->Command(query);
3555 if (res) {
98ca124f 3556 TString cjobId1 = res->GetKey(0,"jobId");
3557 if (!cjobId1.Length()) {
d2a409b2 3558 iscalled = kFALSE;
a03be957 3559 gGrid->Stdout();
3560 gGrid->Stderr();
3561 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3562 return kFALSE;
d2a409b2 3563 } else {
3564 Info("StartAnalysis", "\n_______________________________________________________________________ \
3565 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3566 \n_______________________________________________________________________",
98ca124f 3567 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 3568 jobID += cjobId1;
3569 jobID += " ";
98ca124f 3570 lastmaster = cjobId1.Atoi();
d2a409b2 3571 if (!firstmaster) firstmaster = lastmaster;
3572 fNsubmitted++;
3573 }
3574 delete res;
a03be957 3575 } else {
3576 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3577 return kFALSE;
d2a409b2 3578 }
3579 }
3580 iscalled = kFALSE;
a03be957 3581 return kTRUE;
d2a409b2 3582}
3583
c57f56b7 3584//______________________________________________________________________________
3585void AliAnalysisAlien::WriteAnalysisFile()
3586{
f10e8481 3587// Write current analysis manager into the file <analysisFile>
3588 TString analysisFile = fExecutable;
3589 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3590 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3591 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3592 if (!mgr || !mgr->IsInitialized()) {
3593 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3594 return;
3595 }
3596 // Check analysis type
3597 TObject *handler;
3598 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3599 handler = (TObject*)mgr->GetInputEventHandler();
3600 if (handler) {
65eb22f8 3601 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3602 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3603 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3604 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3605 } else {
3606 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3607 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3608 }
c57f56b7 3609 }
3610 TDirectory *cdir = gDirectory;
f10e8481 3611 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3612 if (file) {
e16a394c 3613 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3614 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3615 // Unless merging makes no sense
3616 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3617 mgr->Write();
3618 delete file;
fe2d7fc2 3619 // Enable termination for local jobs
3620 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3621 }
3622 if (cdir) cdir->cd();
f10e8481 3623 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3624 }
3625 Bool_t copy = kTRUE;
d3b18c4c 3626 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3627 if (copy) {
3628 CdWork();
3629 TString workdir = gGrid->GetHomeDirectory();
3630 workdir += fGridWorkingDir;
d3b18c4c 3631 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3632 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3633 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3634 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3635 }
3636}
3637
3638//______________________________________________________________________________
3639void AliAnalysisAlien::WriteAnalysisMacro()
3640{
3641// Write the analysis macro that will steer the analysis in grid mode.
3642 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3643 ofstream out;
3644 out.open(fAnalysisMacro.Data(), ios::out);
3645 if (!out.good()) {
3646 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3647 return;
3648 }
5b273635 3649 Bool_t hasSTEERBase = kFALSE;
3650 Bool_t hasESD = kFALSE;
3651 Bool_t hasAOD = kFALSE;
3652 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3653 Bool_t hasOADB = kFALSE;
5b273635 3654 Bool_t hasANALYSISalice = kFALSE;
3655 Bool_t hasCORRFW = kFALSE;
c57f56b7 3656 TString func = fAnalysisMacro;
3657 TString type = "ESD";
3658 TString comment = "// Analysis using ";
c85cfc0f 3659 if (IsUseMCchain()) {
3660 type = "MC";
3661 comment += "MC";
3662 } else {
3663 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3664 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3665 type = "AOD";
3666 comment += "AOD";
3667 }
3668 }
0df6ccf2 3669 if (type!="AOD" && fFriendChainName!="") {
3670 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3671 return;
3672 }
c57f56b7 3673 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3674 else comment += " data";
3675 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3676 func.ReplaceAll(".C", "");
3677 out << "void " << func.Data() << "()" << endl;
3678 out << "{" << endl;
3679 out << comment.Data() << endl;
3680 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3681 out << " TStopwatch timer;" << endl;
3682 out << " timer.Start();" << endl << endl;
13f0c00b 3683 // Change temp directory to current one
3684 out << "// Set temporary merging directory to current one" << endl;
3685 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3686 out << "// Set temporary compilation directory to current one" << endl;
3687 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3688 // Reset existing include path
3689 out << "// Reset existing include path and add current directory first in the search" << endl;
3690 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3691 if (!fExecutableCommand.Contains("aliroot")) {
3692 out << "// load base root libraries" << endl;
3693 out << " gSystem->Load(\"libTree\");" << endl;
3694 out << " gSystem->Load(\"libGeom\");" << endl;
3695 out << " gSystem->Load(\"libVMC\");" << endl;
3696 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3697 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3698 }
d5c6455a 3699 if (fAdditionalRootLibs.Length()) {
47a4137d 3700 // in principle libtree /lib geom libvmc etc. can go into this list, too
3701 out << "// Add aditional libraries" << endl;
3702 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3703 TIter next(list);
3704 TObjString *str;
3705 while((str=(TObjString*)next())) {
3706 if (str->GetString().Contains(".so"))
3707 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3708 }
47a4137d 3709 if (list) delete list;
d5c6455a 3710 }
57377eb5 3711 out << "// Load analysis framework libraries" << endl;
652561d5 3712 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3713 if (!fPackages) {
f790bc1b 3714 if (!fExecutableCommand.Contains("aliroot")) {
3715 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3716 out << " gSystem->Load(\"libESD\");" << endl;
3717 out << " gSystem->Load(\"libAOD\");" << endl;
3718 }
4e5c5506 3719 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3720 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3721 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3722 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3723 } else {
4e5c5506 3724 TIter next(fPackages);
3725 TObject *obj;
57377eb5 3726 TString pkgname;
57377eb5 3727 while ((obj=next())) {
3728 pkgname = obj->GetName();
4478e6f1 3729 if (pkgname == "STEERBase" ||
3730 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3731 if (pkgname == "ESD" ||
3732 pkgname == "ESD.par") hasESD = kTRUE;
3733 if (pkgname == "AOD" ||
3734 pkgname == "AOD.par") hasAOD = kTRUE;
3735 if (pkgname == "ANALYSIS" ||
3736 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 3737 if (pkgname == "OADB" ||
3738 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 3739 if (pkgname == "ANALYSISalice" ||
3740 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3741 if (pkgname == "CORRFW" ||
3742 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3743 }
3744 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3745 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3746 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3747 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3748 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3749 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3750 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3751 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3752 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 3753 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
3754 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 3755 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3756 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3757 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3758 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3759 out << "// Compile other par packages" << endl;
3760 next.Reset();
fcc9bb6f 3761 while ((obj=next())) {
3762 pkgname = obj->GetName();
4478e6f1 3763 if (pkgname == "STEERBase" ||
3764 pkgname == "STEERBase.par" ||
3765 pkgname == "ESD" ||
3766 pkgname == "ESD.par" ||
3767 pkgname == "AOD" ||
3768 pkgname == "AOD.par" ||
3769 pkgname == "ANALYSIS" ||
3770 pkgname == "ANALYSIS.par" ||
7ea40ac8 3771 pkgname == "OADB" ||
3772 pkgname == "OADB.par" ||
4478e6f1 3773 pkgname == "ANALYSISalice" ||
3774 pkgname == "ANALYSISalice.par" ||
3775 pkgname == "CORRFW" ||
3776 pkgname == "CORRFW.par") continue;
5b273635 3777 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3778 }
4e5c5506 3779 }
b3e07543 3780 out << "// include path" << endl;
3781 // Get the include path from the interpreter and remove entries pointing to AliRoot
3782 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3783 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3784 out << " TIter nextpath(listpaths);" << endl;
3785 out << " TObjString *pname;" << endl;
3786 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3787 out << " TString current = pname->GetName();" << endl;
3788 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3789 out << " gSystem->AddIncludePath(current);" << endl;
3790 out << " }" << endl;
3791 out << " if (listpaths) delete listpaths;" << endl;
3792 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3793 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3794 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3795 if (fAdditionalLibs.Length()) {
3796 out << "// Add aditional AliRoot libraries" << endl;
3797 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3798 TIter next(list);
3799 TObjString *str;
3800 while((str=(TObjString*)next())) {
3801 if (str->GetString().Contains(".so"))
3802 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3803 if (str->GetString().Contains(".par"))
3804 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3805 }
3806 if (list) delete list;
3807 }
3808 out << endl;
c57f56b7 3809 out << "// analysis source to be compiled at runtime (if any)" << endl;
3810 if (fAnalysisSource.Length()) {
3811 TObjArray *list = fAnalysisSource.Tokenize(" ");
3812 TIter next(list);
3813 TObjString *str;
3814 while((str=(TObjString*)next())) {
3815 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3816 }
3817 if (list) delete list;
3818 }
3819 out << endl;
b3e07543 3820// out << " printf(\"Currently load libraries:\\n\");" << endl;
3821// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3822 if (fFastReadOption) {
d3339be3 3823 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3824 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3825 out << "// fast xrootd reading enabled" << endl;
3826 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3827 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3828 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3829 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3830 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3831 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 3832 }
3833 if (!IsLocalTest()) {
3834 out << "// connect to AliEn and make the chain" << endl;
3835 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
0f389141 3836 }
c57f56b7 3837 out << "// read the analysis manager from file" << endl;
f10e8481 3838 TString analysisFile = fExecutable;
3839 analysisFile.ReplaceAll(".sh", ".root");
105631b5 3840 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
3841 << analysisFile << "\");" << endl;
3842 out << " if (!mgr) return;" << endl;
21ca8e59 3843 if (IsLocalTest()) {
3844 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
3845 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 3846 if (fFileForTestMode.IsNull())
3847 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
3848 else
3849 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 3850 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
21ca8e59 3851 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 3852 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 3853 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 3854 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
3855 } else {
7a366e0b 3856 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 3857 out << " mgr->SetNSysInfo(100);" << endl;
3858 }
21ca8e59 3859 }
c57f56b7 3860 out << " mgr->PrintStatus();" << endl;
52b6a92b 3861 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3862 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3863 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3864 } else {
9a8288ff 3865 if (TestBit(AliAnalysisGrid::kTest))
3866 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3867 else
3868 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3869 }
3870 }
21ca8e59 3871 if (!IsLocalTest()) {
242accb2 3872 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 3873 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
3874 } else {
3875 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 3876 }
f7498086 3877 out << " timer.Stop();" << endl;
3878 out << " timer.Print();" << endl;
c57f56b7 3879 out << "}" << endl << endl;
21ca8e59 3880 if (!IsLocalTest()) {
fcc9bb6f 3881 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3882 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3883 out << "{" << endl;
3884 out << "// Create a chain using url's from xml file" << endl;
242accb2 3885 out << " TString filename;" << endl;
3886 out << " Int_t run = 0;" << endl;
c85cfc0f 3887 if (IsUseMCchain()) {
3888 out << " TString treename = \"TE\";" << endl;
3889 } else {
3890 out << " TString treename = type;" << endl;
3891 out << " treename.ToLower();" << endl;
3892 out << " treename += \"Tree\";" << endl;
3893 }
e02fee64 3894 out << " printf(\"***************************************\\n\");" << endl;
3895 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3896 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3897 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3898 out << " if (!coll) {" << endl;
3899 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3900 out << " return NULL;" << endl;
3901 out << " }" << endl;
242accb2 3902 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3903 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3904 if(fFriendChainName!="") {
3905 out << " TChain *chainFriend = new TChain(treename);" << endl;
3906 }
c57f56b7 3907 out << " coll->Reset();" << endl;
0df6ccf2 3908 out << " while (coll->Next()) {" << endl;
242accb2 3909 out << " filename = coll->GetTURL("");" << endl;
3910 out << " if (mgr) {" << endl;
3911 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3912 out << " if (nrun && nrun != run) {" << endl;
3913 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3914 out << " mgr->SetRunFromPath(nrun);" << endl;
3915 out << " run = nrun;" << endl;
3916 out << " }" << endl;
3917 out << " }" << endl;
3918 out << " chain->Add(filename);" << endl;
0df6ccf2 3919 if(fFriendChainName!="") {
3920 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3921 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3922 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3923 out << " chainFriend->Add(fileFriend.Data());" << endl;
3924 }
3925 out << " }" << endl;
c57f56b7 3926 out << " if (!chain->GetNtrees()) {" << endl;
3927 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3928 out << " return NULL;" << endl;
3929 out << " }" << endl;
0df6ccf2 3930 if(fFriendChainName!="") {
3931 out << " chain->AddFriend(chainFriend);" << endl;
3932 }
c57f56b7 3933 out << " return chain;" << endl;
fcc9bb6f 3934 out << "}" << endl << endl;
c57f56b7 3935 }
5b273635 3936 if (hasANALYSISalice) {
3937 out <<"//________________________________________________________________________________" << endl;
3938 out << "Bool_t SetupPar(const char *package) {" << endl;
3939 out << "// Compile the package and set it up." << endl;
3940 out << " TString pkgdir = package;" << endl;
3941 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3942 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3943 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3944 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3945 out << " // Check for BUILD.sh and execute" << endl;
3946 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3947 out << " printf(\"*******************************\\n\");" << endl;
3948 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3949 out << " printf(\"*******************************\\n\");" << endl;
3950 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3951 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3952 out << " gSystem->ChangeDirectory(cdir);" << endl;
3953 out << " return kFALSE;" << endl;
3954 out << " }" << endl;
3955 out << " } else {" << endl;
3956 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3957 out << " gSystem->ChangeDirectory(cdir);" << endl;
3958 out << " return kFALSE;" << endl;
3959 out << " }" << endl;
3960 out << " // Check for SETUP.C and execute" << endl;
3961 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3962 out << " printf(\"*******************************\\n\");" << endl;
3963 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3964 out << " printf(\"*******************************\\n\");" << endl;
3965 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3966 out << " } else {" << endl;
3967 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3968 out << " gSystem->ChangeDirectory(cdir);" << endl;
3969 out << " return kFALSE;" << endl;
3970 out << " }" << endl;
3971 out << " // Restore original workdir" << endl;
3972 out << " gSystem->ChangeDirectory(cdir);" << endl;
3973 out << " return kTRUE;" << endl;
3974 out << "}" << endl;
3975 }
c57f56b7 3976 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3977 }
3978 Bool_t copy = kTRUE;
d3b18c4c 3979 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3980 if (copy) {
3981 CdWork();
3982 TString workdir = gGrid->GetHomeDirectory();
3983 workdir += fGridWorkingDir;
3984 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 3985 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 3986// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3987 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
3988 Form("alien://%s/%s", workdir.Data(),
3989 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 3990 }
3991}
3992
0f389141 3993//______________________________________________________________________________
3994void AliAnalysisAlien::WriteMergingMacro()
3995{
3996// Write a macro to merge the outputs per master job.
3997 if (!fMergeViaJDL) return;
3998 if (!fOutputFiles.Length()) {
3999 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4000 return;
4001 }
4002 TString mergingMacro = fExecutable;
4003 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4004 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4005 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4006 ofstream out;
4007 out.open(mergingMacro.Data(), ios::out);
4008 if (!out.good()) {
4009 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4010 return;
4011 }
5b273635 4012 Bool_t hasSTEERBase = kFALSE;
4013 Bool_t hasESD = kFALSE;
4014 Bool_t hasAOD = kFALSE;
4015 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4016 Bool_t hasOADB = kFALSE;
5b273635 4017 Bool_t hasANALYSISalice = kFALSE;
4018 Bool_t hasCORRFW = kFALSE;
0f389141 4019 TString func = mergingMacro;
4020 TString comment;
4021 func.ReplaceAll(".C", "");
f47d5cb4 4022 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4023 out << "{" << endl;
4024 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4025 out << " TStopwatch timer;" << endl;
4026 out << " timer.Start();" << endl << endl;
b3e07543 4027 // Reset existing include path
4028 out << "// Reset existing include path and add current directory first in the search" << endl;
4029 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4030 if (!fExecutableCommand.Contains("aliroot")) {
4031 out << "// load base root libraries" << endl;
4032 out << " gSystem->Load(\"libTree\");" << endl;
4033 out << " gSystem->Load(\"libGeom\");" << endl;
4034 out << " gSystem->Load(\"libVMC\");" << endl;
4035 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4036 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4037 }
0f389141 4038 if (fAdditionalRootLibs.Length()) {
4039 // in principle libtree /lib geom libvmc etc. can go into this list, too
4040 out << "// Add aditional libraries" << endl;
4041 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4042 TIter next(list);
4043 TObjString *str;
4044 while((str=(TObjString*)next())) {
4045 if (str->GetString().Contains(".so"))
4046 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4047 }
4048 if (list) delete list;
4049 }
0f389141 4050 out << "// Load analysis framework libraries" << endl;
4051 if (!fPackages) {
7c2cd90a 4052 if (!fExecutableCommand.Contains("aliroot")) {
4053 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4054 out << " gSystem->Load(\"libESD\");" << endl;
4055 out << " gSystem->Load(\"libAOD\");" << endl;
4056 }
0f389141 4057 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4058 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4059 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4060 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4061 } else {
4062 TIter next(fPackages);
4063 TObject *obj;
4064 TString pkgname;
5b273635 4065 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4066 while ((obj=next())) {
4067 pkgname = obj->GetName();
4068 if (pkgname == "STEERBase" ||
4069 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4070 if (pkgname == "ESD" ||
4071 pkgname == "ESD.par") hasESD = kTRUE;
4072 if (pkgname == "AOD" ||
4073 pkgname == "AOD.par") hasAOD = kTRUE;
4074 if (pkgname == "ANALYSIS" ||
4075 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4076 if (pkgname == "OADB" ||
4077 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4078 if (pkgname == "ANALYSISalice" ||
4079 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4080 if (pkgname == "CORRFW" ||
4081 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4082 }
5b273635 4083 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4084 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4085 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4086 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4087 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4088 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4089 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4090 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4091 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4092 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4093 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4094 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4095 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4096 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4097 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4098 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4099 out << "// Compile other par packages" << endl;
4100 next.Reset();
4101 while ((obj=next())) {
4102 pkgname = obj->GetName();
4103 if (pkgname == "STEERBase" ||
4104 pkgname == "STEERBase.par" ||
4105 pkgname == "ESD" ||
4106 pkgname == "ESD.par" ||
4107 pkgname == "AOD" ||
4108 pkgname == "AOD.par" ||
4109 pkgname == "ANALYSIS" ||
4110 pkgname == "ANALYSIS.par" ||
7ea40ac8 4111 pkgname == "OADB" ||
4112 pkgname == "OADB.par" ||
0f389141 4113 pkgname == "ANALYSISalice" ||
4114 pkgname == "ANALYSISalice.par" ||
4115 pkgname == "CORRFW" ||
4116 pkgname == "CORRFW.par") continue;
5b273635 4117 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4118 }
4119 }
b3e07543 4120 out << "// include path" << endl;
4121 // Get the include path from the interpreter and remove entries pointing to AliRoot
4122 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4123 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4124 out << " TIter nextpath(listpaths);" << endl;
4125 out << " TObjString *pname;" << endl;
4126 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4127 out << " TString current = pname->GetName();" << endl;
4128 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4129 out << " gSystem->AddIncludePath(current);" << endl;
4130 out << " }" << endl;
4131 out << " if (listpaths) delete listpaths;" << endl;
4132 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4133 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4134 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4135 if (fAdditionalLibs.Length()) {
4136 out << "// Add aditional AliRoot libraries" << endl;
4137 TObjArray *list = fAdditionalLibs.Tokenize(" ");
4138 TIter next(list);
4139 TObjString *str;
4140 while((str=(TObjString*)next())) {
4141 if (str->GetString().Contains(".so"))
4142 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4143 }
4144 if (list) delete list;
4145 }
4146 out << endl;
4147 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4148 if (fAnalysisSource.Length()) {
4149 TObjArray *list = fAnalysisSource.Tokenize(" ");
4150 TIter next(list);
4151 TObjString *str;
4152 while((str=(TObjString*)next())) {
4153 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4154 }
4155 if (list) delete list;
4156 }
149d288c 4157 out << endl;
4158
0f389141 4159 if (fFastReadOption) {
4160 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4161 out << "// fast xrootd reading enabled" << endl;
4162 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4163 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4164 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4165 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4166 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4167 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4168 }
4169 // Change temp directory to current one
4170 out << "// Set temporary merging directory to current one" << endl;
4171 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4172 out << "// Set temporary compilation directory to current one" << endl;
4173 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
0f389141 4174 out << "// Connect to AliEn" << endl;
4175 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 4176 out << " TString outputDir = dir;" << endl;
f790bc1b 4177 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4178 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4179 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4180 out << " TIter *iter = new TIter(list);" << endl;
4181 out << " TObjString *str;" << endl;
a2f5fc01 4182 out << " TString outputFile;" << endl;
0f389141 4183 out << " Bool_t merged = kTRUE;" << endl;
4184 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4185 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4186 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4187 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4188 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4189 out << " // Skip already merged outputs" << endl;
a2f5fc01 4190 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
4191 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 4192 out << " continue;" << endl;
4193 out << " }" << endl;
a2f5fc01 4194 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4195 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4196 out << " if (!merged) {" << endl;
a2f5fc01 4197 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4198 out << " return;" << endl;
0f389141 4199 out << " }" << endl;
4200 out << " }" << endl;
7c2cd90a 4201 out << " // all outputs merged, validate" << endl;
4202 out << " ofstream out;" << endl;
4203 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4204 out << " out.close();" << endl;
4205 out << " // read the analysis manager from file" << endl;
f866cba5 4206 TString analysisFile = fExecutable;
4207 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 4208 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
105631b5 4209 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4210 << analysisFile << "\");" << endl;
4211 out << " if (!mgr) return;" << endl;
5c4250fc 4212 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4213 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4214 out << " mgr->PrintStatus();" << endl;
4215 if (AliAnalysisManager::GetAnalysisManager()) {
4216 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
4217 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4218 } else {
9a8288ff 4219 if (TestBit(AliAnalysisGrid::kTest))
4220 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4221 else
4222 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4223 }
4224 }
9a8288ff 4225 out << " TTree *tree = NULL;" << endl;
4226 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4227 out << "}" << endl << endl;
5b273635 4228 if (hasANALYSISalice) {
4229 out <<"//________________________________________________________________________________" << endl;
4230 out << "Bool_t SetupPar(const char *package) {" << endl;
4231 out << "// Compile the package and set it up." << endl;
4232 out << " TString pkgdir = package;" << endl;
4233 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4234 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4235 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4236 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4237 out << " // Check for BUILD.sh and execute" << endl;
4238 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4239 out << " printf(\"*******************************\\n\");" << endl;
4240 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4241 out << " printf(\"*******************************\\n\");" << endl;
4242 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4243 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4244 out << " gSystem->ChangeDirectory(cdir);" << endl;
4245 out << " return kFALSE;" << endl;
4246 out << " }" << endl;
4247 out << " } else {" << endl;
4248 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4249 out << " gSystem->ChangeDirectory(cdir);" << endl;
4250 out << " return kFALSE;" << endl;
4251 out << " }" << endl;
4252 out << " // Check for SETUP.C and execute" << endl;
4253 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4254 out << " printf(\"*******************************\\n\");" << endl;
4255 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4256 out << " printf(\"*******************************\\n\");" << endl;
4257 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4258 out << " } else {" << endl;
4259 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4260 out << " gSystem->ChangeDirectory(cdir);" << endl;
4261 out << " return kFALSE;" << endl;
4262 out << " }" << endl;
4263 out << " // Restore original workdir" << endl;
4264 out << " gSystem->ChangeDirectory(cdir);" << endl;
4265 out << " return kTRUE;" << endl;
4266 out << "}" << endl;
4267 }
0f389141 4268 }
4269 Bool_t copy = kTRUE;
d3b18c4c 4270 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4271 if (copy) {
4272 CdWork();
4273 TString workdir = gGrid->GetHomeDirectory();
4274 workdir += fGridWorkingDir;
4275 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4276 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4277// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4278 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4279 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4280 }
4281}
4282
4283//______________________________________________________________________________
4284Bool_t AliAnalysisAlien::SetupPar(const char *package)
4285{
205b201f 4286// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4287// Note that for loading the compiled library. The current directory should have precedence in
4288// LD_LIBRARY_PATH
4289 TString pkgdir = package;
4290 pkgdir.ReplaceAll(".par","");
b50a41c3 4291 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4292 TString cdir = gSystem->WorkingDirectory();
4293 gSystem->ChangeDirectory(pkgdir);
4294 // Check for BUILD.sh and execute
4295 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4296 printf("**************************************************\n");
4297 printf("*** Building PAR archive %s\n", package);
4298 printf("**************************************************\n");
4299 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4300 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4301 gSystem->ChangeDirectory(cdir);
4302 return kFALSE;
4303 }
4304 } else {
4305 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4306 gSystem->ChangeDirectory(cdir);
4307 return kFALSE;
4308 }
4309 // Check for SETUP.C and execute
4310 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4311 printf("**************************************************\n");
4312 printf("*** Setup PAR archive %s\n", package);
4313 printf("**************************************************\n");
4314 gROOT->Macro("PROOF-INF/SETUP.C");
4315 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4316 } else {
4317 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4318 gSystem->ChangeDirectory(cdir);
4319 return kFALSE;
4320 }
4321 // Restore original workdir
4322 gSystem->ChangeDirectory(cdir);
4323 return kTRUE;
4324}
4325
c57f56b7 4326//______________________________________________________________________________
4327void AliAnalysisAlien::WriteExecutable()
4328{
4329// Generate the alien executable script.
4330 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4331 ofstream out;
4332 out.open(fExecutable.Data(), ios::out);
4333 if (out.bad()) {
5513444a 4334 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4335 return;
4336 }
4337 out << "#!/bin/bash" << endl;
2a4d5166 4338 // Make sure we can properly compile par files
4339 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4340 out << "echo \"=========================================\"" << endl;
4341 out << "echo \"############## PATH : ##############\"" << endl;
4342 out << "echo $PATH" << endl;
4343 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4344 out << "echo $LD_LIBRARY_PATH" << endl;
4345 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4346 out << "echo $ROOTSYS" << endl;
4347 out << "echo \"############## which root : ##############\"" << endl;
4348 out << "which root" << endl;
4349 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4350 out << "echo $ALICE_ROOT" << endl;
4351 out << "echo \"############## which aliroot : ##############\"" << endl;
4352 out << "which aliroot" << endl;
9c5ddadc 4353 out << "echo \"############## system limits : ##############\"" << endl;
4354 out << "ulimit -a" << endl;
4355 out << "echo \"############## memory : ##############\"" << endl;
4356 out << "free -m" << endl;
c57f56b7 4357 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4358 out << fExecutableCommand << " ";
370c9880 4359 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4360 out << "RET=$?" << endl;
4361 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4362 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4363 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4364 out << " let sig=\"$RET - 128\""<<endl;
4365 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4366 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4367 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4368 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4369 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4370 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4371 out << " fi"<<endl;
4372 out << " exit $RET"<< endl;
4373 out << "fi" << endl << endl ;
4374 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4375 out << "echo \"############## memory after: ##############\"" << endl;
4376 out << "free -m" << endl;
c57f56b7 4377 }
4378 Bool_t copy = kTRUE;
d3b18c4c 4379 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4380 if (copy) {
4381 CdWork();
4382 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4383 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4384 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4385 workdir += fGridWorkingDir;
4386 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4387 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4388 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4389// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4390 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4391 executable.Data())) Fatal("","Terminating");
c57f56b7 4392 }
4393}
4394
0f389141 4395//______________________________________________________________________________
4396void AliAnalysisAlien::WriteMergeExecutable()
4397{
4398// Generate the alien executable script for the merging job.
4399 if (!fMergeViaJDL) return;
4400 TString mergeExec = fExecutable;
4401 mergeExec.ReplaceAll(".sh", "_merge.sh");
4402 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4403 ofstream out;
4404 out.open(mergeExec.Data(), ios::out);
4405 if (out.bad()) {
4406 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4407 return;
4408 }
4409 out << "#!/bin/bash" << endl;
2a4d5166 4410 // Make sure we can properly compile par files
4411 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4412 out << "echo \"=========================================\"" << endl;
4413 out << "echo \"############## PATH : ##############\"" << endl;
4414 out << "echo $PATH" << endl;
4415 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4416 out << "echo $LD_LIBRARY_PATH" << endl;
4417 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4418 out << "echo $ROOTSYS" << endl;
4419 out << "echo \"############## which root : ##############\"" << endl;
4420 out << "which root" << endl;
4421 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4422 out << "echo $ALICE_ROOT" << endl;
4423 out << "echo \"############## which aliroot : ##############\"" << endl;
4424 out << "which aliroot" << endl;
4425 out << "echo \"############## system limits : ##############\"" << endl;
4426 out << "ulimit -a" << endl;
4427 out << "echo \"############## memory : ##############\"" << endl;
4428 out << "free -m" << endl;
4429 out << "echo \"=========================================\"" << endl << endl;
0f389141 4430 TString mergeMacro = fExecutable;
4431 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4432 if (IsOneStageMerging())
4433 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4434 else
f47d5cb4 4435 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4436 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4437 out << "RET=$?" << endl;
4438 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4439 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4440 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4441 out << " let sig=\"$RET - 128\""<<endl;
4442 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4443 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4444 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4445 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4446 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4447 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4448 out << " fi"<<endl;
4449 out << " exit $RET"<< endl;
4450 out << "fi" << endl << endl ;
0f389141 4451 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4452 out << "echo \"############## memory after: ##############\"" << endl;
4453 out << "free -m" << endl;
0f389141 4454 }
4455 Bool_t copy = kTRUE;
d3b18c4c 4456 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4457 if (copy) {
4458 CdWork();
4459 TString workdir = gGrid->GetHomeDirectory();
4460 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4461 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4462 workdir += fGridWorkingDir;
4463 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4464 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4465 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4466// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4467 if (!copyLocal2Alien("WriteMergeExecutable",
4468 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4469 }
4470}
4471
c57f56b7 4472//______________________________________________________________________________
5513444a 4473void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4474{
4475// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4476// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4477// Next lines: full_path_to_dataset XXX (XXX is a string)
4478// To submit, one has to: submit jdl XXX for all lines
4479 ofstream out;
4480 out.open(filename, ios::out);
4481 if (out.bad()) {
4482 Error("WriteProductionFile", "Bad file name: %s", filename);
4483 return;
4484 }
d3b18c4c 4485 TString workdir;
4486 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4487 workdir = gGrid->GetHomeDirectory();
5513444a 4488 workdir += fGridWorkingDir;
f5e8c702 4489 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4490 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4491 out << locjdl << " " << njobspermaster << endl;
5513444a 4492 Int_t nmasterjobs = fInputFiles->GetEntries();
4493 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4494 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4495 runOutDir.ReplaceAll(".xml", "");
4496 if (fOutputToRunNo)
4497 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4498 else
4499 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4500 }
d3b18c4c 4501 if (gGrid) {
4502 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4503 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4504// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4505 if (!copyLocal2Alien("WriteProductionFile", filename,
4506 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4507 }
5513444a 4508}
4509
4510//______________________________________________________________________________
0f389141 4511void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4512{
4513// Generate the alien validation script.
4514 // Generate the validation script
4515 TObjString *os;
d3b18c4c 4516 if (fValidationScript.IsNull()) {
4517 fValidationScript = fExecutable;
4518 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4519 }
4520 TString validationScript = fValidationScript;
4521 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4522 if (!Connect()) {
4523 Error("WriteValidationScript", "Alien connection required");
4524 return;
4525 }
d3b18c4c 4526 if (!fTerminateFiles.IsNull()) {
4527 fTerminateFiles.Strip();
4528 fTerminateFiles.ReplaceAll(" ",",");
4529 }
a2f5fc01 4530 TString outStream = "";
4531 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4532 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4533 ofstream out;
0d5d317c 4534 out.open(validationScript, ios::out);
c57f56b7 4535 out << "#!/bin/bash" << endl;
4536 out << "##################################################" << endl;
4537 out << "validateout=`dirname $0`" << endl;
4538 out << "validatetime=`date`" << endl;
4539 out << "validated=\"0\";" << endl;
4540 out << "error=0" << endl;
4541 out << "if [ -z $validateout ]" << endl;
4542 out << "then" << endl;
4543 out << " validateout=\".\"" << endl;
4544 out << "fi" << endl << endl;
4545 out << "cd $validateout;" << endl;
4546 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4547 out << "echo \"*******************************************************\"" << outStream << endl;
4548 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4549 out << "" << endl;
a2f5fc01 4550 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4551 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4552 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4553 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4554 out << "ls -la ./" << outStream << endl;
4555 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4556 out << "##################################################" << endl;
ebec370a 4557 out << "" << endl;
4558
4559 out << "if [ ! -f stderr ] ; then" << endl;
4560 out << " error=1" << endl;
a2f5fc01 4561 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4562 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4563 out << "fi" << endl;
4564
b34c9f51 4565 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4566 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4567 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4568 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4569 out << "" << endl;
4570
ebec370a 4571 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4572 out << " error=1" << endl;
a2f5fc01 4573 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4574 out << " echo \"$parArch\" " << outStream << endl;
4575 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4576 out << "fi" << endl;
4577
4578 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4579 out << " error=1" << endl;
a2f5fc01 4580 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4581 out << " echo \"$segViol\" " << outStream << endl;
4582 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4583 out << "fi" << endl;
4584
4585 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4586 out << " error=1" << endl;
a2f5fc01 4587 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4588 out << " echo \"$segFault\" " << outStream << endl;
4589 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4590 out << "fi" << endl;
4591
b34c9f51 4592 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4593 out << " error=1" << endl;
a2f5fc01 4594 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4595 out << " echo \"$glibcErr\" " << outStream << endl;
4596 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4597 out << "fi" << endl;
4598
ebec370a 4599 // Part dedicated to the specific analyses running into the train
4600
d3b18c4c 4601 TString outputFiles = fOutputFiles;
4602 if (merge && !fTerminateFiles.IsNull()) {
4603 outputFiles += ",";
4604 outputFiles += fTerminateFiles;
4605 }
4606 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4607 TIter next1(arr);
a2f5fc01 4608 TString outputFile;
bbd88db8 4609 while (!merge && (os=(TObjString*)next1())) {
4610 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4611 outputFile = os->GetString();
4612 Int_t index = outputFile.Index("@");
4613 if (index > 0) outputFile.Remove(index);
bbd88db8 4614 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4615 if (outputFile.Contains("*")) continue;
4616 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4617 out << " error=1" << endl;
7c2cd90a 4618 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4619 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4620 out << "fi" << endl;
4621 }
4622 delete arr;
7c2cd90a 4623 out << "if ! [ -f outputs_valid ] ; then" << endl;
4624 out << " error=1" << endl;
4625 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4626 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4627 out << "fi" << endl;
923e2ca5 4628
c57f56b7 4629 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4630 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4631 if (!IsKeepLogs()) {
4632 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4633 outStream = "";
149d288c 4634 out << " rm -f std*" << endl;
4635 }
c57f56b7 4636 out << "fi" << endl;
4637
a2f5fc01 4638 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4639 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4640 out << "cd -" << endl;
4641 out << "exit $error" << endl;
4642 }
4643 Bool_t copy = kTRUE;
d3b18c4c 4644 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4645 if (copy) {
4646 CdWork();
4647 TString workdir = gGrid->GetHomeDirectory();
4648 workdir += fGridWorkingDir;
d3b18c4c 4649 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4650 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 4651// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4652 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
4653 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 4654 }
4655}