]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Fix for savannah bug report 87728 (Laurent) + fix invalid read found with valgrind...
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
51ClassImp(AliAnalysisAlien)
6c395669 52#if 0
53;
54#endif
c57f56b7 55
6c395669 56namespace {
57 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
58 {
59 TString sl(Form("file:%s", loc));
60 TString sr(Form("alien://%s", rem));
61 Bool_t ret = TFile::Cp(sl, sr);
62 if (!ret) {
63 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
64 }
65 return ret;
66 }
67}
68
c57f56b7 69//______________________________________________________________________________
70AliAnalysisAlien::AliAnalysisAlien()
71 :AliAnalysisGrid(),
72 fGridJDL(NULL),
0f389141 73 fMergingJDL(NULL),
c57f56b7 74 fPrice(0),
75 fTTL(0),
76 fSplitMaxInputFileNumber(0),
77 fMaxInitFailed(0),
78 fMasterResubmitThreshold(0),
bb885a9e 79 fNtestFiles(0),
319593fb 80 fNrunsPerMaster(0),
16a4353c 81 fMaxMergeFiles(0),
b3e07543 82 fMaxMergeStages(0),
d2a409b2 83 fNsubmitted(0),
a3e84053 84 fProductionMode(0),
cd11251e 85 fOutputToRunNo(0),
0f389141 86 fMergeViaJDL(0),
87 fFastReadOption(0),
e1c22e21 88 fOverwriteMode(1),
149d288c 89 fNreplicas(2),
3bdcb562 90 fNproofWorkers(0),
27734f0e 91 fNproofWorkersPerSlave(0),
3bdcb562 92 fProofReset(0),
c57f56b7 93 fRunNumbers(),
94 fExecutable(),
0a1c1f7f 95 fExecutableCommand(),
c57f56b7 96 fArguments(),
631c0b05 97 fExecutableArgs(),
c57f56b7 98 fAnalysisMacro(),
99 fAnalysisSource(),
d3b18c4c 100 fValidationScript(),
d5c6455a 101 fAdditionalRootLibs(),
c57f56b7 102 fAdditionalLibs(),
103 fSplitMode(),
104 fAPIVersion(),
105 fROOTVersion(),
106 fAliROOTVersion(),
648174cf 107 fExternalPackages(),
c57f56b7 108 fUser(),
109 fGridWorkingDir(),
110 fGridDataDir(),
111 fDataPattern(),
112 fGridOutputDir(),
113 fOutputArchive(),
114 fOutputFiles(),
115 fInputFormat(),
e7c71df0 116 fDatasetName(),
c57f56b7 117 fJDLName(),
d3b18c4c 118 fTerminateFiles(),
bb885a9e 119 fMergeExcludes(),
f965131e 120 fIncludePath(),
bb885a9e 121 fCloseSE(),
0df6ccf2 122 fFriendChainName(),
c6cb3634 123 fJobTag(),
648174cf 124 fOutputSingle(),
5fce53f4 125 fRunPrefix(),
3bdcb562 126 fProofCluster(),
127 fProofDataSet(),
128 fFileForTestMode(),
129 fRootVersionForProof(),
130 fAliRootMode(),
f47d5cb4 131 fMergeDirName(),
4e5c5506 132 fInputFiles(0),
28ce6c9c 133 fPackages(0),
77f34eae 134 fModules(0),
28ce6c9c 135 fProofParam()
c57f56b7 136{
137// Dummy ctor.
138 SetDefaults();
139}
140
141//______________________________________________________________________________
142AliAnalysisAlien::AliAnalysisAlien(const char *name)
143 :AliAnalysisGrid(name),
144 fGridJDL(NULL),
0f389141 145 fMergingJDL(NULL),
c57f56b7 146 fPrice(0),
147 fTTL(0),
148 fSplitMaxInputFileNumber(0),
149 fMaxInitFailed(0),
150 fMasterResubmitThreshold(0),
bb885a9e 151 fNtestFiles(0),
319593fb 152 fNrunsPerMaster(0),
16a4353c 153 fMaxMergeFiles(0),
b3e07543 154 fMaxMergeStages(0),
d2a409b2 155 fNsubmitted(0),
a3e84053 156 fProductionMode(0),
cd11251e 157 fOutputToRunNo(0),
0f389141 158 fMergeViaJDL(0),
159 fFastReadOption(0),
e1c22e21 160 fOverwriteMode(1),
149d288c 161 fNreplicas(2),
3bdcb562 162 fNproofWorkers(0),
27734f0e 163 fNproofWorkersPerSlave(0),
3bdcb562 164 fProofReset(0),
c57f56b7 165 fRunNumbers(),
166 fExecutable(),
0a1c1f7f 167 fExecutableCommand(),
c57f56b7 168 fArguments(),
631c0b05 169 fExecutableArgs(),
c57f56b7 170 fAnalysisMacro(),
171 fAnalysisSource(),
d3b18c4c 172 fValidationScript(),
d5c6455a 173 fAdditionalRootLibs(),
c57f56b7 174 fAdditionalLibs(),
175 fSplitMode(),
176 fAPIVersion(),
177 fROOTVersion(),
178 fAliROOTVersion(),
648174cf 179 fExternalPackages(),
c57f56b7 180 fUser(),
181 fGridWorkingDir(),
182 fGridDataDir(),
183 fDataPattern(),
184 fGridOutputDir(),
185 fOutputArchive(),
186 fOutputFiles(),
187 fInputFormat(),
e7c71df0 188 fDatasetName(),
c57f56b7 189 fJDLName(),
d3b18c4c 190 fTerminateFiles(),
bb885a9e 191 fMergeExcludes(),
f965131e 192 fIncludePath(),
bb885a9e 193 fCloseSE(),
0df6ccf2 194 fFriendChainName(),
c6cb3634 195 fJobTag(),
648174cf 196 fOutputSingle(),
5fce53f4 197 fRunPrefix(),
3bdcb562 198 fProofCluster(),
199 fProofDataSet(),
200 fFileForTestMode(),
201 fRootVersionForProof(),
202 fAliRootMode(),
f47d5cb4 203 fMergeDirName(),
4e5c5506 204 fInputFiles(0),
28ce6c9c 205 fPackages(0),
77f34eae 206 fModules(0),
28ce6c9c 207 fProofParam()
c57f56b7 208{
209// Default ctor.
210 SetDefaults();
211}
212
213//______________________________________________________________________________
214AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
215 :AliAnalysisGrid(other),
216 fGridJDL(NULL),
0f389141 217 fMergingJDL(NULL),
c57f56b7 218 fPrice(other.fPrice),
219 fTTL(other.fTTL),
220 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
221 fMaxInitFailed(other.fMaxInitFailed),
222 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 223 fNtestFiles(other.fNtestFiles),
319593fb 224 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 225 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 226 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 227 fNsubmitted(other.fNsubmitted),
a3e84053 228 fProductionMode(other.fProductionMode),
cd11251e 229 fOutputToRunNo(other.fOutputToRunNo),
0f389141 230 fMergeViaJDL(other.fMergeViaJDL),
231 fFastReadOption(other.fFastReadOption),
232 fOverwriteMode(other.fOverwriteMode),
149d288c 233 fNreplicas(other.fNreplicas),
3bdcb562 234 fNproofWorkers(other.fNproofWorkers),
27734f0e 235 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 236 fProofReset(other.fProofReset),
c57f56b7 237 fRunNumbers(other.fRunNumbers),
238 fExecutable(other.fExecutable),
0a1c1f7f 239 fExecutableCommand(other.fExecutableCommand),
c57f56b7 240 fArguments(other.fArguments),
631c0b05 241 fExecutableArgs(other.fExecutableArgs),
c57f56b7 242 fAnalysisMacro(other.fAnalysisMacro),
243 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 244 fValidationScript(other.fValidationScript),
d5c6455a 245 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 246 fAdditionalLibs(other.fAdditionalLibs),
247 fSplitMode(other.fSplitMode),
248 fAPIVersion(other.fAPIVersion),
249 fROOTVersion(other.fROOTVersion),
250 fAliROOTVersion(other.fAliROOTVersion),
648174cf 251 fExternalPackages(other.fExternalPackages),
c57f56b7 252 fUser(other.fUser),
253 fGridWorkingDir(other.fGridWorkingDir),
254 fGridDataDir(other.fGridDataDir),
255 fDataPattern(other.fDataPattern),
256 fGridOutputDir(other.fGridOutputDir),
257 fOutputArchive(other.fOutputArchive),
258 fOutputFiles(other.fOutputFiles),
259 fInputFormat(other.fInputFormat),
e7c71df0 260 fDatasetName(other.fDatasetName),
c57f56b7 261 fJDLName(other.fJDLName),
d3b18c4c 262 fTerminateFiles(other.fTerminateFiles),
bb885a9e 263 fMergeExcludes(other.fMergeExcludes),
f965131e 264 fIncludePath(other.fIncludePath),
bb885a9e 265 fCloseSE(other.fCloseSE),
0df6ccf2 266 fFriendChainName(other.fFriendChainName),
c6cb3634 267 fJobTag(other.fJobTag),
648174cf 268 fOutputSingle(other.fOutputSingle),
5fce53f4 269 fRunPrefix(other.fRunPrefix),
3bdcb562 270 fProofCluster(other.fProofCluster),
271 fProofDataSet(other.fProofDataSet),
272 fFileForTestMode(other.fFileForTestMode),
273 fRootVersionForProof(other.fRootVersionForProof),
274 fAliRootMode(other.fAliRootMode),
f47d5cb4 275 fMergeDirName(other.fMergeDirName),
4e5c5506 276 fInputFiles(0),
28ce6c9c 277 fPackages(0),
77f34eae 278 fModules(0),
28ce6c9c 279 fProofParam()
c57f56b7 280{
281// Copy ctor.
282 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 283 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 284 fRunRange[0] = other.fRunRange[0];
285 fRunRange[1] = other.fRunRange[1];
c57f56b7 286 if (other.fInputFiles) {
287 fInputFiles = new TObjArray();
288 TIter next(other.fInputFiles);
289 TObject *obj;
290 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
291 fInputFiles->SetOwner();
292 }
4e5c5506 293 if (other.fPackages) {
294 fPackages = new TObjArray();
295 TIter next(other.fPackages);
296 TObject *obj;
297 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
298 fPackages->SetOwner();
299 }
77f34eae 300 if (other.fModules) {
301 fModules = new TObjArray();
302 fModules->SetOwner();
303 TIter next(other.fModules);
304 AliAnalysisTaskCfg *mod, *crt;
305 while ((crt=(AliAnalysisTaskCfg*)next())) {
306 mod = new AliAnalysisTaskCfg(*crt);
307 fModules->Add(mod);
308 }
309 }
c57f56b7 310}
311
312//______________________________________________________________________________
313AliAnalysisAlien::~AliAnalysisAlien()
314{
315// Destructor.
77f34eae 316 delete fGridJDL;
317 delete fMergingJDL;
318 delete fInputFiles;
319 delete fPackages;
320 delete fModules;
28ce6c9c 321 fProofParam.DeleteAll();
c57f56b7 322}
323
324//______________________________________________________________________________
325AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
326{
327// Assignment.
328 if (this != &other) {
329 AliAnalysisGrid::operator=(other);
330 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 331 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 332 fPrice = other.fPrice;
333 fTTL = other.fTTL;
334 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
335 fMaxInitFailed = other.fMaxInitFailed;
336 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 337 fNtestFiles = other.fNtestFiles;
a3e84053 338 fNrunsPerMaster = other.fNrunsPerMaster;
339 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 340 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 341 fNsubmitted = other.fNsubmitted;
342 fProductionMode = other.fProductionMode;
cd11251e 343 fOutputToRunNo = other.fOutputToRunNo;
0f389141 344 fMergeViaJDL = other.fMergeViaJDL;
345 fFastReadOption = other.fFastReadOption;
346 fOverwriteMode = other.fOverwriteMode;
149d288c 347 fNreplicas = other.fNreplicas;
3bdcb562 348 fNproofWorkers = other.fNproofWorkers;
27734f0e 349 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 350 fProofReset = other.fProofReset;
c57f56b7 351 fRunNumbers = other.fRunNumbers;
352 fExecutable = other.fExecutable;
0a1c1f7f 353 fExecutableCommand = other.fExecutableCommand;
c57f56b7 354 fArguments = other.fArguments;
631c0b05 355 fExecutableArgs = other.fExecutableArgs;
c57f56b7 356 fAnalysisMacro = other.fAnalysisMacro;
357 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 358 fValidationScript = other.fValidationScript;
d5c6455a 359 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 360 fAdditionalLibs = other.fAdditionalLibs;
361 fSplitMode = other.fSplitMode;
362 fAPIVersion = other.fAPIVersion;
363 fROOTVersion = other.fROOTVersion;
364 fAliROOTVersion = other.fAliROOTVersion;
648174cf 365 fExternalPackages = other.fExternalPackages;
c57f56b7 366 fUser = other.fUser;
367 fGridWorkingDir = other.fGridWorkingDir;
368 fGridDataDir = other.fGridDataDir;
369 fDataPattern = other.fDataPattern;
370 fGridOutputDir = other.fGridOutputDir;
371 fOutputArchive = other.fOutputArchive;
372 fOutputFiles = other.fOutputFiles;
373 fInputFormat = other.fInputFormat;
e7c71df0 374 fDatasetName = other.fDatasetName;
c57f56b7 375 fJDLName = other.fJDLName;
d3b18c4c 376 fTerminateFiles = other.fTerminateFiles;
bb885a9e 377 fMergeExcludes = other.fMergeExcludes;
f965131e 378 fIncludePath = other.fIncludePath;
bb885a9e 379 fCloseSE = other.fCloseSE;
0df6ccf2 380 fFriendChainName = other.fFriendChainName;
c6cb3634 381 fJobTag = other.fJobTag;
648174cf 382 fOutputSingle = other.fOutputSingle;
5fce53f4 383 fRunPrefix = other.fRunPrefix;
3bdcb562 384 fProofCluster = other.fProofCluster;
385 fProofDataSet = other.fProofDataSet;
386 fFileForTestMode = other.fFileForTestMode;
387 fRootVersionForProof = other.fRootVersionForProof;
388 fAliRootMode = other.fAliRootMode;
f47d5cb4 389 fMergeDirName = other.fMergeDirName;
c57f56b7 390 if (other.fInputFiles) {
391 fInputFiles = new TObjArray();
392 TIter next(other.fInputFiles);
393 TObject *obj;
394 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
395 fInputFiles->SetOwner();
396 }
4e5c5506 397 if (other.fPackages) {
398 fPackages = new TObjArray();
399 TIter next(other.fPackages);
400 TObject *obj;
401 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
402 fPackages->SetOwner();
403 }
77f34eae 404 if (other.fModules) {
405 fModules = new TObjArray();
406 fModules->SetOwner();
407 TIter next(other.fModules);
408 AliAnalysisTaskCfg *mod, *crt;
409 while ((crt=(AliAnalysisTaskCfg*)next())) {
410 mod = new AliAnalysisTaskCfg(*crt);
411 fModules->Add(mod);
412 }
413 }
c57f56b7 414 }
415 return *this;
416}
417
77f34eae 418//______________________________________________________________________________
419void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
420{
421// Adding a module. Checks if already existing. Becomes owned by this.
422 if (!module) return;
423 if (GetModule(module->GetName())) {
424 Error("AddModule", "A module having the same name %s already added", module->GetName());
425 return;
426 }
427 if (!fModules) {
428 fModules = new TObjArray();
429 fModules->SetOwner();
430 }
431 fModules->Add(module);
432}
433
434//______________________________________________________________________________
435void AliAnalysisAlien::AddModules(TObjArray *list)
436{
437// Adding a list of modules. Checks if already existing. Becomes owned by this.
438 TIter next(list);
439 AliAnalysisTaskCfg *module;
440 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
441}
442
443//______________________________________________________________________________
444Bool_t AliAnalysisAlien::CheckDependencies()
445{
446// Check if all dependencies are satisfied. Reorder modules if needed.
447 Int_t nmodules = GetNmodules();
448 if (!nmodules) {
449 Warning("CheckDependencies", "No modules added yet to check their dependencies");
450 return kTRUE;
451 }
452 AliAnalysisTaskCfg *mod = 0;
453 AliAnalysisTaskCfg *dep = 0;
454 TString depname;
455 Int_t i, j, k;
456 for (i=0; i<nmodules; i++) {
457 mod = (AliAnalysisTaskCfg*) fModules->At(i);
458 Int_t ndeps = mod->GetNdeps();
459 Int_t istart = i;
460 for (j=0; j<ndeps; j++) {
461 depname = mod->GetDependency(j);
462 dep = GetModule(depname);
463 if (!dep) {
464 Error("CheckDependencies","Dependency %s not added for module %s",
465 depname.Data(), mod->GetName());
466 return kFALSE;
467 }
468 if (dep->NeedsDependency(mod->GetName())) {
469 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
470 mod->GetName(), dep->GetName());
471 return kFALSE;
472 }
473 Int_t idep = fModules->IndexOf(dep);
474 // The dependency task must come first
475 if (idep>i) {
476 // Remove at idep and move all objects below up one slot
477 // down to index i included.
478 fModules->RemoveAt(idep);
3e40fd4c 479 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 480 fModules->AddAt(dep, i++);
481 }
482 //Redo from istart if dependencies were inserted
483 if (i>istart) i=istart-1;
484 }
485 }
486 return kTRUE;
487}
488
3e40fd4c 489//______________________________________________________________________________
490AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
491{
492// Create the analysis manager and optionally execute the macro in filename.
493 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
494 if (mgr) return mgr;
495 mgr = new AliAnalysisManager(name);
496 mgr->SetGridHandler((AliAnalysisGrid*)this);
497 if (strlen(filename)) {
498 TString line = gSystem->ExpandPathName(filename);
499 line.Prepend(".x ");
500 gROOT->ProcessLine(line.Data());
501 }
502 return mgr;
503}
504
77f34eae 505//______________________________________________________________________________
506Int_t AliAnalysisAlien::GetNmodules() const
507{
508// Get number of modules.
509 if (!fModules) return 0;
510 return fModules->GetEntries();
511}
512
513//______________________________________________________________________________
514AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
515{
516// Get a module by name.
517 if (!fModules) return 0;
518 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
519}
520
521//______________________________________________________________________________
522Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
523{
524// Load a given module.
525 if (mod->IsLoaded()) return kTRUE;
526 Int_t ndeps = mod->GetNdeps();
527 TString depname;
528 for (Int_t j=0; j<ndeps; j++) {
529 depname = mod->GetDependency(j);
530 AliAnalysisTaskCfg *dep = GetModule(depname);
531 if (!dep) {
532 Error("LoadModule","Dependency %s not existing for module %s",
533 depname.Data(), mod->GetName());
534 return kFALSE;
535 }
536 if (!LoadModule(dep)) {
537 Error("LoadModule","Dependency %s for module %s could not be loaded",
538 depname.Data(), mod->GetName());
539 return kFALSE;
540 }
541 }
542 // Load libraries for the module
543 if (!mod->CheckLoadLibraries()) {
544 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
545 return kFALSE;
546 }
547 // Execute the macro
548 if (mod->ExecuteMacro()<0) {
549 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
550 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
551 return kFALSE;
552 }
553 // Configure dependencies
554 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
555 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
556 mod->GetConfigMacro()->GetTitle(), mod->GetName());
557 return kFALSE;
558 }
5e8c7c22 559 // Adjust extra libraries
560 Int_t nlibs = mod->GetNlibs();
561 TString lib;
562 for (Int_t i=0; i<nlibs; i++) {
563 lib = mod->GetLibrary(i);
564 if (fAdditionalLibs.Contains(lib)) continue;
565 lib = Form("lib%s.so", lib.Data());
566 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
567 fAdditionalLibs += lib;
568 }
77f34eae 569 return kTRUE;
570}
571
139fe6ba 572//______________________________________________________________________________
573Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
574{
575// Generate the full train.
576 fAdditionalLibs = "";
577 if (!LoadModules()) return kFALSE;
578 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
579 if (!mgr->InitAnalysis()) return kFALSE;
580 mgr->PrintStatus();
581 Int_t productionMode = fProductionMode;
582 SetProductionMode();
583 TString macro = fAnalysisMacro;
584 TString executable = fExecutable;
585 TString validation = fValidationScript;
586 TString execCommand = fExecutableCommand;
587 SetAnalysisMacro(Form("%s.C", name));
588 SetExecutable(Form("%s.sh", name));
589 SetExecutableCommand("aliroot -b -q ");
590 SetValidationScript(Form("%s_validation.sh", name));
591 StartAnalysis();
592 SetProductionMode(productionMode);
593 fAnalysisMacro = macro;
594 fExecutable = executable;
595 fExecutableCommand = execCommand;
596 fValidationScript = validation;
597 return kTRUE;
598}
599
77f34eae 600//______________________________________________________________________________
21ca8e59 601Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 602{
603// Generate test macros for a single module or for the full train.
5e8c7c22 604 fAdditionalLibs = "";
77f34eae 605 if (strlen(modname)) {
21ca8e59 606 if (!CheckDependencies()) return kFALSE;
607 AliAnalysisTaskCfg *mod = GetModule(modname);
608 if (!mod) {
609 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
610 return kFALSE;
611 }
612 if (!LoadModule(mod)) return kFALSE;
613 } else if (!LoadModules()) return kFALSE;
614 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
615 if (!mgr->InitAnalysis()) return kFALSE;
616 mgr->PrintStatus();
617 SetLocalTest(kTRUE);
618 Int_t productionMode = fProductionMode;
619 SetProductionMode();
620 TString macro = fAnalysisMacro;
621 TString executable = fExecutable;
622 TString validation = fValidationScript;
623 TString execCommand = fExecutableCommand;
624 SetAnalysisMacro(Form("%s.C", name));
625 SetExecutable(Form("%s.sh", name));
626 SetExecutableCommand("aliroot -b -q ");
627 SetValidationScript(Form("%s_validation.sh", name));
628 WriteAnalysisFile();
629 WriteAnalysisMacro();
630 WriteExecutable();
631 WriteValidationScript();
632 SetLocalTest(kFALSE);
633 SetProductionMode(productionMode);
634 fAnalysisMacro = macro;
635 fExecutable = executable;
636 fExecutableCommand = execCommand;
637 fValidationScript = validation;
77f34eae 638 return kTRUE;
639}
640
641//______________________________________________________________________________
642Bool_t AliAnalysisAlien::LoadModules()
643{
644// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 645 fAdditionalLibs = "";
21ca8e59 646 Int_t nmodules = GetNmodules();
647 if (!nmodules) {
648 Warning("LoadModules", "No module to be loaded");
649 return kTRUE;
650 }
3e40fd4c 651 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
652 if (!mgr) {
653 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
654 return kFALSE;
655 }
77f34eae 656 if (!CheckDependencies()) return kFALSE;
21ca8e59 657 nmodules = GetNmodules();
77f34eae 658 AliAnalysisTaskCfg *mod;
21ca8e59 659 for (Int_t imod=0; imod<nmodules; imod++) {
660 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 661 if (!LoadModule(mod)) return kFALSE;
662 }
77f34eae 663 return kTRUE;
664}
665
74013535 666//______________________________________________________________________________
667void AliAnalysisAlien::SetRunPrefix(const char *prefix)
668{
669// Set the run number format. Can be a prefix or a format like "%09d"
670 fRunPrefix = prefix;
671 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
672}
673
f965131e 674//______________________________________________________________________________
675void AliAnalysisAlien::AddIncludePath(const char *path)
676{
677// Add include path in the remote analysis macro.
678 TString p(path);
679 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
680 else fIncludePath += Form("-I%s ", path);
681}
682
c57f56b7 683//______________________________________________________________________________
684void AliAnalysisAlien::AddRunNumber(Int_t run)
685{
686// Add a run number to the list of runs to be processed.
687 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 688 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 689}
690
98690895 691//______________________________________________________________________________
692void AliAnalysisAlien::AddRunList(const char* runList)
693{
694// Add several runs into the list of runs; they are expected to be separated by a blank character.
695 TString sList = runList;
696 TObjArray *list = sList.Tokenize(" ");
697 Int_t n = list->GetEntries();
698 for (Int_t i = 0; i < n; i++) {
699 TObjString *os = (TObjString*)list->At(i);
700 AddRunNumber(os->GetString().Atoi());
701 }
702 delete list;
703}
704
ee75cfc3 705//______________________________________________________________________________
706void AliAnalysisAlien::AddRunNumber(const char* run)
707{
708// Add a run number to the list of runs to be processed.
7aa5d539 709 TString runs = run;
710 TObjString *os;
711 TObjArray *arr = runs.Tokenize(" ");
712 TIter next(arr);
713 TString prefix;
714 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
715 while ((os=(TObjString*)next())){
716 if (fRunNumbers.Length()) fRunNumbers += " ";
717 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
718 }
719 delete arr;
ee75cfc3 720}
721
c57f56b7 722//______________________________________________________________________________
723void AliAnalysisAlien::AddDataFile(const char *lfn)
724{
725// Adds a data file to the input to be analysed. The file should be a valid LFN
726// or point to an existing file in the alien workdir.
727 if (!fInputFiles) fInputFiles = new TObjArray();
728 fInputFiles->Add(new TObjString(lfn));
729}
648174cf 730
731//______________________________________________________________________________
732void AliAnalysisAlien::AddExternalPackage(const char *package)
733{
734// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
735 if (fExternalPackages) fExternalPackages += " ";
736 fExternalPackages += package;
737}
738
c57f56b7 739//______________________________________________________________________________
740Bool_t AliAnalysisAlien::Connect()
741{
742// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
743 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 744 if (fProductionMode) return kTRUE;
c57f56b7 745 if (!gGrid) {
746 Info("Connect", "Trying to connect to AliEn ...");
747 TGrid::Connect("alien://");
748 }
749 if (!gGrid || !gGrid->IsConnected()) {
750 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
751 return kFALSE;
752 }
753 fUser = gGrid->GetUser();
754 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
755 return kTRUE;
756}
757
758//______________________________________________________________________________
759void AliAnalysisAlien::CdWork()
760{
761// Check validity of alien workspace. Create directory if possible.
762 if (!Connect()) {
763 Error("CdWork", "Alien connection required");
764 return;
765 }
766 TString homedir = gGrid->GetHomeDirectory();
767 TString workdir = homedir + fGridWorkingDir;
923e2ca5 768 if (DirectoryExists(workdir)) {
769 gGrid->Cd(workdir);
770 return;
771 }
772 // Work directory not existing - create it
773 gGrid->Cd(homedir);
b93f8109 774 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 775 gGrid->Cd(fGridWorkingDir);
d3b18c4c 776 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 777 } else {
d3b18c4c 778 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 779 workdir.Data(), homedir.Data());
780 fGridWorkingDir = "";
781 }
c57f56b7 782}
783
348be253 784//______________________________________________________________________________
785Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
786{
787// Check if file copying is possible.
d3b18c4c 788 if (fProductionMode) return kTRUE;
348be253 789 if (!Connect()) {
790 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
791 return kFALSE;
792 }
d3339be3 793 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
794 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 795 // Check if alien_CLOSE_SE is defined
796 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
797 if (!closeSE.IsNull()) {
798 Info("CheckFileCopy", "Your current close storage is pointing to: \
799 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
800 } else {
801 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
802 }
803 // Check if grid directory exists.
804 if (!DirectoryExists(alienpath)) {
805 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
806 return kFALSE;
807 }
d3339be3 808 TFile f("plugin_test_copy", "RECREATE");
348be253 809 // User may not have write permissions to current directory
810 if (f.IsZombie()) {
811 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
812 gSystem->WorkingDirectory());
813 return kFALSE;
814 }
815 f.Close();
d3339be3 816 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
817 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
818 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 819 \n# 1. Make sure you have write permissions there. If this is the case: \
820 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
821 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
822 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
823 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
824 gSystem->Unlink(f.GetName());
825 return kFALSE;
826 }
827 gSystem->Unlink(f.GetName());
d3339be3 828 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
829 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 830 return kTRUE;
831}
832
c57f56b7 833//______________________________________________________________________________
834Bool_t AliAnalysisAlien::CheckInputData()
835{
836// Check validity of input data. If necessary, create xml files.
d3b18c4c 837 if (fProductionMode) return kTRUE;
a8739e8a 838 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 839 if (!fGridDataDir.Length()) {
840 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
841 return kFALSE;
842 }
9f020952 843 if (fMergeViaJDL) {
844 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
845 return kFALSE;
846 }
d2a409b2 847 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 848 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
849 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 850 return kTRUE;
c57f56b7 851 }
852 // Process declared files
a2f5fc01 853 Bool_t isCollection = kFALSE;
854 Bool_t isXml = kFALSE;
855 Bool_t useTags = kFALSE;
c57f56b7 856 Bool_t checked = kFALSE;
d3b18c4c 857 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 858 TString file;
859 TString workdir = gGrid->GetHomeDirectory();
860 workdir += fGridWorkingDir;
861 if (fInputFiles) {
862 TObjString *objstr;
863 TIter next(fInputFiles);
864 while ((objstr=(TObjString*)next())) {
865 file = workdir;
866 file += "/";
867 file += objstr->GetString();
868 // Store full lfn path
869 if (FileExists(file)) objstr->SetString(file);
870 else {
871 file = objstr->GetName();
872 if (!FileExists(objstr->GetName())) {
873 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
874 objstr->GetName(), workdir.Data());
875 return kFALSE;
876 }
877 }
878 Bool_t iscoll, isxml, usetags;
879 CheckDataType(file, iscoll, isxml, usetags);
880 if (!checked) {
881 checked = kTRUE;
a2f5fc01 882 isCollection = iscoll;
883 isXml = isxml;
884 useTags = usetags;
885 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 886 } else {
a2f5fc01 887 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 888 Error("CheckInputData", "Some conflict was found in the types of inputs");
889 return kFALSE;
890 }
891 }
892 }
893 }
894 // Process requested run numbers
a8739e8a 895 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 896 // Check validity of alien data directory
897 if (!fGridDataDir.Length()) {
898 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
899 return kFALSE;
900 }
923e2ca5 901 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 902 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
903 return kFALSE;
904 }
a2f5fc01 905 if (isCollection) {
c57f56b7 906 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
907 return kFALSE;
908 }
909
a2f5fc01 910 if (checked && !isXml) {
c57f56b7 911 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
912 return kFALSE;
913 }
914 // Check validity of run number(s)
915 TObjArray *arr;
916 TObjString *os;
74013535 917 TString format;
319593fb 918 Int_t nruns = 0;
904f9f5f 919 TString schunk, schunk2;
c57f56b7 920 TString path;
921 if (!checked) {
922 checked = kTRUE;
a2f5fc01 923 useTags = fDataPattern.Contains("tag");
924 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 925 }
a2f5fc01 926 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 927 Error("CheckInputData", "Cannot mix input files using/not using tags");
928 return kFALSE;
929 }
930 if (fRunNumbers.Length()) {
a8739e8a 931 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 932 arr = fRunNumbers.Tokenize(" ");
933 TIter next(arr);
934 while ((os=(TObjString*)next())) {
935 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 936 if (!DirectoryExists(path)) {
937 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 938 continue;
c57f56b7 939 }
940 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
941 TString msg = "\n##### file: ";
942 msg += path;
943 msg += " type: xml_collection;";
a2f5fc01 944 if (useTags) msg += " using_tags: Yes";
c57f56b7 945 else msg += " using_tags: No";
23329835 946 Info("CheckDataType", "%s", msg.Data());
319593fb 947 if (fNrunsPerMaster<2) {
d2a409b2 948 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 949 } else {
950 nruns++;
951 if (((nruns-1)%fNrunsPerMaster) == 0) {
952 schunk = os->GetString();
953 }
954 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
955 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 956 AddDataFile(schunk);
319593fb 957 }
c57f56b7 958 }
959 delete arr;
a8739e8a 960 } else {
961 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
962 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 963 format = Form("%%s/%s ", fRunPrefix.Data());
964 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 965 if (!DirectoryExists(path)) {
a8739e8a 966 continue;
967 }
74013535 968 format = Form("%%s/%s.xml", fRunPrefix.Data());
969 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 970 TString msg = "\n##### file: ";
971 msg += path;
972 msg += " type: xml_collection;";
a2f5fc01 973 if (useTags) msg += " using_tags: Yes";
a8739e8a 974 else msg += " using_tags: No";
23329835 975 Info("CheckDataType", "%s", msg.Data());
319593fb 976 if (fNrunsPerMaster<2) {
74013535 977 format = Form("%s.xml", fRunPrefix.Data());
978 AddDataFile(Form(format.Data(),irun));
319593fb 979 } else {
980 nruns++;
981 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 982 schunk = Form(fRunPrefix.Data(),irun);
319593fb 983 }
74013535 984 format = Form("_%s.xml", fRunPrefix.Data());
985 schunk2 = Form(format.Data(), irun);
319593fb 986 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 987 schunk += schunk2;
d2a409b2 988 AddDataFile(schunk);
319593fb 989 }
a8739e8a 990 }
904f9f5f 991 if (!fInputFiles) {
992 schunk += schunk2;
993 AddDataFile(schunk);
994 }
c57f56b7 995 }
996 return kTRUE;
997}
998
21ca8e59 999//______________________________________________________________________________
139fe6ba 1000Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *anchorfile, const char *outputdir)
21ca8e59 1001{
1002// Copy data from the given grid directory according a pattern and make a local
1003// dataset.
1004 if (!Connect()) {
1005 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
1006 return kFALSE;
1007 }
1008 if (!DirectoryExists(griddir)) {
1009 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
1010 return kFALSE;
1011 }
1012 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1013 printf("Running command: %s\n", command.Data());
1014 TGridResult *res = gGrid->Command(command);
1015 Int_t nfound = res->GetEntries();
1016 if (!nfound) {
1017 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
1018 return kFALSE;
1019 }
1020 printf("... found %d files. Copying locally ...\n", nfound);
1021 // Copy files locally
1022 ofstream out;
1023 out.open(output, ios::out);
1024 TMap *map;
1025 TString turl, dirname, filename, temp;
1026 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1027 gSystem->MakeDirectory(outputdir);
1028 gSystem->ChangeDirectory(outputdir);
21ca8e59 1029 for (Int_t i=0; i<nfound; i++) {
1030 map = (TMap*)res->At(i);
1031 turl = map->GetValue("turl")->GetName();
1032 filename = gSystem->BaseName(turl.Data());
1033 dirname = gSystem->DirName(turl.Data());
1034 dirname = gSystem->BaseName(dirname.Data());
1035 gSystem->MakeDirectory(dirname);
1036 if (TFile::Cp(turl, Form("file:./%s/%s", dirname.Data(), filename.Data()))) {
1037 if (strlen(anchorfile)) filename = Form("%s#%s", filename.Data(), anchorfile);
139fe6ba 1038 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), filename.Data()) << endl;
21ca8e59 1039 }
1040 }
1041 gSystem->ChangeDirectory(cdir);
1042 delete res;
1043 return kTRUE;
1044}
1045
c57f56b7 1046//______________________________________________________________________________
1047Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1048{
1049// Create dataset for the grid data directory + run number.
3af861ff 1050 const Int_t gMaxEntries = 15000;
d3b18c4c 1051 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1052 if (!Connect()) {
1053 Error("CreateDataset", "Cannot create dataset with no grid connection");
1054 return kFALSE;
1055 }
1056
1057 // Cd workspace
d3b18c4c 1058 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1059 TString workdir = gGrid->GetHomeDirectory();
1060 workdir += fGridWorkingDir;
1061
1062 // Compose the 'find' command arguments
74013535 1063 TString format;
c57f56b7 1064 TString command;
1065 TString options = "-x collection ";
bb885a9e 1066 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1067 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1068 TString conditions = "";
3af861ff 1069 Int_t nstart = 0;
1070 Int_t ncount = 0;
1071 Int_t stage = 0;
c57f56b7 1072 TString file;
1073 TString path;
319593fb 1074 Int_t nruns = 0;
904f9f5f 1075 TString schunk, schunk2;
ab254fd1 1076 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1077 if (!fRunNumbers.Length() && !fRunRange[0]) {
1078 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1079 // Make a single data collection from data directory.
1080 path = fGridDataDir;
923e2ca5 1081 if (!DirectoryExists(path)) {
d2a409b2 1082 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1083 return kFALSE;
1084 }
923e2ca5 1085// CdWork();
d2a409b2 1086 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1087 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1088 while (1) {
1089 ncount = 0;
1090 stage++;
1091 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1092 command = "find ";
1093 command += Form("%s -o %d ",options.Data(), nstart);
1094 command += path;
1095 command += " ";
1096 command += pattern;
1097 command += conditions;
1098 printf("command: %s\n", command.Data());
1099 TGridResult *res = gGrid->Command(command);
1100 if (res) delete res;
1101 // Write standard output to file
1102 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1103 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1104 Bool_t nullFile = kFALSE;
1105 if (!hasGrep) {
1106 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1107 } else {
1108 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1109 if (nullFile) {
1110 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1111 gSystem->Exec("rm -f __tmp*");
1112 return kFALSE;
1113 }
1114 TString line;
1115 ifstream in;
1116 in.open("__tmp__");
1117 in >> line;
1118 in.close();
1119 gSystem->Exec("rm -f __tmp__");
1120 ncount = line.Atoi();
1121 }
1122 }
1123 if (ncount == gMaxEntries) {
1124 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1125 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1126 if (!cbase) cbase = cadd;
1127 else {
1128 cbase->Add(cadd);
1129 delete cadd;
defd7a3a 1130 }
3af861ff 1131 nstart += ncount;
1132 } else {
1133 if (cbase) {
1134 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1135 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1136 cbase->Add(cadd);
1137 delete cadd;
1138 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1139 delete cbase; cbase = 0;
1140 } else {
1141 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1142 }
1143 gSystem->Exec("rm -f __tmp*");
1144 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1145 break;
1146 }
0f389141 1147 }
1148 Bool_t fileExists = FileExists(file);
1149 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1150 // Copy xml file to alien space
0f389141 1151 if (fileExists) gGrid->Rm(file);
d2a409b2 1152 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1153 if (!FileExists(file)) {
1154 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1155 return kFALSE;
1156 }
1157 // Update list of files to be processed.
1158 }
1159 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1160 return kTRUE;
1161 }
c57f56b7 1162 // Several runs
a2f5fc01 1163 Bool_t nullResult = kTRUE;
a8739e8a 1164 if (fRunNumbers.Length()) {
1165 TObjArray *arr = fRunNumbers.Tokenize(" ");
1166 TObjString *os;
1167 TIter next(arr);
1168 while ((os=(TObjString*)next())) {
3af861ff 1169 nstart = 0;
1170 stage = 0;
dc030a24 1171 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1172 if (!DirectoryExists(path)) continue;
1173// CdWork();
a8739e8a 1174 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1175 else file = Form("%s.xml", os->GetString().Data());
319593fb 1176 // If local collection file does not exist, create it via 'find' command.
3af861ff 1177 while (1) {
1178 ncount = 0;
1179 stage++;
1180 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1181 command = "find ";
1182 command += Form("%s -o %d ",options.Data(), nstart);
1183 command += path;
1184 command += pattern;
1185 command += conditions;
1186 TGridResult *res = gGrid->Command(command);
1187 if (res) delete res;
1188 // Write standard output to file
1189 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1190 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1191 Bool_t nullFile = kFALSE;
1192 if (!hasGrep) {
1193 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1194 } else {
1195 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1196 if (nullFile) {
1197 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1198 gSystem->Exec("rm -f __tmp*");
1199 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1200 break;
3af861ff 1201 }
1202 TString line;
1203 ifstream in;
1204 in.open("__tmp__");
1205 in >> line;
1206 in.close();
1207 gSystem->Exec("rm -f __tmp__");
1208 ncount = line.Atoi();
1209 }
1210 nullResult = kFALSE;
1211 }
1212 if (ncount == gMaxEntries) {
1213 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1214 if (fNrunsPerMaster > 1) {
1215 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1216 file.Data(),gMaxEntries);
1217 return kFALSE;
1218 }
1219 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1220 if (!cbase) cbase = cadd;
1221 else {
1222 cbase->Add(cadd);
1223 delete cadd;
defd7a3a 1224 }
3af861ff 1225 nstart += ncount;
1226 } else {
c89a84bc 1227 if (cbase && fNrunsPerMaster<2) {
3af861ff 1228 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1229 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1230 cbase->Add(cadd);
1231 delete cadd;
1232 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1233 delete cbase; cbase = 0;
1234 } else {
1235 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1236 }
1237 gSystem->Exec("rm -f __tmp*");
1238 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1239 break;
d3339be3 1240 }
3af861ff 1241 }
a8739e8a 1242 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1243 // Check if there is one run per master job.
1244 if (fNrunsPerMaster<2) {
1245 if (FileExists(file)) {
0f389141 1246 if (fOverwriteMode) gGrid->Rm(file);
1247 else {
1248 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1249 continue;
1250 }
319593fb 1251 }
1252 // Copy xml file to alien space
1253 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1254 if (!FileExists(file)) {
1255 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1256 delete arr;
1257 return kFALSE;
1258 }
1259 } else {
1260 nruns++;
1261 if (((nruns-1)%fNrunsPerMaster) == 0) {
1262 schunk = os->GetString();
1263 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1264 } else {
1265 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1266 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1267 cbase->Add(cadd);
1268 delete cadd;
1269 }
1270 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1271 continue;
1272 }
1273 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1274 if (FileExists(schunk)) {
1275 if (fOverwriteMode) gGrid->Rm(file);
1276 else {
1277 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1278 continue;
1279 }
319593fb 1280 }
84fcd93f 1281 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1282 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1283 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1284 if (!FileExists(schunk)) {
1285 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1286 delete arr;
1287 return kFALSE;
1288 }
d3339be3 1289 }
a8739e8a 1290 }
1291 delete arr;
a2f5fc01 1292 if (nullResult) {
d3339be3 1293 Error("CreateDataset", "No valid dataset corresponding to the query!");
1294 return kFALSE;
1295 }
a8739e8a 1296 } else {
1297 // Process a full run range.
1298 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1299 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1300 nstart = 0;
1301 stage = 0;
74013535 1302 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1303 if (!DirectoryExists(path)) continue;
1304// CdWork();
74013535 1305 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1306 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1307 else file = Form(format.Data(), irun);
0f389141 1308 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1309 if (fOverwriteMode) gGrid->Rm(file);
1310 else {
1311 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1312 continue;
1313 }
a8739e8a 1314 }
319593fb 1315 // If local collection file does not exist, create it via 'find' command.
3af861ff 1316 while (1) {
1317 ncount = 0;
1318 stage++;
1319 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1320 command = "find ";
1321 command += Form("%s -o %d ",options.Data(), nstart);
1322 command += path;
1323 command += pattern;
1324 command += conditions;
1325 TGridResult *res = gGrid->Command(command);
1326 if (res) delete res;
1327 // Write standard output to file
1328 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1329 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1330 Bool_t nullFile = kFALSE;
1331 if (!hasGrep) {
1332 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1333 } else {
1334 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1335 if (nullFile) {
1336 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1337 gSystem->Exec("rm -f __tmp*");
1604c9ed 1338 break;
3af861ff 1339 }
1340 TString line;
1341 ifstream in;
1342 in.open("__tmp__");
1343 in >> line;
1344 in.close();
1345 gSystem->Exec("rm -f __tmp__");
1346 ncount = line.Atoi();
1347 }
1348 nullResult = kFALSE;
1349 }
1350 if (ncount == gMaxEntries) {
1351 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1352 if (fNrunsPerMaster > 1) {
1353 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1354 file.Data(),gMaxEntries);
1355 return kFALSE;
1356 }
1357 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1358 if (!cbase) cbase = cadd;
1359 else {
1360 cbase->Add(cadd);
1361 delete cadd;
defd7a3a 1362 }
3af861ff 1363 nstart += ncount;
1364 } else {
c89a84bc 1365 if (cbase && fNrunsPerMaster<2) {
3af861ff 1366 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1367 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1368 cbase->Add(cadd);
1369 delete cadd;
1370 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1371 delete cbase; cbase = 0;
1372 } else {
1373 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1374 }
1375 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1376 break;
d3339be3 1377 }
319593fb 1378 }
a8739e8a 1379 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1380 // Check if there is one run per master job.
1381 if (fNrunsPerMaster<2) {
1382 if (FileExists(file)) {
0f389141 1383 if (fOverwriteMode) gGrid->Rm(file);
1384 else {
1385 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1386 continue;
1387 }
319593fb 1388 }
1389 // Copy xml file to alien space
1390 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1391 if (!FileExists(file)) {
1392 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1393 return kFALSE;
1394 }
1395 } else {
1396 nruns++;
95e5b448 1397 // Check if the collection for the chunk exist locally.
1398 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1399 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1400 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1401 else continue;
1402 }
84fcd93f 1403 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1404 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1405 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1406 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1407 } else {
1408 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1409 cbase->Add(cadd);
1410 delete cadd;
1411 }
74013535 1412 format = Form("%%s_%s.xml", fRunPrefix.Data());
1413 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1414 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1415 continue;
1416 }
904f9f5f 1417 schunk = schunk2;
319593fb 1418 if (FileExists(schunk)) {
0f389141 1419 if (fOverwriteMode) gGrid->Rm(schunk);
1420 else {
1421 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1422 continue;
1423 }
319593fb 1424 }
84fcd93f 1425 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1426 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1427 if (FileExists(schunk)) {
0f389141 1428 if (fOverwriteMode) gGrid->Rm(schunk);
1429 else {
1430 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1431 continue;
1432 }
95e5b448 1433 }
319593fb 1434 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1435 if (!FileExists(schunk)) {
1436 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1437 return kFALSE;
1438 }
1439 }
c57f56b7 1440 }
a2f5fc01 1441 if (nullResult) {
d3339be3 1442 Error("CreateDataset", "No valid dataset corresponding to the query!");
1443 return kFALSE;
1444 }
a8739e8a 1445 }
c57f56b7 1446 return kTRUE;
1447}
1448
1449//______________________________________________________________________________
1450Bool_t AliAnalysisAlien::CreateJDL()
1451{
1452// Generate a JDL file according to current settings. The name of the file is
1453// specified by fJDLName.
1454 Bool_t error = kFALSE;
1455 TObjArray *arr = 0;
1456 Bool_t copy = kTRUE;
d3b18c4c 1457 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1458 Bool_t generate = kTRUE;
1459 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1460 if (!Connect()) {
1461 Error("CreateJDL", "Alien connection required");
1462 return kFALSE;
1463 }
1464 // Check validity of alien workspace
d3b18c4c 1465 TString workdir;
1466 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1467 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1468 workdir += fGridWorkingDir;
1469 if (generate) {
1470 TObjString *os;
1471 if (!fInputFiles) {
1472 Error("CreateJDL()", "Define some input files for your analysis.");
1473 error = kTRUE;
1474 }
1475 // Compose list of input files
1476 // Check if output files were defined
1477 if (!fOutputFiles.Length()) {
1478 Error("CreateJDL", "You must define at least one output file");
1479 error = kTRUE;
1480 }
1481 // Check if an output directory was defined and valid
1482 if (!fGridOutputDir.Length()) {
1483 Error("CreateJDL", "You must define AliEn output directory");
1484 error = kTRUE;
1485 } else {
d3b18c4c 1486 if (!fProductionMode) {
1487 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1488 if (!DirectoryExists(fGridOutputDir)) {
1489 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1490 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1491 } else {
1492 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1493 // error = kTRUE;
1494 }
67a3b254 1495 } else {
1496 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1497 }
1498 gGrid->Cd(workdir);
1499 }
c57f56b7 1500 }
1501 // Exit if any error up to now
1502 if (error) return kFALSE;
1503 // Set JDL fields
0f389141 1504 if (!fUser.IsNull()) {
1505 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1506 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1507 }
1508 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1509 TString mergeExec = fExecutable;
1510 mergeExec.ReplaceAll(".sh", "_merge.sh");
1511 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1512 mergeExec.ReplaceAll(".sh", ".C");
1513 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1514 if (!fArguments.IsNull())
1515 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1516 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1517 else {
1518 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1519 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1520 }
b3e07543 1521
1f0d1ca2 1522 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1523 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1524 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1525 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1526
0f389141 1527 if (fMaxInitFailed > 0) {
c57f56b7 1528 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1529 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1530 }
1531 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1532 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1533 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1534 }
1535 if (!IsOneStageMerging()) {
f47d5cb4 1536 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1537 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1538 }
1539 if (fSplitMode.Length()) {
c57f56b7 1540 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1541 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1542 }
1543 fMergingJDL->SetValue("Split", "\"se\"");
1544 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1545 if (!fAliROOTVersion.IsNull()) {
1546 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1547 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1548 }
1549 if (!fROOTVersion.IsNull()) {
c57f56b7 1550 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1551 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1552 }
1553 if (!fAPIVersion.IsNull()) {
c57f56b7 1554 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1555 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1556 }
648174cf 1557 if (!fExternalPackages.IsNull()) {
1558 arr = fExternalPackages.Tokenize(" ");
1559 TIter next(arr);
1560 while ((os=(TObjString*)next())) {
1561 TString pkgname = os->GetString();
1562 Int_t index = pkgname.Index("::");
1563 TString pkgversion = pkgname(index+2, pkgname.Length());
1564 pkgname.Remove(index);
1565 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1566 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1567 }
1568 delete arr;
1569 }
0f389141 1570 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1571 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1572 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1573 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1574 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1575 TString analysisFile = fExecutable;
1576 analysisFile.ReplaceAll(".sh", ".root");
1577 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1578 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1579 if (fAdditionalLibs.Length()) {
1580 arr = fAdditionalLibs.Tokenize(" ");
1581 TIter next(arr);
1582 while ((os=(TObjString*)next())) {
1583 if (os->GetString().Contains(".so")) continue;
1584 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1585 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1586 }
1587 delete arr;
1588 }
4e5c5506 1589 if (fPackages) {
1590 TIter next(fPackages);
1591 TObject *obj;
0f389141 1592 while ((obj=next())) {
4e5c5506 1593 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1594 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1595 }
4e5c5506 1596 }
c57f56b7 1597 if (fOutputArchive.Length()) {
1598 arr = fOutputArchive.Tokenize(" ");
1599 TIter next(arr);
0f389141 1600 Bool_t first = kTRUE;
1601 const char *comment = "Files to be archived";
1602 const char *comment1 = comment;
1603 while ((os=(TObjString*)next())) {
1604 if (!first) comment = NULL;
1605 if (!os->GetString().Contains("@") && fCloseSE.Length())
1606 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1607 else
1608 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1609 first = kFALSE;
1610 }
c57f56b7 1611 delete arr;
f790bc1b 1612 // Output archive for the merging jdl
1613 TString outputArchive;
1614 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1615 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1616 // Add normal output files, extra files + terminate files
1617 TString files = GetListOfFiles("outextter");
1618 // Do not register merge excludes
1619 if (!fMergeExcludes.IsNull()) {
1620 arr = fMergeExcludes.Tokenize(" ");
1621 TIter next1(arr);
1622 while ((os=(TObjString*)next1())) {
1623 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1624 files.ReplaceAll(os->GetString(),"");
1625 }
66d71516 1626 delete arr;
0f389141 1627 }
f790bc1b 1628 files.ReplaceAll(".root", "*.root");
b83f84f0 1629 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1630 } else {
95f84779 1631 TString files = fOutputArchive;
1632 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1633 outputArchive = files;
f790bc1b 1634 }
0f389141 1635 arr = outputArchive.Tokenize(" ");
705adb3e 1636 TIter next2(arr);
0f389141 1637 comment = comment1;
1638 first = kTRUE;
705adb3e 1639 while ((os=(TObjString*)next2())) {
0f389141 1640 if (!first) comment = NULL;
0cdf65a8 1641 TString currentfile = os->GetString();
0cdf65a8 1642 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1643 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1644 else
0cdf65a8 1645 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1646 first = kFALSE;
1647 }
1648 delete arr;
c57f56b7 1649 }
149d288c 1650 arr = fOutputFiles.Tokenize(",");
c57f56b7 1651 TIter next(arr);
0f389141 1652 Bool_t first = kTRUE;
f790bc1b 1653 const char *comment = "Files to be saved";
43da816a 1654 while ((os=(TObjString*)next())) {
e1eaf596 1655 // Ignore ouputs in jdl that are also in outputarchive
1656 TString sout = os->GetString();
f790bc1b 1657 sout.ReplaceAll("*", "");
1658 sout.ReplaceAll(".root", "");
e1eaf596 1659 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1660 if (fOutputArchive.Contains(sout)) continue;
0f389141 1661 if (!first) comment = NULL;
43da816a 1662 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1663 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1664 else
0f389141 1665 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1666 first = kFALSE;
f790bc1b 1667 if (fMergeExcludes.Contains(sout)) continue;
1668 if (!os->GetString().Contains("@") && fCloseSE.Length())
1669 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1670 else
1671 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1672 }
c57f56b7 1673 delete arr;
0f389141 1674 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1675 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1676 TString validationScript = fValidationScript;
0f389141 1677 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1678 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1679 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1680 if (fMasterResubmitThreshold) {
1681 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1682 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1683 }
d2a409b2 1684 // Write a jdl with 2 input parameters: collection name and output dir name.
1685 WriteJDL(copy);
c57f56b7 1686 }
1687 // Copy jdl to grid workspace
a8739e8a 1688 if (copy) {
b5e4aaa7 1689 // Check if an output directory was defined and valid
1690 if (!fGridOutputDir.Length()) {
1691 Error("CreateJDL", "You must define AliEn output directory");
1692 return kFALSE;
1693 } else {
1694 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1695 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1696 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1697 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1698 } else {
1699 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1700 return kFALSE;
1701 }
1702 }
1703 gGrid->Cd(workdir);
1704 }
648174cf 1705 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1706 TString mergeJDLName = fExecutable;
1707 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1708 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1709 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1710 if (fProductionMode) {
648174cf 1711 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1712 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1713 }
648174cf 1714 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1715 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1716 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1717 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1718 Fatal("","Terminating");
1719// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1720 if (fMergeViaJDL) {
1721 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1722// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1723 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1724 Fatal("","Terminating");
0f389141 1725 }
648174cf 1726 }
c57f56b7 1727 if (fAdditionalLibs.Length()) {
e7c71df0 1728 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1729 TObjString *os;
1730 TIter next(arr);
1731 while ((os=(TObjString*)next())) {
c57f56b7 1732 if (os->GetString().Contains(".so")) continue;
4e5c5506 1733 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1734 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1735// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1736 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1737 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1738 Fatal("","Terminating");
c57f56b7 1739 }
1740 delete arr;
1741 }
4e5c5506 1742 if (fPackages) {
1743 TIter next(fPackages);
1744 TObject *obj;
1745 while ((obj=next())) {
fdbbc7be 1746 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1747 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1748// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1749 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1750 Form("%s/%s", workdir.Data(), obj->GetName())))
1751 Fatal("","Terminating");
4e5c5506 1752 }
1753 }
c57f56b7 1754 }
1755 return kTRUE;
1756}
1757
a8739e8a 1758//______________________________________________________________________________
d2a409b2 1759Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1760{
1761// Writes one or more JDL's corresponding to findex. If findex is negative,
1762// all run numbers are considered in one go (jdl). For non-negative indices
1763// they correspond to the indices in the array fInputFiles.
1764 if (!fInputFiles) return kFALSE;
d3b18c4c 1765 TObject *os;
1766 TString workdir;
1767 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1768 workdir += fGridWorkingDir;
f47d5cb4 1769 TString stageName = "$2";
1770 if (fProductionMode) stageName = "$4";
1771 if (!fMergeDirName.IsNull()) {
1772 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1773 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1774 } else {
1775 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1776 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1777 }
d3b18c4c 1778 if (fProductionMode) {
a8739e8a 1779 TIter next(fInputFiles);
b3e07543 1780 while ((os=next())) {
d3b18c4c 1781 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1782 }
56754bf6 1783 if (!fOutputToRunNo)
1784 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1785 else
1786 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1787 } else {
1788 if (!fRunNumbers.Length() && !fRunRange[0]) {
1789 // One jdl with no parameters in case input data is specified by name.
1790 TIter next(fInputFiles);
1791 while ((os=next()))
1792 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1793 if (!fOutputSingle.IsNull())
1794 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1795 else {
1796 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1797 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1798 }
1799 } else {
1800 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1801 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1802 if (!fOutputSingle.IsNull()) {
1803 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1804 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1805 } else {
1806 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1807 }
1808 }
a8739e8a 1809 }
1810
a8739e8a 1811 // Generate the JDL as a string
1812 TString sjdl = fGridJDL->Generate();
0f389141 1813 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1814 // Final merge jdl
1815 if (!fMergeDirName.IsNull()) {
1816 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1817 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1818 } else {
1819 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1820 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1821 }
b3e07543 1822 TString sjdl2 = fMergingJDL->Generate();
1823 Int_t index, index1;
a8739e8a 1824 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1825 sjdl.ReplaceAll("(member", "\n (member");
1826 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1827 sjdl.ReplaceAll("{", "{\n ");
1828 sjdl.ReplaceAll("};", "\n};");
1829 sjdl.ReplaceAll("{\n \n", "{\n");
1830 sjdl.ReplaceAll("\n\n", "\n");
1831 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1832 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1833 sjdl1.ReplaceAll("(member", "\n (member");
1834 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1835 sjdl1.ReplaceAll("{", "{\n ");
1836 sjdl1.ReplaceAll("};", "\n};");
1837 sjdl1.ReplaceAll("{\n \n", "{\n");
1838 sjdl1.ReplaceAll("\n\n", "\n");
1839 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1840 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1841 sjdl2.ReplaceAll("(member", "\n (member");
1842 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1843 sjdl2.ReplaceAll("{", "{\n ");
1844 sjdl2.ReplaceAll("};", "\n};");
1845 sjdl2.ReplaceAll("{\n \n", "{\n");
1846 sjdl2.ReplaceAll("\n\n", "\n");
1847 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1848 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1849 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1850 index = sjdl.Index("JDLVariables");
1851 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1852 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1853 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1854 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1855 index = fJobTag.Index(":");
1856 if (index < 0) index = fJobTag.Length();
1857 TString jobTag = fJobTag;
f47d5cb4 1858 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1859 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1860 if (fProductionMode) {
1861 sjdl1.Prepend("# Generated merging jdl (production mode) \
1862 \n# $1 = full alien path to output directory to be merged \
1863 \n# $2 = train number \
1864 \n# $3 = production (like LHC10b) \
1865 \n# $4 = merging stage \
1866 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1867 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1868 sjdl2.Prepend("# Generated merging jdl \
1869 \n# $1 = full alien path to output directory to be merged \
1870 \n# $2 = train number \
1871 \n# $3 = production (like LHC10b) \
1872 \n# $4 = merging stage \
1873 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1874 } else {
1875 sjdl1.Prepend("# Generated merging jdl \
1876 \n# $1 = full alien path to output directory to be merged \
1877 \n# $2 = merging stage \
03335324 1878 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1879 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1880 sjdl2.Prepend("# Generated merging jdl \
1881 \n# $1 = full alien path to output directory to be merged \
1882 \n# $2 = merging stage \
03335324 1883 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1884 }
0f389141 1885 index = sjdl1.Index("JDLVariables");
1886 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1887 index = sjdl2.Index("JDLVariables");
1888 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1889 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1890 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1891 index = sjdl2.Index("Split =");
1892 if (index>=0) {
1893 index1 = sjdl2.Index("\n", index);
1894 sjdl2.Remove(index, index1-index+1);
1895 }
1896 index = sjdl2.Index("SplitMaxInputFileNumber");
1897 if (index>=0) {
1898 index1 = sjdl2.Index("\n", index);
1899 sjdl2.Remove(index, index1-index+1);
1900 }
1901 index = sjdl2.Index("InputDataCollection");
1902 if (index>=0) {
1903 index1 = sjdl2.Index(";", index);
1904 sjdl2.Remove(index, index1-index+1);
1905 }
1906 index = sjdl2.Index("InputDataListFormat");
1907 if (index>=0) {
1908 index1 = sjdl2.Index("\n", index);
1909 sjdl2.Remove(index, index1-index+1);
1910 }
1911 index = sjdl2.Index("InputDataList");
1912 if (index>=0) {
1913 index1 = sjdl2.Index("\n", index);
1914 sjdl2.Remove(index, index1-index+1);
1915 }
f47d5cb4 1916 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1917 // Write jdl to file
a8739e8a 1918 ofstream out;
d2a409b2 1919 out.open(fJDLName.Data(), ios::out);
a8739e8a 1920 if (out.bad()) {
d3b18c4c 1921 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1922 return kFALSE;
1923 }
1924 out << sjdl << endl;
b3e07543 1925 out.close();
0f389141 1926 TString mergeJDLName = fExecutable;
1927 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1928 if (fMergeViaJDL) {
1929 ofstream out1;
1930 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1931 if (out1.bad()) {
d3b18c4c 1932 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1933 return kFALSE;
1934 }
1935 out1 << sjdl1 << endl;
b3e07543 1936 out1.close();
1937 ofstream out2;
1938 TString finalJDL = mergeJDLName;
1939 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1940 out2.open(finalJDL.Data(), ios::out);
1941 if (out2.bad()) {
1942 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1943 return kFALSE;
1944 }
1945 out2 << sjdl2 << endl;
1946 out2.close();
0f389141 1947 }
a8739e8a 1948
1949 // Copy jdl to grid workspace
1950 if (!copy) {
d3b18c4c 1951 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1952 } else {
d2a409b2 1953 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1954 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1955 TString finalJDL = mergeJDLName;
1956 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1957 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1958 if (fProductionMode) {
b5fe9cba 1959 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1960 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1961 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1962 }
d2a409b2 1963 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1964 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 1965 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 1966 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1967// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
1968 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
1969 Fatal("","Terminating");
0f389141 1970 if (fMergeViaJDL) {
b3e07543 1971 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1972// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1973// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
1974 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
1975 Fatal("","Terminating");
1976 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
1977 Fatal("","Terminating");
0f389141 1978 }
a8739e8a 1979 }
1980 return kTRUE;
1981}
1982
c57f56b7 1983//______________________________________________________________________________
5513444a 1984Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1985{
1986// Returns true if file exists.
5513444a 1987 if (!gGrid) return kFALSE;
2ea55496 1988 TString slfn = lfn;
1989 slfn.ReplaceAll("alien://","");
1990 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 1991 if (!res) return kFALSE;
1992 TMap *map = dynamic_cast<TMap*>(res->At(0));
1993 if (!map) {
1994 delete res;
1995 return kFALSE;
1996 }
1997 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1998 if (!objs || !objs->GetString().Length()) {
1999 delete res;
2000 return kFALSE;
2001 }
2002 delete res;
2003 return kTRUE;
2004}
2005
923e2ca5 2006//______________________________________________________________________________
2007Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2008{
2009// Returns true if directory exists. Can be also a path.
2010 if (!gGrid) return kFALSE;
2011 // Check if dirname is a path
2012 TString dirstripped = dirname;
2013 dirstripped = dirstripped.Strip();
2014 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2015 TString dir = gSystem->BaseName(dirstripped);
2016 dir += "/";
2017 TString path = gSystem->DirName(dirstripped);
2018 TGridResult *res = gGrid->Ls(path, "-F");
2019 if (!res) return kFALSE;
2020 TIter next(res);
2021 TMap *map;
2022 TObject *obj;
2023 while ((map=dynamic_cast<TMap*>(next()))) {
2024 obj = map->GetValue("name");
2025 if (!obj) break;
2026 if (dir == obj->GetName()) {
2027 delete res;
2028 return kTRUE;
2029 }
2030 }
2031 delete res;
2032 return kFALSE;
2033}
2034
c57f56b7 2035//______________________________________________________________________________
a2f5fc01 2036void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2037{
2038// Check input data type.
a2f5fc01 2039 isCollection = kFALSE;
2040 isXml = kFALSE;
2041 useTags = kFALSE;
c57f56b7 2042 if (!gGrid) {
2043 Error("CheckDataType", "No connection to grid");
2044 return;
2045 }
a2f5fc01 2046 isCollection = IsCollection(lfn);
c57f56b7 2047 TString msg = "\n##### file: ";
2048 msg += lfn;
a2f5fc01 2049 if (isCollection) {
c57f56b7 2050 msg += " type: raw_collection;";
2051 // special treatment for collections
a2f5fc01 2052 isXml = kFALSE;
c57f56b7 2053 // check for tag files in the collection
2054 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2055 if (!res) {
2056 msg += " using_tags: No (unknown)";
23329835 2057 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2058 return;
2059 }
2060 const char* typeStr = res->GetKey(0, "origLFN");
2061 if (!typeStr || !strlen(typeStr)) {
2062 msg += " using_tags: No (unknown)";
23329835 2063 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2064 return;
2065 }
2066 TString file = typeStr;
a2f5fc01 2067 useTags = file.Contains(".tag");
2068 if (useTags) msg += " using_tags: Yes";
c57f56b7 2069 else msg += " using_tags: No";
23329835 2070 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2071 return;
2072 }
2073 TString slfn(lfn);
2074 slfn.ToLower();
a2f5fc01 2075 isXml = slfn.Contains(".xml");
2076 if (isXml) {
c57f56b7 2077 // Open xml collection and check if there are tag files inside
2078 msg += " type: xml_collection;";
2079 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2080 if (!coll) {
2081 msg += " using_tags: No (unknown)";
23329835 2082 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2083 return;
2084 }
2085 TMap *map = coll->Next();
2086 if (!map) {
2087 msg += " using_tags: No (unknown)";
23329835 2088 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2089 return;
2090 }
2091 map = (TMap*)map->GetValue("");
2092 TString file;
2093 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2094 useTags = file.Contains(".tag");
c57f56b7 2095 delete coll;
a2f5fc01 2096 if (useTags) msg += " using_tags: Yes";
c57f56b7 2097 else msg += " using_tags: No";
23329835 2098 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2099 return;
2100 }
a2f5fc01 2101 useTags = slfn.Contains(".tag");
c57f56b7 2102 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2103 else msg += " type: unknown file;";
a2f5fc01 2104 if (useTags) msg += " using_tags: Yes";
c57f56b7 2105 else msg += " using_tags: No";
23329835 2106 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2107}
2108
4e5c5506 2109//______________________________________________________________________________
2110void AliAnalysisAlien::EnablePackage(const char *package)
2111{
2112// Enables a par file supposed to exist in the current directory.
2113 TString pkg(package);
2114 pkg.ReplaceAll(".par", "");
2115 pkg += ".par";
2116 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2117 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2118 return;
2119 }
2120 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2121 Info("EnablePackage", "AliEn plugin will use .par packages");
2122 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2123 if (!fPackages) {
2124 fPackages = new TObjArray();
2125 fPackages->SetOwner();
2126 }
2127 fPackages->Add(new TObjString(pkg));
2128}
2129
3bdcb562 2130//______________________________________________________________________________
2131TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2132{
2133// Make a tree from files having the location specified in fFileForTestMode.
2134// Inspired from JF's CreateESDChain.
2135 if (fFileForTestMode.IsNull()) {
2136 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2137 return NULL;
2138 }
2139 if (gSystem->AccessPathName(fFileForTestMode)) {
2140 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2141 return NULL;
2142 }
2143 // Open the file
2144 ifstream in;
2145 in.open(fFileForTestMode);
2146 Int_t count = 0;
2147 // Read the input list of files and add them to the chain
2148 TString line;
2149 TChain *chain = new TChain(treeName);
2150 while (in.good())
2151 {
2152 in >> line;
2153 if (line.IsNull()) continue;
2154 if (count++ == fNtestFiles) break;
2155 TString esdFile(line);
2156 TFile *file = TFile::Open(esdFile);
2157 if (file) {
2158 if (!file->IsZombie()) chain->Add(esdFile);
2159 file->Close();
2160 } else {
2161 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2162 }
2163 }
2164 in.close();
2165 if (!chain->GetListOfFiles()->GetEntries()) {
2166 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2167 delete chain;
2168 return NULL;
2169 }
2170// chain->ls();
2171 return chain;
2172}
2173
d2a409b2 2174//______________________________________________________________________________
2175const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2176{
2177// Get job status for all jobs with jobid>jobidstart.
2178 static char mstatus[20];
2179 mstatus[0] = '\0';
2180 nrunning = 0;
2181 nwaiting = 0;
2182 nerror = 0;
2183 ndone = 0;
2184 TGridJobStatusList *list = gGrid->Ps("");
2185 if (!list) return mstatus;
2186 Int_t nentries = list->GetSize();
2187 TGridJobStatus *status;
2188 Int_t pid;
2189 for (Int_t ijob=0; ijob<nentries; ijob++) {
2190 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2191 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2192 if (pid<jobidstart) continue;
2193 if (pid == lastid) {
8ddad121 2194 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2195 }
2196 switch (status->GetStatus()) {
2197 case TGridJobStatus::kWAITING:
2198 nwaiting++; break;
2199 case TGridJobStatus::kRUNNING:
2200 nrunning++; break;
2201 case TGridJobStatus::kABORTED:
2202 case TGridJobStatus::kFAIL:
2203 case TGridJobStatus::kUNKNOWN:
2204 nerror++; break;
2205 case TGridJobStatus::kDONE:
2206 ndone++;
2207 }
2208 }
2209 list->Delete();
2210 delete list;
2211 return mstatus;
2212}
2213
c57f56b7 2214//______________________________________________________________________________
2215Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2216{
2217// Returns true if file is a collection. Functionality duplicated from
2218// TAlien::Type() because we don't want to directly depend on TAlien.
2219 if (!gGrid) {
2220 Error("IsCollection", "No connection to grid");
2221 return kFALSE;
2222 }
2223 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2224 if (!res) return kFALSE;
2225 const char* typeStr = res->GetKey(0, "type");
2226 if (!typeStr || !strlen(typeStr)) return kFALSE;
2227 if (!strcmp(typeStr, "collection")) return kTRUE;
2228 delete res;
2229 return kFALSE;
2230}
2231
fe2d7fc2 2232//______________________________________________________________________________
2233Bool_t AliAnalysisAlien::IsSingleOutput() const
2234{
2235// Check if single-ouput option is on.
2236 return (!fOutputSingle.IsNull());
2237}
2238
16a4353c 2239//______________________________________________________________________________
2240void AliAnalysisAlien::Print(Option_t *) const
2241{
2242// Print current plugin settings.
84fcd93f 2243 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2244 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2245 if (mgr && mgr->IsProofMode()) {
2246 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2247 if (TestBit(AliAnalysisGrid::kTest))
2248 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2249 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2250 if (!fProofDataSet.IsNull())
2251 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2252 if (fProofReset==1)
2253 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2254 if (fProofReset>1)
2255 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2256 if (!fRootVersionForProof.IsNull())
2257 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2258 else
2259 printf("= ROOT version requested________________________ default\n");
2260 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2261 if (!fAliRootMode.IsNull())
2262 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2263 if (fNproofWorkers)
2264 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2265 if (fNproofWorkersPerSlave)
2266 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2267 if (TestSpecialBit(kClearPackages))
2268 printf("= ClearPackages requested...\n");
2269 if (fIncludePath.Data())
2270 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2271 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2272 if (fPackages && fPackages->GetEntries()) {
2273 TIter next(fPackages);
2274 TObject *obj;
2275 TString list;
2276 while ((obj=next())) list += obj->GetName();
2277 printf("= Par files to be used: ________________________ %s\n", list.Data());
2278 }
2279 if (TestSpecialBit(kProofConnectGrid))
2280 printf("= Requested PROOF connection to grid\n");
2281 return;
2282 }
e1c22e21 2283 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2284 if (fOverwriteMode) {
2285 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2286 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2287 }
348be253 2288 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2289 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2290 printf("= Production mode:______________________________ %d\n", fProductionMode);
2291 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2292 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2293 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2294 if (fUser.Length())
84fcd93f 2295 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2296 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2297 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2298 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2299 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2300 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2301 if (fRunNumbers.Length())
84fcd93f 2302 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2303 if (fRunRange[0])
74013535 2304 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2305 if (!fRunRange[0] && !fRunNumbers.Length()) {
2306 TIter next(fInputFiles);
2307 TObject *obj;
2308 TString list;
2309 while ((obj=next())) list += obj->GetName();
84fcd93f 2310 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2311 }
2312 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2313 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2314 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2315 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2316 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
d3b18c4c 2317 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2318 printf("=====================================================================\n");
2319 printf("= Job price: ___________________________________ %d\n", fPrice);
2320 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2321 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2322 if (fMaxInitFailed>0)
84fcd93f 2323 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2324 if (fMasterResubmitThreshold>0)
84fcd93f 2325 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2326 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2327 if (fNrunsPerMaster>0)
84fcd93f 2328 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2329 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2330 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2331 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2332 if (fArguments.Length())
84fcd93f 2333 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2334 if (fExecutableArgs.Length())
2335 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2336 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2337 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2338 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2339 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2340 if (fDatasetName)
84fcd93f 2341 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2342 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2343 if (fIncludePath.Data())
84fcd93f 2344 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2345 if (fCloseSE.Length())
84fcd93f 2346 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2347 if (fFriendChainName.Length())
84fcd93f 2348 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2349 if (fPackages && fPackages->GetEntries()) {
16a4353c 2350 TIter next(fPackages);
2351 TObject *obj;
2352 TString list;
2353 while ((obj=next())) list += obj->GetName();
84fcd93f 2354 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2355 }
2356}
2357
c57f56b7 2358//______________________________________________________________________________
2359void AliAnalysisAlien::SetDefaults()
2360{
2361// Set default values for everything. What cannot be filled will be left empty.
2362 if (fGridJDL) delete fGridJDL;
2363 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2364 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2365 fPrice = 1;
2366 fTTL = 30000;
2367 fSplitMaxInputFileNumber = 100;
2368 fMaxInitFailed = 0;
2369 fMasterResubmitThreshold = 0;
bb885a9e 2370 fNtestFiles = 10;
149d288c 2371 fNreplicas = 2;
a8739e8a 2372 fRunRange[0] = 0;
2373 fRunRange[1] = 0;
8e3bd06a 2374 fRunPrefix = "%d";
319593fb 2375 fNrunsPerMaster = 1;
16a4353c 2376 fMaxMergeFiles = 100;
c57f56b7 2377 fRunNumbers = "";
2378 fExecutable = "analysis.sh";
0a1c1f7f 2379 fExecutableCommand = "root -b -q";
c57f56b7 2380 fArguments = "";
631c0b05 2381 fExecutableArgs = "";
c57f56b7 2382 fAnalysisMacro = "myAnalysis.C";
2383 fAnalysisSource = "";
2384 fAdditionalLibs = "";
2385 fSplitMode = "se";
2386 fAPIVersion = "";
2387 fROOTVersion = "";
2388 fAliROOTVersion = "";
2389 fUser = ""; // Your alien user name
2390 fGridWorkingDir = "";
2391 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2392 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2393 fFriendChainName = "";
c57f56b7 2394 fGridOutputDir = "output";
b83f84f0 2395 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2396 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2397 fInputFormat = "xml-single";
2398 fJDLName = "analysis.jdl";
c6cb3634 2399 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2400 fMergeExcludes = "";
0f389141 2401 fMergeViaJDL = 0;
348be253 2402 SetUseCopy(kTRUE);
2403 SetCheckCopy(kTRUE);
149d288c 2404 SetDefaultOutputs(kTRUE);
e1c22e21 2405 fOverwriteMode = 1;
c57f56b7 2406}
2407
0f389141 2408//______________________________________________________________________________
b3e07543 2409Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2410{
b3e07543 2411// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2412 // First check if the result is already in the output directory.
2413 if (FileExists(Form("%s/%s",aliendir,filename))) {
2414 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2415 return kFALSE;
2416 }
b3e07543 2417 // Now check the last stage done.
2418 Int_t stage = 0;
2419 while (1) {
2420 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2421 stage++;
2422 }
2423 // Next stage of merging
2424 stage++;
2425 TString pattern = "*root_archive.zip";
2426 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2427 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2428 if (res) delete res;
2429 // Write standard output to file
2430 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2431 // Count the number of files inside
2432 ifstream ifile;
2433 ifile.open(Form("Stage_%d.xml",stage));
2434 if (!ifile.good()) {
2435 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2436 return kFALSE;
b3e07543 2437 }
2438 TString line;
2439 Int_t nfiles = 0;
2440 while (!ifile.eof()) {
2441 ifile >> line;
2442 if (line.Contains("/event")) nfiles++;
7c2cd90a 2443 }
b3e07543 2444 ifile.close();
2445 if (!nfiles) {
2446 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2447 return kFALSE;
2448 } else {
2449 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2450 }
b3e07543 2451 // Copy the file in the output directory
2452 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2453// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2454 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2455 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2456 // Check if this is the last stage to be done.
2457 Bool_t laststage = (nfiles<nperchunk);
2458 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2459 if (laststage) {
2460 printf("### Submiting final merging stage %d\n", stage);
2461 TString finalJDL = jdl;
2462 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2463 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2464 Int_t jobId = SubmitSingleJob(query);
2465 if (!jobId) return kFALSE;
2466 } else {
2467 printf("### Submiting merging stage %d\n", stage);
03335324 2468 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2469 Int_t jobId = SubmitSingleJob(query);
2470 if (!jobId) return kFALSE;
7c2cd90a 2471 }
b3e07543 2472 return kTRUE;
2473}
7c2cd90a 2474
105631b5 2475//______________________________________________________________________________
2476AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2477{
2478// Loat the analysis manager from a file.
2479 TFile *file = TFile::Open(fname);
2480 if (!file) {
2481 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2482 return 0;
2483 }
2484 TIter nextkey(file->GetListOfKeys());
2485 AliAnalysisManager *mgr = 0;
2486 TKey *key;
2487 while ((key=(TKey*)nextkey())) {
2488 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2489 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2490 }
2491 if (!mgr)
2492 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2493 return mgr;
2494}
2495
7c2cd90a 2496//______________________________________________________________________________
2497Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2498{
2499// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2500 if (!gGrid) return 0;
2501 printf("=> %s ------> ",query);
2502 TGridResult *res = gGrid->Command(query);
2503 if (!res) return 0;
2504 TString jobId = res->GetKey(0,"jobId");
2505 delete res;
2506 if (jobId.IsNull()) {
2507 printf("submission failed. Reason:\n");
2508 gGrid->Stdout();
2509 gGrid->Stderr();
2510 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2511 return 0;
2512 }
2513 printf(" Job id: %s\n", jobId.Data());
2514 return atoi(jobId);
2515}
2516
2517//______________________________________________________________________________
b3e07543 2518Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2519{
b3e07543 2520// Merge given output files from basedir. Basedir can be an alien output directory
2521// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2522// files in a group (ignored for xml input). Merging can be done in stages:
2523// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2524// stage=1 : works with an xml of all root_archive.zip in the output directory
2525// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2526 TString outputFile = output;
0f389141 2527 TString command;
a2f5fc01 2528 TString outputChunk;
2529 TString previousChunk = "";
b3e07543 2530 TObjArray *listoffiles = new TObjArray();
2531// listoffiles->SetOwner();
a2f5fc01 2532 Int_t countChunk = 0;
2533 Int_t countZero = nmaxmerge;
0f389141 2534 Bool_t merged = kTRUE;
a2f5fc01 2535 Int_t index = outputFile.Index("@");
2536 if (index > 0) outputFile.Remove(index);
7c2cd90a 2537 TString inputFile = outputFile;
b3e07543 2538 TString sbasedir = basedir;
2539 if (sbasedir.Contains(".xml")) {
2540 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2541 nmaxmerge = 9999999;
2542 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2543 if (!coll) {
2544 ::Error("MergeOutput", "Input XML collection empty.");
2545 return kFALSE;
2546 }
2547 // Iterate grid collection
2548 while (coll->Next()) {
2549 TString fname = gSystem->DirName(coll->GetTURL());
2550 fname += "/";
2551 fname += inputFile;
2552 listoffiles->Add(new TNamed(fname.Data(),""));
2553 }
2554 } else {
2555 command = Form("find %s/ *%s", basedir, inputFile.Data());
2556 printf("command: %s\n", command.Data());
2557 TGridResult *res = gGrid->Command(command);
2558 if (!res) {
2559 ::Error("MergeOutput","No result for the find command\n");
2560 delete listoffiles;
2561 return kFALSE;
2562 }
2563 TIter nextmap(res);
2564 TMap *map = 0;
2565 while ((map=(TMap*)nextmap())) {
2566 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2567 if (!objs || !objs->GetString().Length()) {
2568 // Nothing found - skip this output
2569 delete res;
2570 delete listoffiles;
2571 return kFALSE;
2572 }
2573 listoffiles->Add(new TNamed(objs->GetName(),""));
2574 }
2575 delete res;
2576 }
2577 if (!listoffiles->GetEntries()) {
7c2cd90a 2578 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2579 delete listoffiles;
0f389141 2580 return kFALSE;
2581 }
2582
2583 TFileMerger *fm = 0;
b3e07543 2584 TIter next0(listoffiles);
2585 TObjArray *listoffilestmp = new TObjArray();
2586 listoffilestmp->SetOwner();
2587 TObject *nextfile;
2588 TString snextfile;
2589 // Keep only the files at upper level
2590 Int_t countChar = 0;
2591 while ((nextfile=next0())) {
2592 snextfile = nextfile->GetName();
2593 Int_t crtCount = snextfile.CountChar('/');
2594 if (nextfile == listoffiles->First()) countChar = crtCount;
2595 if (crtCount < countChar) countChar = crtCount;
2596 }
2597 next0.Reset();
2598 while ((nextfile=next0())) {
2599 snextfile = nextfile->GetName();
2600 Int_t crtCount = snextfile.CountChar('/');
2601 if (crtCount > countChar) {
2602 delete nextfile;
2603 continue;
2604 }
2605 listoffilestmp->Add(nextfile);
2606 }
2607 delete listoffiles;
2608 listoffiles = listoffilestmp; // Now contains 'good' files
2609 listoffiles->Print();
2610 TIter next(listoffiles);
7c2cd90a 2611 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2612 outputChunk = outputFile;
2613 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2614 // Check for existent temporary merge files
e1c22e21 2615 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2616 // Preserve old merging functionality for stage 0.
2617 if (stage==0) {
2618 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2619 while (1) {
2620 // Skip as many input files as in a chunk
2ea55496 2621 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2622 nextfile = next();
2623 if (!nextfile) {
2ea55496 2624 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2625 delete listoffiles;
2ea55496 2626 return kFALSE;
2627 }
b3e07543 2628 snextfile = nextfile->GetName();
7c2cd90a 2629 }
2630 outputChunk = outputFile;
2631 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2632 countChunk++;
2633 if (gSystem->AccessPathName(outputChunk)) continue;
2634 // Merged file with chunks up to <countChunk> found
2635 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2636 previousChunk = outputChunk;
2637 break;
2638 }
2639 }
2640 countZero = nmaxmerge;
2641
b3e07543 2642 while ((nextfile=next())) {
2643 snextfile = nextfile->GetName();
2ea55496 2644 // Loop 'find' results and get next LFN
7c2cd90a 2645 if (countZero == nmaxmerge) {
2646 // First file in chunk - create file merger and add previous chunk if any.
2b895f34 2647 fm = new TFileMerger(kTRUE);
7c2cd90a 2648 fm->SetFastMethod(kTRUE);
2649 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2650 outputChunk = outputFile;
2651 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2652 }
2653 // If last file found, put merged results in the output file
b3e07543 2654 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2655 // Add file to be merged and decrement chunk counter.
b3e07543 2656 fm->AddFile(snextfile);
7c2cd90a 2657 countZero--;
b3e07543 2658 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2659 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2660 // Nothing found - skip this output
2661 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2662 merged = kFALSE;
2663 break;
7c2cd90a 2664 }
2665 fm->OutputFile(outputChunk);
2666 // Merge the outputs, then go to next chunk
2667 if (!fm->Merge()) {
2668 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2669 merged = kFALSE;
2670 break;
7c2cd90a 2671 } else {
2672 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2673 gSystem->Unlink(previousChunk);
2674 }
b3e07543 2675 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2676 countChunk++;
2677 countZero = nmaxmerge;
2678 previousChunk = outputChunk;
0f389141 2679 }
0f389141 2680 }
b3e07543 2681 delete listoffiles;
2682 delete fm;
7c2cd90a 2683 return merged;
2684 }
2685 // Merging stage different than 0.
2686 // Move to the begining of the requested chunk.
2b895f34 2687 fm = new TFileMerger(kTRUE);
7c2cd90a 2688 fm->SetFastMethod(kTRUE);
b3e07543 2689 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2690 delete listoffiles;
7c2cd90a 2691 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2692 // Nothing found - skip this output
2693 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2694 delete fm;
2695 return kFALSE;
2696 }
b3e07543 2697 fm->OutputFile(outputFile);
7c2cd90a 2698 // Merge the outputs
2699 if (!fm->Merge()) {
2700 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2701 delete fm;
2702 return kFALSE;
2703 } else {
b3e07543 2704 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2705 }
7c2cd90a 2706 delete fm;
2707 return kTRUE;
0f389141 2708}
2709
c57f56b7 2710//______________________________________________________________________________
2711Bool_t AliAnalysisAlien::MergeOutputs()
2712{
2713// Merge analysis outputs existing in the AliEn space.
2714 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2715 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2716 if (!Connect()) {
2717 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2718 return kFALSE;
0f389141 2719 }
c9e8f7fd 2720 if (fMergeViaJDL) {
2721 if (!TestBit(AliAnalysisGrid::kMerge)) {
2722 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2723 return kFALSE;
2724 }
2725 if (fProductionMode) {
2726 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2727 return kFALSE;
2728 }
0f389141 2729 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2730 if (!SubmitMerging()) return kFALSE;
0f389141 2731 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2732 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2733 return kFALSE;
c57f56b7 2734 }
2735 // Get the output path
dc030a24 2736 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2737 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2738 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2739 return kFALSE;
2740 }
2741 if (!fOutputFiles.Length()) {
2742 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2743 return kFALSE;
0f389141 2744 }
2745 // Check if fast read option was requested
e1c22e21 2746 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2747 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2748 if (fFastReadOption) {
d3339be3 2749 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2750 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2751 gEnv->SetValue("XNet.ConnectTimeout",50);
2752 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2753 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2754 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2755 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2756 }
e8b839ab 2757 // Make sure we change the temporary directory
2758 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2759 // Set temporary compilation directory to current one
2760 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2761 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2762 TIter next(list);
2763 TObjString *str;
a2f5fc01 2764 TString outputFile;
c57f56b7 2765 Bool_t merged = kTRUE;
2766 while((str=(TObjString*)next())) {
a2f5fc01 2767 outputFile = str->GetString();
2768 Int_t index = outputFile.Index("@");
2769 if (index > 0) outputFile.Remove(index);
2770 TString outputChunk = outputFile;
2771 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2772 // Skip already merged outputs
a2f5fc01 2773 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2774 if (fOverwriteMode) {
a2f5fc01 2775 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2776 gSystem->Unlink(outputFile);
2777 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2778 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2779 outputChunk.Data());
2780 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2781 }
2782 } else {
a2f5fc01 2783 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2784 continue;
2785 }
2786 } else {
a2f5fc01 2787 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2788 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2789 outputChunk.Data());
2790 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2791 }
2792 }
bb885a9e 2793 if (fMergeExcludes.Length() &&
a2f5fc01 2794 fMergeExcludes.Contains(outputFile.Data())) continue;
16a4353c 2795 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2796 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2797 if (!merged) {
2798 Error("MergeOutputs", "Terminate() will NOT be executed");
2799 return kFALSE;
ff07ec61 2800 }
2801 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2802 if (fileOpened) fileOpened->Close();
c57f56b7 2803 }
0f389141 2804 return kTRUE;
c57f56b7 2805}
2806
bb885a9e 2807//______________________________________________________________________________
2808void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2809{
2810// Use the output files connected to output containers from the analysis manager
2811// rather than the files defined by SetOutputFiles
2812 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2813 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2814 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2815}
2816
149d288c 2817//______________________________________________________________________________
2818void AliAnalysisAlien::SetOutputFiles(const char *list)
2819{
2820// Manually set the output files list.
2821// Removes duplicates. Not allowed if default outputs are not disabled.
2822 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2823 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2824 return;
2825 }
2826 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2827 fOutputFiles = "";
2828 TString slist = list;
2829 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2830 TObjArray *arr = slist.Tokenize(" ");
2831 TObjString *os;
2832 TIter next(arr);
2833 TString sout;
2834 while ((os=(TObjString*)next())) {
2835 sout = os->GetString();
2836 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2837 if (fOutputFiles.Contains(sout)) continue;
2838 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2839 fOutputFiles += sout;
2840 }
2841 delete arr;
f790bc1b 2842}
149d288c 2843
2844//______________________________________________________________________________
2845void AliAnalysisAlien::SetOutputArchive(const char *list)
2846{
2847// Manually set the output archive list. Free text - you are on your own...
2848// Not allowed if default outputs are not disabled.
2849 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2850 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2851 return;
2852 }
2853 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2854 fOutputArchive = list;
2855}
2856
2857//______________________________________________________________________________
2858void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2859{
2860// Setting a prefered output SE is not allowed anymore.
2861 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2862}
2863
28ce6c9c 2864//______________________________________________________________________________
2865void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
2866{
2867// Set some PROOF special parameter.
2868 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2869 if (pair) {
2870 TObject *old = pair->Key();
2871 TObject *val = pair->Value();
2872 fProofParam.Remove(old);
2873 delete old;
2874 delete val;
2875 }
2876 fProofParam.Add(new TObjString(pname), new TObjString(value));
2877}
2878
2879//______________________________________________________________________________
2880const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
2881{
2882// Returns a special PROOF parameter.
2883 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2884 if (!pair) return 0;
2885 return pair->Value()->GetName();
2886}
2887
5513444a 2888//______________________________________________________________________________
2889Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2890{
2891// Start remote grid analysis.
3bdcb562 2892 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2893 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2894 if (!mgr || !mgr->IsInitialized()) {
2895 Error("StartAnalysis", "You need an initialized analysis manager for this");
2896 return kFALSE;
2897 }
2898 // Are we in PROOF mode ?
2899 if (mgr->IsProofMode()) {
baa34407 2900 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
2901 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2902 if (fProofCluster.IsNull()) {
2903 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2904 return kFALSE;
2905 }
2906 if (fProofDataSet.IsNull() && !testMode) {
2907 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2908 return kFALSE;
2909 }
2910 // Set the needed environment
2911 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2912 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2913 if (fProofReset && !testMode) {
2914 if (fProofReset==1) {
2915 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2916 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2917 } else {
2918 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2919 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2920 }
2921 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2922 return kFALSE;
2923 }
baa34407 2924
2925 if (!testMode) {
2926 // Check if there is an old active session
2927 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
2928 if (nsessions) {
2929 Error("StartAnalysis","You have to reset your old session first\n");
2930 return kFALSE;
2931 }
53911ca6 2932 }
3bdcb562 2933 // Do we need to change the ROOT version ? The success of this cannot be checked.
2934 if (!fRootVersionForProof.IsNull() && !testMode) {
2935 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2936 fProofCluster.Data(), fRootVersionForProof.Data()));
2937 }
2938 // Connect to PROOF and check the status
2939 Long_t proof = 0;
27734f0e 2940 TString sworkers;
2941 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2942 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2943 if (!testMode) {
27734f0e 2944 if (!sworkers.IsNull())
2945 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2946 else
2947 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2948 } else {
2949 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2950 if (!proof) {
2951 Error("StartAnalysis", "Could not start PROOF in test mode");
2952 return kFALSE;
2953 }
2954 }
2955 if (!proof) {
2956 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2957 return kFALSE;
2958 }
27734f0e 2959 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2960 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 2961 // Set proof special parameters if any
2962 TIter nextpp(&fProofParam);
2963 TObject *proofparam;
2964 while ((proofparam=nextpp())) {
2965 TString svalue = GetProofParameter(proofparam->GetName());
2966 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
2967 }
3bdcb562 2968 // Is dataset existing ?
2969 if (!testMode) {
2970 TString dataset = fProofDataSet;
2971 Int_t index = dataset.Index("#");
2972 if (index>=0) dataset.Remove(index);
830acc4c 2973// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
2974// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
2975// return kFALSE;
2976// }
2977// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 2978 }
2979 // Is ClearPackages() needed ?
2980 if (TestSpecialBit(kClearPackages)) {
2981 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
2982 gROOT->ProcessLine("gProof->ClearPackages();");
2983 }
2984 // Is a given aliroot mode requested ?
2985 TList optionsList;
1ac846e9 2986 TString parLibs;
830acc4c 2987 if (!fAliRootMode.IsNull()) {
3bdcb562 2988 TString alirootMode = fAliRootMode;
2989 if (alirootMode == "default") alirootMode = "";
2990 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
2991 optionsList.SetOwner();
2992 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
2993 // Check the additional libs to be loaded
2994 TString extraLibs;
1ac846e9 2995 Bool_t parMode = kFALSE;
dc1ffbcb 2996 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 2997 // Parse the extra libs for .so
2998 if (fAdditionalLibs.Length()) {
2999 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3000 TIter next(list);
3001 TObjString *str;
1ac846e9 3002 while((str=(TObjString*)next())) {
3003 if (str->GetString().Contains(".so")) {
3004 if (parMode) {
3005 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3006 break;
3007 }
3008 TString stmp = str->GetName();
3009 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3010 stmp.ReplaceAll(".so","");
3011 if (!extraLibs.IsNull()) extraLibs += ":";
3012 extraLibs += stmp;
3013 continue;
3014 }
3015 if (str->GetString().Contains(".par")) {
3016 // The first par file found in the list will not allow any further .so
3017 parMode = kTRUE;
3018 if (!parLibs.IsNull()) parLibs += ":";
3019 parLibs += str->GetName();
3020 continue;
3021 }
3bdcb562 3022 }
3023 if (list) delete list;
3024 }
dc030a24 3025 if (!extraLibs.IsNull()) {
3026 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3027 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3028 }
830acc4c 3029 // Check extra includes
3030 if (!fIncludePath.IsNull()) {
3031 TString includePath = fIncludePath;
3032 includePath.ReplaceAll(" ",":");
dc030a24 3033 includePath.ReplaceAll("$ALICE_ROOT/","");
3034 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3035 includePath.ReplaceAll("-I","");
dc030a24 3036 includePath.Remove(TString::kTrailing, ':');
830acc4c 3037 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3038 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3039 }
27734f0e 3040 // Check if connection to grid is requested
3041 if (TestSpecialBit(kProofConnectGrid))
3042 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3043 // Enable AliRoot par
3044 if (testMode) {
3045 // Enable proof lite package
3046 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3047 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3048 TNamed *obj = (TNamed*)optionsList.At(i);
3049 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3050 }
3051 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3052 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3053 Info("StartAnalysis", "AliRootProofLite enabled");
3054 } else {
3055 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3056 return kFALSE;
3057 }
3058 } else {
dc030a24 3059 if ( ! fAliROOTVersion.IsNull() ) {
3060 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3061 fAliROOTVersion.Data(), &optionsList))) {
3062 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3063 return kFALSE;
3064 }
3065 }
27734f0e 3066 }
1ac846e9 3067 // Enable first par files from fAdditionalLibs
3068 if (!parLibs.IsNull()) {
3069 TObjArray *list = parLibs.Tokenize(":");
3070 TIter next(list);
3071 TObjString *package;
3072 while((package=(TObjString*)next())) {
40501bb7 3073 TString spkg = package->GetName();
3074 spkg.ReplaceAll(".par", "");
3075 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3076 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3077 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3078 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3079 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3080 return kFALSE;
3081 }
3082 } else {
3083 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3084 return kFALSE;
3085 }
3086 }
3087 if (list) delete list;
3088 }
3bdcb562 3089 } else {
3090 if (fAdditionalLibs.Contains(".so") && !testMode) {
3091 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3092 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3093 return kFALSE;
3094 }
3095 }
3096 // Enable par files if requested
3097 if (fPackages && fPackages->GetEntries()) {
3098 TIter next(fPackages);
3099 TObject *package;
3100 while ((package=next())) {
1ac846e9 3101 // Skip packages already enabled
3102 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3103 TString spkg = package->GetName();
3104 spkg.ReplaceAll(".par", "");
3105 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3106 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3107 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3108 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3109 return kFALSE;
3110 }
3111 } else {
3112 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3113 return kFALSE;
3114 }
3115 }
3116 }
3117 // Do we need to load analysis source files ?
3118 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3119 if (fAnalysisSource.Length()) {
3120 TObjArray *list = fAnalysisSource.Tokenize(" ");
3121 TIter next(list);
3122 TObjString *str;
3123 while((str=(TObjString*)next())) {
3124 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3125 }
3bdcb562 3126 if (list) delete list;
3127 }
830acc4c 3128 if (testMode) {
3129 // Register dataset to proof lite.
3130 if (fFileForTestMode.IsNull()) {
3131 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3132 return kFALSE;
3133 }
3134 if (gSystem->AccessPathName(fFileForTestMode)) {
3135 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3136 return kFALSE;
3137 }
3138 TFileCollection *coll = new TFileCollection();
3139 coll->AddFromFile(fFileForTestMode);
8ddad121 3140 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3141 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3142 }
3bdcb562 3143 return kTRUE;
3144 }
c57f56b7 3145
43da816a 3146 // Check if output files have to be taken from the analysis manager
3147 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3148 // Add output files and AOD files
3149 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3150 // Add extra files registered to the analysis manager
f790bc1b 3151 TString extra = GetListOfFiles("ext");
3152 if (!extra.IsNull()) {
149d288c 3153 extra.ReplaceAll(".root", "*.root");
f790bc1b 3154 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3155 fOutputFiles += extra;
84fcd93f 3156 }
149d288c 3157 // Compose the output archive.
b83f84f0 3158 fOutputArchive = "log_archive.zip:std*@disk=1 ";
3159 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3160 }
f7b1cbc2 3161// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3162 if (TestBit(AliAnalysisGrid::kOffline)) {
3163 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3164 \n there nor any job run. You can revise the JDL and analysis \
3165 \n macro then run the same in \"submit\" mode.");
3166 } else if (TestBit(AliAnalysisGrid::kTest)) {
3167 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3168 \n dataset.");
3169 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3170 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3171 \n space and job submitted.");
3172 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3173 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3174 if (fMergeViaJDL) CheckInputData();
5513444a 3175 return kTRUE;
c57f56b7 3176 } else {
3177 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3178 }
3179
348be253 3180 Print();
c57f56b7 3181 if (!Connect()) {
3182 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3183 return kFALSE;
16a4353c 3184 }
d3b18c4c 3185 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3186 if (!CheckInputData()) {
3187 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3188 return kFALSE;
c57f56b7 3189 }
d3339be3 3190 if (!CreateDataset(fDataPattern)) {
3191 TString serror;
3192 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3193 if (fRunNumbers.Length()) serror = "run numbers";
3194 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3195 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3196 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3197 return kFALSE;
3198 }
21ca8e59 3199 WriteAnalysisFile();
c57f56b7 3200 WriteAnalysisMacro();
3201 WriteExecutable();
3202 WriteValidationScript();
0f389141 3203 if (fMergeViaJDL) {
3204 WriteMergingMacro();
3205 WriteMergeExecutable();
3206 WriteValidationScript(kTRUE);
3207 }
5513444a 3208 if (!CreateJDL()) return kFALSE;
3209 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3210 if (testMode) {
c57f56b7 3211 // Locally testing the analysis
3212 Info("StartAnalysis", "\n_______________________________________________________________________ \
3213 \n Running analysis script in a daughter shell as on a worker node \
3214 \n_______________________________________________________________________");
149d288c 3215 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3216 TIter next(list);
3217 TObjString *str;
a2f5fc01 3218 TString outputFile;
c57f56b7 3219 while((str=(TObjString*)next())) {
a2f5fc01 3220 outputFile = str->GetString();
3221 Int_t index = outputFile.Index("@");
3222 if (index > 0) outputFile.Remove(index);
3223 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3224 }
3225 delete list;
3226 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3227 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3228// gSystem->Exec("cat stdout");
5513444a 3229 return kFALSE;
c57f56b7 3230 }
5513444a 3231 // Check if submitting is managed by LPM manager
a3e84053 3232 if (fProductionMode) {
5513444a 3233 TString prodfile = fJDLName;
3234 prodfile.ReplaceAll(".jdl", ".prod");
3235 WriteProductionFile(prodfile);
3236 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3237 return kFALSE;
3238 }
a8739e8a 3239 // Submit AliEn job(s)
d2a409b2 3240 gGrid->Cd(fGridOutputDir);
a8739e8a 3241 TGridResult *res;
c57f56b7 3242 TString jobID = "";
d2a409b2 3243 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3244 // Submit a given xml or a set of runs
a8739e8a 3245 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3246 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3247 if (res) {
3248 const char *cjobId = res->GetKey(0,"jobId");
3249 if (!cjobId) {
a03be957 3250 gGrid->Stdout();
3251 gGrid->Stderr();
a8739e8a 3252 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3253 return kFALSE;
a8739e8a 3254 } else {
3255 Info("StartAnalysis", "\n_______________________________________________________________________ \
3256 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3257 \n_______________________________________________________________________",
3258 fJDLName.Data(), cjobId);
3259 jobID = cjobId;
3260 }
3261 delete res;
a03be957 3262 } else {
3263 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3264 return kFALSE;
a8739e8a 3265 }
3266 } else {
d2a409b2 3267 // Submit for a range of enumeration of runs.
a03be957 3268 if (!Submit()) return kFALSE;
c57f56b7 3269 }
a8739e8a 3270
c57f56b7 3271 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3272 \n You may exit at any time and terminate the job later using the option <terminate> \
3273 \n ##################################################################################", jobID.Data());
bb885a9e 3274 gSystem->Exec("aliensh");
5513444a 3275 return kTRUE;
c57f56b7 3276}
3277
f790bc1b 3278//______________________________________________________________________________
3279const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3280{
3281// Get a comma-separated list of output files of the requested type.
3282// Type can be (case unsensitive):
3283// aod - list of aod files (std, extensions and filters)
3284// out - list of output files connected to containers (but not aod's or extras)
3285// ext - list of extra files registered to the manager
3286// ter - list of files produced in terminate
3287 static TString files;
3288 files = "";
3289 TString stype = type;
3290 stype.ToLower();
3291 TString aodfiles, extra;
3292 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3293 if (!mgr) {
3294 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3295 return files.Data();
3296 }
3297 if (mgr->GetOutputEventHandler()) {
3298 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3299 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3300 if (!extraaod.IsNull()) {
3301 aodfiles += ",";
3302 aodfiles += extraaod;
3303 }
3304 }
3305 if (stype.Contains("aod")) {
3306 files = aodfiles;
3307 if (stype == "aod") return files.Data();
3308 }
3309 // Add output files that are not in the list of AOD files
3310 TString outputfiles = "";
3311 TIter next(mgr->GetOutputs());
3312 AliAnalysisDataContainer *output;
3313 const char *filename = 0;
3314 while ((output=(AliAnalysisDataContainer*)next())) {
3315 filename = output->GetFileName();
3316 if (!(strcmp(filename, "default"))) continue;
3317 if (outputfiles.Contains(filename)) continue;
3318 if (aodfiles.Contains(filename)) continue;
3319 if (!outputfiles.IsNull()) outputfiles += ",";
3320 outputfiles += filename;
3321 }
3322 if (stype.Contains("out")) {
3323 if (!files.IsNull()) files += ",";
3324 files += outputfiles;
3325 if (stype == "out") return files.Data();
3326 }
3327 // Add extra files registered to the analysis manager
3328 TString sextra;
3329 extra = mgr->GetExtraFiles();
3330 if (!extra.IsNull()) {
3331 extra.Strip();
3332 extra.ReplaceAll(" ", ",");
3333 TObjArray *fextra = extra.Tokenize(",");
3334 TIter nextx(fextra);
3335 TObject *obj;
3336 while ((obj=nextx())) {
3337 if (aodfiles.Contains(obj->GetName())) continue;
3338 if (outputfiles.Contains(obj->GetName())) continue;
3339 if (sextra.Contains(obj->GetName())) continue;
3340 if (!sextra.IsNull()) sextra += ",";
3341 sextra += obj->GetName();
3342 }
3343 delete fextra;
3344 if (stype.Contains("ext")) {
3345 if (!files.IsNull()) files += ",";
3346 files += sextra;
3347 }
3348 }
3349 if (stype == "ext") return files.Data();
3350 TString termfiles;
3351 if (!fTerminateFiles.IsNull()) {
3352 fTerminateFiles.Strip();
3353 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3354 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3355 TIter nextx(fextra);
3356 TObject *obj;
3357 while ((obj=nextx())) {
3358 if (aodfiles.Contains(obj->GetName())) continue;
3359 if (outputfiles.Contains(obj->GetName())) continue;
3360 if (termfiles.Contains(obj->GetName())) continue;
3361 if (sextra.Contains(obj->GetName())) continue;
3362 if (!termfiles.IsNull()) termfiles += ",";
3363 termfiles += obj->GetName();
3364 }
3365 delete fextra;
3366 }
3367 if (stype.Contains("ter")) {
3368 if (!files.IsNull() && !termfiles.IsNull()) {
3369 files += ",";
3370 files += termfiles;
3371 }
3372 }
3373 return files.Data();
3374}
3375
d2a409b2 3376//______________________________________________________________________________
a03be957 3377Bool_t AliAnalysisAlien::Submit()
d2a409b2 3378{
3379// Submit all master jobs.
3380 Int_t nmasterjobs = fInputFiles->GetEntries();
3381 Long_t tshoot = gSystem->Now();
a03be957 3382 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3383 while (fNsubmitted < nmasterjobs) {
3384 Long_t now = gSystem->Now();
3385 if ((now-tshoot)>30000) {
3386 tshoot = now;
a03be957 3387 if (!SubmitNext()) return kFALSE;
d2a409b2 3388 }
3389 }
a03be957 3390 return kTRUE;
d2a409b2 3391}
3392
0f389141 3393//______________________________________________________________________________
a03be957 3394Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3395{
3396// Submit all merging jobs.
dc030a24 3397 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3398 gGrid->Cd(fGridOutputDir);
3399 TString mergeJDLName = fExecutable;
3400 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3401 if (!fInputFiles) {
3402 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3403 return kFALSE;
3404 }
0f389141 3405 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3406 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3407 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3408 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3409 if (fOutputToRunNo) {
3410 // The output directory is the run number
3411 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3412 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3413 } else {
17b4e9a4 3414 if (!fRunNumbers.Length() && !fRunRange[0]) {
3415 // The output directory is the grid outdir
3416 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3417 runOutDir = fGridOutputDir;
3418 } else {
3419 // The output directory is the master number in 3 digits format
3420 printf("### Submitting merging job for master <%03d>\n", i);
3421 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3422 }
7c2cd90a 3423 }
3424 // Check now the number of merging stages.
2b222f7c 3425 TObjArray *list = fOutputFiles.Tokenize(",");
3426 TIter next(list);
3427 TObjString *str;
3428 TString outputFile;
3429 while((str=(TObjString*)next())) {
3430 outputFile = str->GetString();
3431 Int_t index = outputFile.Index("@");
3432 if (index > 0) outputFile.Remove(index);
3433 if (!fMergeExcludes.Contains(outputFile)) break;
3434 }
3435 delete list;
b3e07543 3436 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3437 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3438 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3439 }
a03be957 3440 if (!ntosubmit) return kTRUE;
9f020952 3441 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3442 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3443 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3444 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3445 \n ################################################################################################################");
0f389141 3446 gSystem->Exec("aliensh");
a03be957 3447 return kTRUE;
0f389141 3448}
3449
d2a409b2 3450//______________________________________________________________________________
a03be957 3451Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3452{
08d5b699 3453// Submit next bunch of master jobs if the queue is free. The first master job is
3454// submitted right away, while the next will not be unless the previous was split.
3455// The plugin will not submit new master jobs if there are more that 500 jobs in
3456// waiting phase.
d2a409b2 3457 static Bool_t iscalled = kFALSE;
3458 static Int_t firstmaster = 0;
3459 static Int_t lastmaster = 0;
3460 static Int_t npermaster = 0;
a03be957 3461 if (iscalled) return kTRUE;
d2a409b2 3462 iscalled = kTRUE;
3463 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3464 Int_t ntosubmit = 0;
3465 TGridResult *res;
3466 TString jobID = "";
3bdcb562 3467 Int_t nmasterjobs = fInputFiles->GetEntries();
3468 if (!fNsubmitted) {
3469 ntosubmit = 1;
3470 if (!IsUseSubmitPolicy()) {
90d50a8c 3471 if (nmasterjobs>5)
3bdcb562 3472 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3473 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3474 ntosubmit = nmasterjobs;
3475 }
3476 } else {
d2a409b2 3477 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3478 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3479 // If last master not split, just return
a03be957 3480 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3481 // No more than 100 waiting jobs
08d5b699 3482 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3483 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3484 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3485 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3486 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3487 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3488 }
d2a409b2 3489 for (Int_t i=0; i<ntosubmit; i++) {
3490 // Submit for a range of enumeration of runs.
a03be957 3491 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3492 TString query;
cd11251e 3493 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3494 runOutDir.ReplaceAll(".xml", "");
3495 if (fOutputToRunNo)
3496 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3497 else
3498 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3499 printf("********* %s\n",query.Data());
d2a409b2 3500 res = gGrid->Command(query);
3501 if (res) {
98ca124f 3502 TString cjobId1 = res->GetKey(0,"jobId");
3503 if (!cjobId1.Length()) {
d2a409b2 3504 iscalled = kFALSE;
a03be957 3505 gGrid->Stdout();
3506 gGrid->Stderr();
3507 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3508 return kFALSE;
d2a409b2 3509 } else {
3510 Info("StartAnalysis", "\n_______________________________________________________________________ \
3511 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3512 \n_______________________________________________________________________",
98ca124f 3513 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 3514 jobID += cjobId1;
3515 jobID += " ";
98ca124f 3516 lastmaster = cjobId1.Atoi();
d2a409b2 3517 if (!firstmaster) firstmaster = lastmaster;
3518 fNsubmitted++;
3519 }
3520 delete res;
a03be957 3521 } else {
3522 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3523 return kFALSE;
d2a409b2 3524 }
3525 }
3526 iscalled = kFALSE;
a03be957 3527 return kTRUE;
d2a409b2 3528}
3529
c57f56b7 3530//______________________________________________________________________________
3531void AliAnalysisAlien::WriteAnalysisFile()
3532{
f10e8481 3533// Write current analysis manager into the file <analysisFile>
3534 TString analysisFile = fExecutable;
3535 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3536 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3537 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3538 if (!mgr || !mgr->IsInitialized()) {
3539 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3540 return;
3541 }
3542 // Check analysis type
3543 TObject *handler;
3544 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3545 handler = (TObject*)mgr->GetInputEventHandler();
3546 if (handler) {
65eb22f8 3547 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3548 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3549 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3550 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3551 } else {
3552 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3553 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3554 }
c57f56b7 3555 }
3556 TDirectory *cdir = gDirectory;
f10e8481 3557 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3558 if (file) {
e16a394c 3559 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3560 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3561 // Unless merging makes no sense
3562 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3563 mgr->Write();
3564 delete file;
fe2d7fc2 3565 // Enable termination for local jobs
3566 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3567 }
3568 if (cdir) cdir->cd();
f10e8481 3569 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3570 }
3571 Bool_t copy = kTRUE;
d3b18c4c 3572 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3573 if (copy) {
3574 CdWork();
3575 TString workdir = gGrid->GetHomeDirectory();
3576 workdir += fGridWorkingDir;
d3b18c4c 3577 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3578 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3579 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3580 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3581 }
3582}
3583
3584//______________________________________________________________________________
3585void AliAnalysisAlien::WriteAnalysisMacro()
3586{
3587// Write the analysis macro that will steer the analysis in grid mode.
3588 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3589 ofstream out;
3590 out.open(fAnalysisMacro.Data(), ios::out);
3591 if (!out.good()) {
3592 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3593 return;
3594 }
5b273635 3595 Bool_t hasSTEERBase = kFALSE;
3596 Bool_t hasESD = kFALSE;
3597 Bool_t hasAOD = kFALSE;
3598 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3599 Bool_t hasOADB = kFALSE;
5b273635 3600 Bool_t hasANALYSISalice = kFALSE;
3601 Bool_t hasCORRFW = kFALSE;
c57f56b7 3602 TString func = fAnalysisMacro;
3603 TString type = "ESD";
3604 TString comment = "// Analysis using ";
c85cfc0f 3605 if (IsUseMCchain()) {
3606 type = "MC";
3607 comment += "MC";
3608 } else {
3609 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3610 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3611 type = "AOD";
3612 comment += "AOD";
3613 }
3614 }
0df6ccf2 3615 if (type!="AOD" && fFriendChainName!="") {
3616 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3617 return;
3618 }
c57f56b7 3619 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3620 else comment += " data";
3621 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3622 func.ReplaceAll(".C", "");
3623 out << "void " << func.Data() << "()" << endl;
3624 out << "{" << endl;
3625 out << comment.Data() << endl;
3626 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3627 out << " TStopwatch timer;" << endl;
3628 out << " timer.Start();" << endl << endl;
13f0c00b 3629 // Change temp directory to current one
3630 out << "// Set temporary merging directory to current one" << endl;
3631 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3632 out << "// Set temporary compilation directory to current one" << endl;
3633 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3634 // Reset existing include path
3635 out << "// Reset existing include path and add current directory first in the search" << endl;
3636 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3637 if (!fExecutableCommand.Contains("aliroot")) {
3638 out << "// load base root libraries" << endl;
3639 out << " gSystem->Load(\"libTree\");" << endl;
3640 out << " gSystem->Load(\"libGeom\");" << endl;
3641 out << " gSystem->Load(\"libVMC\");" << endl;
3642 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3643 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3644 }
d5c6455a 3645 if (fAdditionalRootLibs.Length()) {
47a4137d 3646 // in principle libtree /lib geom libvmc etc. can go into this list, too
3647 out << "// Add aditional libraries" << endl;
3648 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3649 TIter next(list);
3650 TObjString *str;
3651 while((str=(TObjString*)next())) {
3652 if (str->GetString().Contains(".so"))
3653 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3654 }
47a4137d 3655 if (list) delete list;
d5c6455a 3656 }
57377eb5 3657 out << "// Load analysis framework libraries" << endl;
652561d5 3658 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3659 if (!fPackages) {
f790bc1b 3660 if (!fExecutableCommand.Contains("aliroot")) {
3661 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3662 out << " gSystem->Load(\"libESD\");" << endl;
3663 out << " gSystem->Load(\"libAOD\");" << endl;
3664 }
4e5c5506 3665 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3666 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3667 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3668 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3669 } else {
4e5c5506 3670 TIter next(fPackages);
3671 TObject *obj;
57377eb5 3672 TString pkgname;
57377eb5 3673 while ((obj=next())) {
3674 pkgname = obj->GetName();
4478e6f1 3675 if (pkgname == "STEERBase" ||
3676 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3677 if (pkgname == "ESD" ||
3678 pkgname == "ESD.par") hasESD = kTRUE;
3679 if (pkgname == "AOD" ||
3680 pkgname == "AOD.par") hasAOD = kTRUE;
3681 if (pkgname == "ANALYSIS" ||
3682 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 3683 if (pkgname == "OADB" ||
3684 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 3685 if (pkgname == "ANALYSISalice" ||
3686 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3687 if (pkgname == "CORRFW" ||
3688 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3689 }
3690 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3691 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3692 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3693 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3694 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3695 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3696 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3697 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3698 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 3699 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
3700 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 3701 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3702 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3703 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3704 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3705 out << "// Compile other par packages" << endl;
3706 next.Reset();
fcc9bb6f 3707 while ((obj=next())) {
3708 pkgname = obj->GetName();
4478e6f1 3709 if (pkgname == "STEERBase" ||
3710 pkgname == "STEERBase.par" ||
3711 pkgname == "ESD" ||
3712 pkgname == "ESD.par" ||
3713 pkgname == "AOD" ||
3714 pkgname == "AOD.par" ||
3715 pkgname == "ANALYSIS" ||
3716 pkgname == "ANALYSIS.par" ||
7ea40ac8 3717 pkgname == "OADB" ||
3718 pkgname == "OADB.par" ||
4478e6f1 3719 pkgname == "ANALYSISalice" ||
3720 pkgname == "ANALYSISalice.par" ||
3721 pkgname == "CORRFW" ||
3722 pkgname == "CORRFW.par") continue;
5b273635 3723 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3724 }
4e5c5506 3725 }
b3e07543 3726 out << "// include path" << endl;
3727 // Get the include path from the interpreter and remove entries pointing to AliRoot
3728 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3729 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3730 out << " TIter nextpath(listpaths);" << endl;
3731 out << " TObjString *pname;" << endl;
3732 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3733 out << " TString current = pname->GetName();" << endl;
3734 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3735 out << " gSystem->AddIncludePath(current);" << endl;
3736 out << " }" << endl;
3737 out << " if (listpaths) delete listpaths;" << endl;
3738 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3739 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3740 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3741 if (fAdditionalLibs.Length()) {
3742 out << "// Add aditional AliRoot libraries" << endl;
3743 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3744 TIter next(list);
3745 TObjString *str;
3746 while((str=(TObjString*)next())) {
3747 if (str->GetString().Contains(".so"))
3748 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3749 if (str->GetString().Contains(".par"))
3750 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3751 }
3752 if (list) delete list;
3753 }
3754 out << endl;
c57f56b7 3755 out << "// analysis source to be compiled at runtime (if any)" << endl;
3756 if (fAnalysisSource.Length()) {
3757 TObjArray *list = fAnalysisSource.Tokenize(" ");
3758 TIter next(list);
3759 TObjString *str;
3760 while((str=(TObjString*)next())) {
3761 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3762 }
3763 if (list) delete list;
3764 }
3765 out << endl;
b3e07543 3766// out << " printf(\"Currently load libraries:\\n\");" << endl;
3767// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3768 if (fFastReadOption) {
d3339be3 3769 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3770 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3771 out << "// fast xrootd reading enabled" << endl;
3772 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3773 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3774 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3775 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3776 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3777 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 3778 }
3779 if (!IsLocalTest()) {
3780 out << "// connect to AliEn and make the chain" << endl;
3781 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
0f389141 3782 }
c57f56b7 3783 out << "// read the analysis manager from file" << endl;
f10e8481 3784 TString analysisFile = fExecutable;
3785 analysisFile.ReplaceAll(".sh", ".root");
105631b5 3786 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
3787 << analysisFile << "\");" << endl;
3788 out << " if (!mgr) return;" << endl;
21ca8e59 3789 if (IsLocalTest()) {
3790 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
3791 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 3792 if (fFileForTestMode.IsNull())
3793 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
3794 else
3795 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 3796 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
21ca8e59 3797 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 3798 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 3799 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 3800 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
3801 } else {
7a366e0b 3802 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 3803 out << " mgr->SetNSysInfo(100);" << endl;
3804 }
21ca8e59 3805 }
c57f56b7 3806 out << " mgr->PrintStatus();" << endl;
52b6a92b 3807 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3808 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3809 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3810 } else {
9a8288ff 3811 if (TestBit(AliAnalysisGrid::kTest))
3812 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3813 else
3814 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3815 }
3816 }
21ca8e59 3817 if (!IsLocalTest()) {
242accb2 3818 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 3819 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
3820 } else {
3821 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 3822 }
f7498086 3823 out << " timer.Stop();" << endl;
3824 out << " timer.Print();" << endl;
c57f56b7 3825 out << "}" << endl << endl;
21ca8e59 3826 if (!IsLocalTest()) {
fcc9bb6f 3827 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3828 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3829 out << "{" << endl;
3830 out << "// Create a chain using url's from xml file" << endl;
242accb2 3831 out << " TString filename;" << endl;
3832 out << " Int_t run = 0;" << endl;
c85cfc0f 3833 if (IsUseMCchain()) {
3834 out << " TString treename = \"TE\";" << endl;
3835 } else {
3836 out << " TString treename = type;" << endl;
3837 out << " treename.ToLower();" << endl;
3838 out << " treename += \"Tree\";" << endl;
3839 }
e02fee64 3840 out << " printf(\"***************************************\\n\");" << endl;
3841 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3842 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3843 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3844 out << " if (!coll) {" << endl;
3845 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3846 out << " return NULL;" << endl;
3847 out << " }" << endl;
242accb2 3848 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3849 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3850 if(fFriendChainName!="") {
3851 out << " TChain *chainFriend = new TChain(treename);" << endl;
3852 }
c57f56b7 3853 out << " coll->Reset();" << endl;
0df6ccf2 3854 out << " while (coll->Next()) {" << endl;
242accb2 3855 out << " filename = coll->GetTURL("");" << endl;
3856 out << " if (mgr) {" << endl;
3857 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3858 out << " if (nrun && nrun != run) {" << endl;
3859 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3860 out << " mgr->SetRunFromPath(nrun);" << endl;
3861 out << " run = nrun;" << endl;
3862 out << " }" << endl;
3863 out << " }" << endl;
3864 out << " chain->Add(filename);" << endl;
0df6ccf2 3865 if(fFriendChainName!="") {
3866 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3867 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3868 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3869 out << " chainFriend->Add(fileFriend.Data());" << endl;
3870 }
3871 out << " }" << endl;
c57f56b7 3872 out << " if (!chain->GetNtrees()) {" << endl;
3873 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3874 out << " return NULL;" << endl;
3875 out << " }" << endl;
0df6ccf2 3876 if(fFriendChainName!="") {
3877 out << " chain->AddFriend(chainFriend);" << endl;
3878 }
c57f56b7 3879 out << " return chain;" << endl;
fcc9bb6f 3880 out << "}" << endl << endl;
c57f56b7 3881 }
5b273635 3882 if (hasANALYSISalice) {
3883 out <<"//________________________________________________________________________________" << endl;
3884 out << "Bool_t SetupPar(const char *package) {" << endl;
3885 out << "// Compile the package and set it up." << endl;
3886 out << " TString pkgdir = package;" << endl;
3887 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3888 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3889 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3890 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3891 out << " // Check for BUILD.sh and execute" << endl;
3892 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3893 out << " printf(\"*******************************\\n\");" << endl;
3894 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3895 out << " printf(\"*******************************\\n\");" << endl;
3896 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3897 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3898 out << " gSystem->ChangeDirectory(cdir);" << endl;
3899 out << " return kFALSE;" << endl;
3900 out << " }" << endl;
3901 out << " } else {" << endl;
3902 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3903 out << " gSystem->ChangeDirectory(cdir);" << endl;
3904 out << " return kFALSE;" << endl;
3905 out << " }" << endl;
3906 out << " // Check for SETUP.C and execute" << endl;
3907 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3908 out << " printf(\"*******************************\\n\");" << endl;
3909 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3910 out << " printf(\"*******************************\\n\");" << endl;
3911 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3912 out << " } else {" << endl;
3913 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3914 out << " gSystem->ChangeDirectory(cdir);" << endl;
3915 out << " return kFALSE;" << endl;
3916 out << " }" << endl;
3917 out << " // Restore original workdir" << endl;
3918 out << " gSystem->ChangeDirectory(cdir);" << endl;
3919 out << " return kTRUE;" << endl;
3920 out << "}" << endl;
3921 }
c57f56b7 3922 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3923 }
3924 Bool_t copy = kTRUE;
d3b18c4c 3925 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3926 if (copy) {
3927 CdWork();
3928 TString workdir = gGrid->GetHomeDirectory();
3929 workdir += fGridWorkingDir;
3930 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 3931 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 3932// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3933 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
3934 Form("alien://%s/%s", workdir.Data(),
3935 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 3936 }
3937}
3938
0f389141 3939//______________________________________________________________________________
3940void AliAnalysisAlien::WriteMergingMacro()
3941{
3942// Write a macro to merge the outputs per master job.
3943 if (!fMergeViaJDL) return;
3944 if (!fOutputFiles.Length()) {
3945 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3946 return;
3947 }
3948 TString mergingMacro = fExecutable;
3949 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 3950 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3951 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3952 ofstream out;
3953 out.open(mergingMacro.Data(), ios::out);
3954 if (!out.good()) {
3955 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3956 return;
3957 }
5b273635 3958 Bool_t hasSTEERBase = kFALSE;
3959 Bool_t hasESD = kFALSE;
3960 Bool_t hasAOD = kFALSE;
3961 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3962 Bool_t hasOADB = kFALSE;
5b273635 3963 Bool_t hasANALYSISalice = kFALSE;
3964 Bool_t hasCORRFW = kFALSE;
0f389141 3965 TString func = mergingMacro;
3966 TString comment;
3967 func.ReplaceAll(".C", "");
f47d5cb4 3968 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 3969 out << "{" << endl;
3970 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
3971 out << " TStopwatch timer;" << endl;
3972 out << " timer.Start();" << endl << endl;
b3e07543 3973 // Reset existing include path
3974 out << "// Reset existing include path and add current directory first in the search" << endl;
3975 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 3976 if (!fExecutableCommand.Contains("aliroot")) {
3977 out << "// load base root libraries" << endl;
3978 out << " gSystem->Load(\"libTree\");" << endl;
3979 out << " gSystem->Load(\"libGeom\");" << endl;
3980 out << " gSystem->Load(\"libVMC\");" << endl;
3981 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3982 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3983 }
0f389141 3984 if (fAdditionalRootLibs.Length()) {
3985 // in principle libtree /lib geom libvmc etc. can go into this list, too
3986 out << "// Add aditional libraries" << endl;
3987 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3988 TIter next(list);
3989 TObjString *str;
3990 while((str=(TObjString*)next())) {
3991 if (str->GetString().Contains(".so"))
3992 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3993 }
3994 if (list) delete list;
3995 }
0f389141 3996 out << "// Load analysis framework libraries" << endl;
3997 if (!fPackages) {
7c2cd90a 3998 if (!fExecutableCommand.Contains("aliroot")) {
3999 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4000 out << " gSystem->Load(\"libESD\");" << endl;
4001 out << " gSystem->Load(\"libAOD\");" << endl;
4002 }
0f389141 4003 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4004 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4005 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4006 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4007 } else {
4008 TIter next(fPackages);
4009 TObject *obj;
4010 TString pkgname;
5b273635 4011 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4012 while ((obj=next())) {
4013 pkgname = obj->GetName();
4014 if (pkgname == "STEERBase" ||
4015 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4016 if (pkgname == "ESD" ||
4017 pkgname == "ESD.par") hasESD = kTRUE;
4018 if (pkgname == "AOD" ||
4019 pkgname == "AOD.par") hasAOD = kTRUE;
4020 if (pkgname == "ANALYSIS" ||
4021 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4022 if (pkgname == "OADB" ||
4023 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4024 if (pkgname == "ANALYSISalice" ||
4025 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4026 if (pkgname == "CORRFW" ||
4027 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4028 }
5b273635 4029 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4030 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4031 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4032 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4033 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4034 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4035 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4036 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4037 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4038 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4039 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4040 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4041 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4042 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4043 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4044 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4045 out << "// Compile other par packages" << endl;
4046 next.Reset();
4047 while ((obj=next())) {
4048 pkgname = obj->GetName();
4049 if (pkgname == "STEERBase" ||
4050 pkgname == "STEERBase.par" ||
4051 pkgname == "ESD" ||
4052 pkgname == "ESD.par" ||
4053 pkgname == "AOD" ||
4054 pkgname == "AOD.par" ||
4055 pkgname == "ANALYSIS" ||
4056 pkgname == "ANALYSIS.par" ||
7ea40ac8 4057 pkgname == "OADB" ||
4058 pkgname == "OADB.par" ||
0f389141 4059 pkgname == "ANALYSISalice" ||
4060 pkgname == "ANALYSISalice.par" ||
4061 pkgname == "CORRFW" ||
4062 pkgname == "CORRFW.par") continue;
5b273635 4063 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4064 }
4065 }
b3e07543 4066 out << "// include path" << endl;
4067 // Get the include path from the interpreter and remove entries pointing to AliRoot
4068 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4069 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4070 out << " TIter nextpath(listpaths);" << endl;
4071 out << " TObjString *pname;" << endl;
4072 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4073 out << " TString current = pname->GetName();" << endl;
4074 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4075 out << " gSystem->AddIncludePath(current);" << endl;
4076 out << " }" << endl;
4077 out << " if (listpaths) delete listpaths;" << endl;
4078 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4079 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4080 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4081 if (fAdditionalLibs.Length()) {
4082 out << "// Add aditional AliRoot libraries" << endl;
4083 TObjArray *list = fAdditionalLibs.Tokenize(" ");
4084 TIter next(list);
4085 TObjString *str;
4086 while((str=(TObjString*)next())) {
4087 if (str->GetString().Contains(".so"))
4088 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4089 }
4090 if (list) delete list;
4091 }
4092 out << endl;
4093 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4094 if (fAnalysisSource.Length()) {
4095 TObjArray *list = fAnalysisSource.Tokenize(" ");
4096 TIter next(list);
4097 TObjString *str;
4098 while((str=(TObjString*)next())) {
4099 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4100 }
4101 if (list) delete list;
4102 }
149d288c 4103 out << endl;
4104
0f389141 4105 if (fFastReadOption) {
4106 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4107 out << "// fast xrootd reading enabled" << endl;
4108 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4109 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4110 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4111 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4112 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4113 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4114 }
4115 // Change temp directory to current one
4116 out << "// Set temporary merging directory to current one" << endl;
4117 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4118 out << "// Set temporary compilation directory to current one" << endl;
4119 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
0f389141 4120 out << "// Connect to AliEn" << endl;
4121 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 4122 out << " TString outputDir = dir;" << endl;
f790bc1b 4123 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
0f389141 4124 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
58268c13 4125 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4126 out << " TIter *iter = new TIter(list);" << endl;
4127 out << " TObjString *str;" << endl;
a2f5fc01 4128 out << " TString outputFile;" << endl;
0f389141 4129 out << " Bool_t merged = kTRUE;" << endl;
4130 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4131 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4132 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4133 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4134 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4135 out << " // Skip already merged outputs" << endl;
a2f5fc01 4136 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
4137 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 4138 out << " continue;" << endl;
4139 out << " }" << endl;
a2f5fc01 4140 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4141 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4142 out << " if (!merged) {" << endl;
a2f5fc01 4143 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4144 out << " return;" << endl;
0f389141 4145 out << " }" << endl;
4146 out << " }" << endl;
7c2cd90a 4147 out << " // all outputs merged, validate" << endl;
4148 out << " ofstream out;" << endl;
4149 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4150 out << " out.close();" << endl;
4151 out << " // read the analysis manager from file" << endl;
f866cba5 4152 TString analysisFile = fExecutable;
4153 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 4154 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
105631b5 4155 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4156 << analysisFile << "\");" << endl;
4157 out << " if (!mgr) return;" << endl;
5c4250fc 4158 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4159 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4160 out << " mgr->PrintStatus();" << endl;
4161 if (AliAnalysisManager::GetAnalysisManager()) {
4162 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
4163 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4164 } else {
9a8288ff 4165 if (TestBit(AliAnalysisGrid::kTest))
4166 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4167 else
4168 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4169 }
4170 }
9a8288ff 4171 out << " TTree *tree = NULL;" << endl;
4172 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4173 out << "}" << endl << endl;
5b273635 4174 if (hasANALYSISalice) {
4175 out <<"//________________________________________________________________________________" << endl;
4176 out << "Bool_t SetupPar(const char *package) {" << endl;
4177 out << "// Compile the package and set it up." << endl;
4178 out << " TString pkgdir = package;" << endl;
4179 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4180 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4181 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4182 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4183 out << " // Check for BUILD.sh and execute" << endl;
4184 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4185 out << " printf(\"*******************************\\n\");" << endl;
4186 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4187 out << " printf(\"*******************************\\n\");" << endl;
4188 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4189 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4190 out << " gSystem->ChangeDirectory(cdir);" << endl;
4191 out << " return kFALSE;" << endl;
4192 out << " }" << endl;
4193 out << " } else {" << endl;
4194 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4195 out << " gSystem->ChangeDirectory(cdir);" << endl;
4196 out << " return kFALSE;" << endl;
4197 out << " }" << endl;
4198 out << " // Check for SETUP.C and execute" << endl;
4199 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4200 out << " printf(\"*******************************\\n\");" << endl;
4201 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4202 out << " printf(\"*******************************\\n\");" << endl;
4203 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4204 out << " } else {" << endl;
4205 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4206 out << " gSystem->ChangeDirectory(cdir);" << endl;
4207 out << " return kFALSE;" << endl;
4208 out << " }" << endl;
4209 out << " // Restore original workdir" << endl;
4210 out << " gSystem->ChangeDirectory(cdir);" << endl;
4211 out << " return kTRUE;" << endl;
4212 out << "}" << endl;
4213 }
0f389141 4214 }
4215 Bool_t copy = kTRUE;
d3b18c4c 4216 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4217 if (copy) {
4218 CdWork();
4219 TString workdir = gGrid->GetHomeDirectory();
4220 workdir += fGridWorkingDir;
4221 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4222 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4223// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4224 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4225 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4226 }
4227}
4228
4229//______________________________________________________________________________
4230Bool_t AliAnalysisAlien::SetupPar(const char *package)
4231{
205b201f 4232// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4233// Note that for loading the compiled library. The current directory should have precedence in
4234// LD_LIBRARY_PATH
4235 TString pkgdir = package;
4236 pkgdir.ReplaceAll(".par","");
b50a41c3 4237 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4238 TString cdir = gSystem->WorkingDirectory();
4239 gSystem->ChangeDirectory(pkgdir);
4240 // Check for BUILD.sh and execute
4241 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4242 printf("**************************************************\n");
4243 printf("*** Building PAR archive %s\n", package);
4244 printf("**************************************************\n");
4245 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4246 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4247 gSystem->ChangeDirectory(cdir);
4248 return kFALSE;
4249 }
4250 } else {
4251 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4252 gSystem->ChangeDirectory(cdir);
4253 return kFALSE;
4254 }
4255 // Check for SETUP.C and execute
4256 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4257 printf("**************************************************\n");
4258 printf("*** Setup PAR archive %s\n", package);
4259 printf("**************************************************\n");
4260 gROOT->Macro("PROOF-INF/SETUP.C");
4261 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4262 } else {
4263 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4264 gSystem->ChangeDirectory(cdir);
4265 return kFALSE;
4266 }
4267 // Restore original workdir
4268 gSystem->ChangeDirectory(cdir);
4269 return kTRUE;
4270}
4271
c57f56b7 4272//______________________________________________________________________________
4273void AliAnalysisAlien::WriteExecutable()
4274{
4275// Generate the alien executable script.
4276 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4277 ofstream out;
4278 out.open(fExecutable.Data(), ios::out);
4279 if (out.bad()) {
5513444a 4280 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4281 return;
4282 }
4283 out << "#!/bin/bash" << endl;
2a4d5166 4284 // Make sure we can properly compile par files
4285 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4286 out << "echo \"=========================================\"" << endl;
4287 out << "echo \"############## PATH : ##############\"" << endl;
4288 out << "echo $PATH" << endl;
4289 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4290 out << "echo $LD_LIBRARY_PATH" << endl;
4291 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4292 out << "echo $ROOTSYS" << endl;
4293 out << "echo \"############## which root : ##############\"" << endl;
4294 out << "which root" << endl;
4295 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4296 out << "echo $ALICE_ROOT" << endl;
4297 out << "echo \"############## which aliroot : ##############\"" << endl;
4298 out << "which aliroot" << endl;
9c5ddadc 4299 out << "echo \"############## system limits : ##############\"" << endl;
4300 out << "ulimit -a" << endl;
4301 out << "echo \"############## memory : ##############\"" << endl;
4302 out << "free -m" << endl;
c57f56b7 4303 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4304 out << fExecutableCommand << " ";
631c0b05 4305 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 4306 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
4307 out << "echo \"############## memory after: ##############\"" << endl;
4308 out << "free -m" << endl;
c57f56b7 4309 }
4310 Bool_t copy = kTRUE;
d3b18c4c 4311 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4312 if (copy) {
4313 CdWork();
4314 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4315 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4316 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4317 workdir += fGridWorkingDir;
4318 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4319 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4320 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4321// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4322 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4323 executable.Data())) Fatal("","Terminating");
c57f56b7 4324 }
4325}
4326
0f389141 4327//______________________________________________________________________________
4328void AliAnalysisAlien::WriteMergeExecutable()
4329{
4330// Generate the alien executable script for the merging job.
4331 if (!fMergeViaJDL) return;
4332 TString mergeExec = fExecutable;
4333 mergeExec.ReplaceAll(".sh", "_merge.sh");
4334 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4335 ofstream out;
4336 out.open(mergeExec.Data(), ios::out);
4337 if (out.bad()) {
4338 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4339 return;
4340 }
4341 out << "#!/bin/bash" << endl;
2a4d5166 4342 // Make sure we can properly compile par files
4343 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4344 out << "echo \"=========================================\"" << endl;
4345 out << "echo \"############## PATH : ##############\"" << endl;
4346 out << "echo $PATH" << endl;
4347 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4348 out << "echo $LD_LIBRARY_PATH" << endl;
4349 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4350 out << "echo $ROOTSYS" << endl;
4351 out << "echo \"############## which root : ##############\"" << endl;
4352 out << "which root" << endl;
4353 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4354 out << "echo $ALICE_ROOT" << endl;
4355 out << "echo \"############## which aliroot : ##############\"" << endl;
4356 out << "which aliroot" << endl;
4357 out << "echo \"############## system limits : ##############\"" << endl;
4358 out << "ulimit -a" << endl;
4359 out << "echo \"############## memory : ##############\"" << endl;
4360 out << "free -m" << endl;
4361 out << "echo \"=========================================\"" << endl << endl;
0f389141 4362 TString mergeMacro = fExecutable;
4363 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4364 if (IsOneStageMerging())
4365 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4366 else
f47d5cb4 4367 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4368 out << fExecutableCommand << " " << "$ARG" << endl;
4369 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4370 out << "echo \"############## memory after: ##############\"" << endl;
4371 out << "free -m" << endl;
0f389141 4372 }
4373 Bool_t copy = kTRUE;
d3b18c4c 4374 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4375 if (copy) {
4376 CdWork();
4377 TString workdir = gGrid->GetHomeDirectory();
4378 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4379 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4380 workdir += fGridWorkingDir;
4381 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4382 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4383 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4384// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4385 if (!copyLocal2Alien("WriteMergeExecutable",
4386 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4387 }
4388}
4389
c57f56b7 4390//______________________________________________________________________________
5513444a 4391void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4392{
4393// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4394// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4395// Next lines: full_path_to_dataset XXX (XXX is a string)
4396// To submit, one has to: submit jdl XXX for all lines
4397 ofstream out;
4398 out.open(filename, ios::out);
4399 if (out.bad()) {
4400 Error("WriteProductionFile", "Bad file name: %s", filename);
4401 return;
4402 }
d3b18c4c 4403 TString workdir;
4404 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4405 workdir = gGrid->GetHomeDirectory();
5513444a 4406 workdir += fGridWorkingDir;
f5e8c702 4407 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4408 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4409 out << locjdl << " " << njobspermaster << endl;
5513444a 4410 Int_t nmasterjobs = fInputFiles->GetEntries();
4411 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4412 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4413 runOutDir.ReplaceAll(".xml", "");
4414 if (fOutputToRunNo)
4415 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4416 else
4417 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4418 }
d3b18c4c 4419 if (gGrid) {
4420 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4421 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4422// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4423 if (!copyLocal2Alien("WriteProductionFile", filename,
4424 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4425 }
5513444a 4426}
4427
4428//______________________________________________________________________________
0f389141 4429void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4430{
4431// Generate the alien validation script.
4432 // Generate the validation script
4433 TObjString *os;
d3b18c4c 4434 if (fValidationScript.IsNull()) {
4435 fValidationScript = fExecutable;
4436 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4437 }
4438 TString validationScript = fValidationScript;
4439 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4440 if (!Connect()) {
4441 Error("WriteValidationScript", "Alien connection required");
4442 return;
4443 }
d3b18c4c 4444 if (!fTerminateFiles.IsNull()) {
4445 fTerminateFiles.Strip();
4446 fTerminateFiles.ReplaceAll(" ",",");
4447 }
a2f5fc01 4448 TString outStream = "";
4449 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4450 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4451 ofstream out;
0d5d317c 4452 out.open(validationScript, ios::out);
c57f56b7 4453 out << "#!/bin/bash" << endl;
4454 out << "##################################################" << endl;
4455 out << "validateout=`dirname $0`" << endl;
4456 out << "validatetime=`date`" << endl;
4457 out << "validated=\"0\";" << endl;
4458 out << "error=0" << endl;
4459 out << "if [ -z $validateout ]" << endl;
4460 out << "then" << endl;
4461 out << " validateout=\".\"" << endl;
4462 out << "fi" << endl << endl;
4463 out << "cd $validateout;" << endl;
4464 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4465 out << "echo \"*******************************************************\"" << outStream << endl;
4466 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4467 out << "" << endl;
a2f5fc01 4468 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4469 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4470 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4471 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4472 out << "ls -la ./" << outStream << endl;
4473 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4474 out << "##################################################" << endl;
ebec370a 4475 out << "" << endl;
4476
4477 out << "if [ ! -f stderr ] ; then" << endl;
4478 out << " error=1" << endl;
a2f5fc01 4479 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4480 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4481 out << "fi" << endl;
4482
b34c9f51 4483 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4484 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4485 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4486 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4487 out << "" << endl;
4488
ebec370a 4489 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4490 out << " error=1" << endl;
a2f5fc01 4491 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4492 out << " echo \"$parArch\" " << outStream << endl;
4493 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4494 out << "fi" << endl;
4495
4496 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4497 out << " error=1" << endl;
a2f5fc01 4498 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4499 out << " echo \"$segViol\" " << outStream << endl;
4500 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4501 out << "fi" << endl;
4502
4503 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4504 out << " error=1" << endl;
a2f5fc01 4505 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4506 out << " echo \"$segFault\" " << outStream << endl;
4507 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4508 out << "fi" << endl;
4509
b34c9f51 4510 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4511 out << " error=1" << endl;
a2f5fc01 4512 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4513 out << " echo \"$glibcErr\" " << outStream << endl;
4514 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4515 out << "fi" << endl;
4516
ebec370a 4517 // Part dedicated to the specific analyses running into the train
4518
d3b18c4c 4519 TString outputFiles = fOutputFiles;
4520 if (merge && !fTerminateFiles.IsNull()) {
4521 outputFiles += ",";
4522 outputFiles += fTerminateFiles;
4523 }
4524 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4525 TIter next1(arr);
a2f5fc01 4526 TString outputFile;
bbd88db8 4527 while (!merge && (os=(TObjString*)next1())) {
4528 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4529 outputFile = os->GetString();
4530 Int_t index = outputFile.Index("@");
4531 if (index > 0) outputFile.Remove(index);
bbd88db8 4532 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4533 if (outputFile.Contains("*")) continue;
4534 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4535 out << " error=1" << endl;
7c2cd90a 4536 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4537 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4538 out << "fi" << endl;
4539 }
4540 delete arr;
7c2cd90a 4541 out << "if ! [ -f outputs_valid ] ; then" << endl;
4542 out << " error=1" << endl;
4543 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4544 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4545 out << "fi" << endl;
923e2ca5 4546
c57f56b7 4547 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4548 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4549 if (!IsKeepLogs()) {
4550 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4551 outStream = "";
149d288c 4552 out << " rm -f std*" << endl;
4553 }
c57f56b7 4554 out << "fi" << endl;
4555
a2f5fc01 4556 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4557 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4558 out << "cd -" << endl;
4559 out << "exit $error" << endl;
4560 }
4561 Bool_t copy = kTRUE;
d3b18c4c 4562 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4563 if (copy) {
4564 CdWork();
4565 TString workdir = gGrid->GetHomeDirectory();
4566 workdir += fGridWorkingDir;
d3b18c4c 4567 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4568 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 4569// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4570 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
4571 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 4572 }
4573}