]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
-fix for weighted TH1,2,3
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
c82bb898 51using std::ofstream;
52using std::ifstream;
53using std::ios;
54using std::endl;
c57f56b7 55ClassImp(AliAnalysisAlien)
6c395669 56#if 0
57;
58#endif
c57f56b7 59
6c395669 60namespace {
61 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
62 {
63 TString sl(Form("file:%s", loc));
64 TString sr(Form("alien://%s", rem));
65 Bool_t ret = TFile::Cp(sl, sr);
66 if (!ret) {
67 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
68 }
69 return ret;
70 }
71}
72
c57f56b7 73//______________________________________________________________________________
74AliAnalysisAlien::AliAnalysisAlien()
75 :AliAnalysisGrid(),
76 fGridJDL(NULL),
0f389141 77 fMergingJDL(NULL),
c57f56b7 78 fPrice(0),
79 fTTL(0),
80 fSplitMaxInputFileNumber(0),
81 fMaxInitFailed(0),
82 fMasterResubmitThreshold(0),
bb885a9e 83 fNtestFiles(0),
319593fb 84 fNrunsPerMaster(0),
16a4353c 85 fMaxMergeFiles(0),
b3e07543 86 fMaxMergeStages(0),
d2a409b2 87 fNsubmitted(0),
a3e84053 88 fProductionMode(0),
cd11251e 89 fOutputToRunNo(0),
0f389141 90 fMergeViaJDL(0),
91 fFastReadOption(0),
e1c22e21 92 fOverwriteMode(1),
149d288c 93 fNreplicas(2),
3bdcb562 94 fNproofWorkers(0),
27734f0e 95 fNproofWorkersPerSlave(0),
3bdcb562 96 fProofReset(0),
c57f56b7 97 fRunNumbers(),
98 fExecutable(),
0a1c1f7f 99 fExecutableCommand(),
c57f56b7 100 fArguments(),
631c0b05 101 fExecutableArgs(),
c57f56b7 102 fAnalysisMacro(),
103 fAnalysisSource(),
d3b18c4c 104 fValidationScript(),
d5c6455a 105 fAdditionalRootLibs(),
c57f56b7 106 fAdditionalLibs(),
107 fSplitMode(),
108 fAPIVersion(),
109 fROOTVersion(),
110 fAliROOTVersion(),
648174cf 111 fExternalPackages(),
c57f56b7 112 fUser(),
113 fGridWorkingDir(),
114 fGridDataDir(),
115 fDataPattern(),
116 fGridOutputDir(),
117 fOutputArchive(),
118 fOutputFiles(),
119 fInputFormat(),
e7c71df0 120 fDatasetName(),
c57f56b7 121 fJDLName(),
d3b18c4c 122 fTerminateFiles(),
bb885a9e 123 fMergeExcludes(),
37d21c01 124 fRegisterExcludes(),
f965131e 125 fIncludePath(),
bb885a9e 126 fCloseSE(),
0df6ccf2 127 fFriendChainName(),
c6cb3634 128 fJobTag(),
648174cf 129 fOutputSingle(),
5fce53f4 130 fRunPrefix(),
3bdcb562 131 fProofCluster(),
132 fProofDataSet(),
133 fFileForTestMode(),
3bdcb562 134 fAliRootMode(),
d88e4159 135 fProofProcessOpt(),
f47d5cb4 136 fMergeDirName(),
4e5c5506 137 fInputFiles(0),
28ce6c9c 138 fPackages(0),
77f34eae 139 fModules(0),
d88e4159 140 fProofParam(),
a41b2882 141 fDropToShell(true),
142 fGridJobIDs(""),
143 fGridStages(""),
780ec36d 144 fFriendLibs(""),
145 fTreeName()
c57f56b7 146{
147// Dummy ctor.
148 SetDefaults();
149}
150
151//______________________________________________________________________________
152AliAnalysisAlien::AliAnalysisAlien(const char *name)
153 :AliAnalysisGrid(name),
154 fGridJDL(NULL),
0f389141 155 fMergingJDL(NULL),
c57f56b7 156 fPrice(0),
157 fTTL(0),
158 fSplitMaxInputFileNumber(0),
159 fMaxInitFailed(0),
160 fMasterResubmitThreshold(0),
bb885a9e 161 fNtestFiles(0),
319593fb 162 fNrunsPerMaster(0),
16a4353c 163 fMaxMergeFiles(0),
b3e07543 164 fMaxMergeStages(0),
d2a409b2 165 fNsubmitted(0),
a3e84053 166 fProductionMode(0),
cd11251e 167 fOutputToRunNo(0),
0f389141 168 fMergeViaJDL(0),
169 fFastReadOption(0),
e1c22e21 170 fOverwriteMode(1),
149d288c 171 fNreplicas(2),
3bdcb562 172 fNproofWorkers(0),
27734f0e 173 fNproofWorkersPerSlave(0),
3bdcb562 174 fProofReset(0),
c57f56b7 175 fRunNumbers(),
176 fExecutable(),
0a1c1f7f 177 fExecutableCommand(),
c57f56b7 178 fArguments(),
631c0b05 179 fExecutableArgs(),
c57f56b7 180 fAnalysisMacro(),
181 fAnalysisSource(),
d3b18c4c 182 fValidationScript(),
d5c6455a 183 fAdditionalRootLibs(),
c57f56b7 184 fAdditionalLibs(),
185 fSplitMode(),
186 fAPIVersion(),
187 fROOTVersion(),
188 fAliROOTVersion(),
648174cf 189 fExternalPackages(),
c57f56b7 190 fUser(),
191 fGridWorkingDir(),
192 fGridDataDir(),
193 fDataPattern(),
194 fGridOutputDir(),
195 fOutputArchive(),
196 fOutputFiles(),
197 fInputFormat(),
e7c71df0 198 fDatasetName(),
c57f56b7 199 fJDLName(),
d3b18c4c 200 fTerminateFiles(),
bb885a9e 201 fMergeExcludes(),
37d21c01 202 fRegisterExcludes(),
f965131e 203 fIncludePath(),
bb885a9e 204 fCloseSE(),
0df6ccf2 205 fFriendChainName(),
c6cb3634 206 fJobTag(),
648174cf 207 fOutputSingle(),
5fce53f4 208 fRunPrefix(),
3bdcb562 209 fProofCluster(),
210 fProofDataSet(),
211 fFileForTestMode(),
3bdcb562 212 fAliRootMode(),
d88e4159 213 fProofProcessOpt(),
f47d5cb4 214 fMergeDirName(),
4e5c5506 215 fInputFiles(0),
28ce6c9c 216 fPackages(0),
77f34eae 217 fModules(0),
d88e4159 218 fProofParam(),
0ad3ea1f 219 fDropToShell(true),
220 fGridJobIDs(""),
a41b2882 221 fGridStages(""),
780ec36d 222 fFriendLibs(""),
223 fTreeName()
c57f56b7 224{
225// Default ctor.
226 SetDefaults();
227}
228
229//______________________________________________________________________________
230AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
231 :AliAnalysisGrid(other),
232 fGridJDL(NULL),
0f389141 233 fMergingJDL(NULL),
c57f56b7 234 fPrice(other.fPrice),
235 fTTL(other.fTTL),
236 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
237 fMaxInitFailed(other.fMaxInitFailed),
238 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 239 fNtestFiles(other.fNtestFiles),
319593fb 240 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 241 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 242 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 243 fNsubmitted(other.fNsubmitted),
a3e84053 244 fProductionMode(other.fProductionMode),
cd11251e 245 fOutputToRunNo(other.fOutputToRunNo),
0f389141 246 fMergeViaJDL(other.fMergeViaJDL),
247 fFastReadOption(other.fFastReadOption),
248 fOverwriteMode(other.fOverwriteMode),
149d288c 249 fNreplicas(other.fNreplicas),
3bdcb562 250 fNproofWorkers(other.fNproofWorkers),
27734f0e 251 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 252 fProofReset(other.fProofReset),
c57f56b7 253 fRunNumbers(other.fRunNumbers),
254 fExecutable(other.fExecutable),
0a1c1f7f 255 fExecutableCommand(other.fExecutableCommand),
c57f56b7 256 fArguments(other.fArguments),
631c0b05 257 fExecutableArgs(other.fExecutableArgs),
c57f56b7 258 fAnalysisMacro(other.fAnalysisMacro),
259 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 260 fValidationScript(other.fValidationScript),
d5c6455a 261 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 262 fAdditionalLibs(other.fAdditionalLibs),
263 fSplitMode(other.fSplitMode),
264 fAPIVersion(other.fAPIVersion),
265 fROOTVersion(other.fROOTVersion),
266 fAliROOTVersion(other.fAliROOTVersion),
648174cf 267 fExternalPackages(other.fExternalPackages),
c57f56b7 268 fUser(other.fUser),
269 fGridWorkingDir(other.fGridWorkingDir),
270 fGridDataDir(other.fGridDataDir),
271 fDataPattern(other.fDataPattern),
272 fGridOutputDir(other.fGridOutputDir),
273 fOutputArchive(other.fOutputArchive),
274 fOutputFiles(other.fOutputFiles),
275 fInputFormat(other.fInputFormat),
e7c71df0 276 fDatasetName(other.fDatasetName),
c57f56b7 277 fJDLName(other.fJDLName),
d3b18c4c 278 fTerminateFiles(other.fTerminateFiles),
bb885a9e 279 fMergeExcludes(other.fMergeExcludes),
37d21c01 280 fRegisterExcludes(other.fRegisterExcludes),
f965131e 281 fIncludePath(other.fIncludePath),
bb885a9e 282 fCloseSE(other.fCloseSE),
0df6ccf2 283 fFriendChainName(other.fFriendChainName),
c6cb3634 284 fJobTag(other.fJobTag),
648174cf 285 fOutputSingle(other.fOutputSingle),
5fce53f4 286 fRunPrefix(other.fRunPrefix),
3bdcb562 287 fProofCluster(other.fProofCluster),
288 fProofDataSet(other.fProofDataSet),
289 fFileForTestMode(other.fFileForTestMode),
3bdcb562 290 fAliRootMode(other.fAliRootMode),
d88e4159 291 fProofProcessOpt(other.fProofProcessOpt),
f47d5cb4 292 fMergeDirName(other.fMergeDirName),
4e5c5506 293 fInputFiles(0),
28ce6c9c 294 fPackages(0),
77f34eae 295 fModules(0),
d88e4159 296 fProofParam(),
0ad3ea1f 297 fDropToShell(other.fDropToShell),
298 fGridJobIDs(other.fGridJobIDs),
a41b2882 299 fGridStages(other.fGridStages),
780ec36d 300 fFriendLibs(other.fFriendLibs),
301 fTreeName(other.fTreeName)
c57f56b7 302{
303// Copy ctor.
304 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 305 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 306 fRunRange[0] = other.fRunRange[0];
307 fRunRange[1] = other.fRunRange[1];
c57f56b7 308 if (other.fInputFiles) {
309 fInputFiles = new TObjArray();
310 TIter next(other.fInputFiles);
311 TObject *obj;
312 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
313 fInputFiles->SetOwner();
314 }
4e5c5506 315 if (other.fPackages) {
316 fPackages = new TObjArray();
317 TIter next(other.fPackages);
318 TObject *obj;
319 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
320 fPackages->SetOwner();
321 }
77f34eae 322 if (other.fModules) {
323 fModules = new TObjArray();
324 fModules->SetOwner();
325 TIter next(other.fModules);
326 AliAnalysisTaskCfg *mod, *crt;
327 while ((crt=(AliAnalysisTaskCfg*)next())) {
328 mod = new AliAnalysisTaskCfg(*crt);
329 fModules->Add(mod);
330 }
331 }
c57f56b7 332}
333
334//______________________________________________________________________________
335AliAnalysisAlien::~AliAnalysisAlien()
336{
337// Destructor.
77f34eae 338 delete fGridJDL;
339 delete fMergingJDL;
340 delete fInputFiles;
341 delete fPackages;
342 delete fModules;
28ce6c9c 343 fProofParam.DeleteAll();
c57f56b7 344}
345
346//______________________________________________________________________________
347AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
348{
349// Assignment.
350 if (this != &other) {
351 AliAnalysisGrid::operator=(other);
352 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 353 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 354 fPrice = other.fPrice;
355 fTTL = other.fTTL;
356 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
357 fMaxInitFailed = other.fMaxInitFailed;
358 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 359 fNtestFiles = other.fNtestFiles;
a3e84053 360 fNrunsPerMaster = other.fNrunsPerMaster;
361 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 362 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 363 fNsubmitted = other.fNsubmitted;
364 fProductionMode = other.fProductionMode;
cd11251e 365 fOutputToRunNo = other.fOutputToRunNo;
0f389141 366 fMergeViaJDL = other.fMergeViaJDL;
367 fFastReadOption = other.fFastReadOption;
368 fOverwriteMode = other.fOverwriteMode;
149d288c 369 fNreplicas = other.fNreplicas;
3bdcb562 370 fNproofWorkers = other.fNproofWorkers;
27734f0e 371 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 372 fProofReset = other.fProofReset;
c57f56b7 373 fRunNumbers = other.fRunNumbers;
374 fExecutable = other.fExecutable;
0a1c1f7f 375 fExecutableCommand = other.fExecutableCommand;
c57f56b7 376 fArguments = other.fArguments;
631c0b05 377 fExecutableArgs = other.fExecutableArgs;
c57f56b7 378 fAnalysisMacro = other.fAnalysisMacro;
379 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 380 fValidationScript = other.fValidationScript;
d5c6455a 381 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 382 fAdditionalLibs = other.fAdditionalLibs;
383 fSplitMode = other.fSplitMode;
384 fAPIVersion = other.fAPIVersion;
385 fROOTVersion = other.fROOTVersion;
386 fAliROOTVersion = other.fAliROOTVersion;
648174cf 387 fExternalPackages = other.fExternalPackages;
c57f56b7 388 fUser = other.fUser;
389 fGridWorkingDir = other.fGridWorkingDir;
390 fGridDataDir = other.fGridDataDir;
391 fDataPattern = other.fDataPattern;
392 fGridOutputDir = other.fGridOutputDir;
393 fOutputArchive = other.fOutputArchive;
394 fOutputFiles = other.fOutputFiles;
395 fInputFormat = other.fInputFormat;
e7c71df0 396 fDatasetName = other.fDatasetName;
c57f56b7 397 fJDLName = other.fJDLName;
d3b18c4c 398 fTerminateFiles = other.fTerminateFiles;
bb885a9e 399 fMergeExcludes = other.fMergeExcludes;
37d21c01 400 fRegisterExcludes = other.fRegisterExcludes;
f965131e 401 fIncludePath = other.fIncludePath;
bb885a9e 402 fCloseSE = other.fCloseSE;
0df6ccf2 403 fFriendChainName = other.fFriendChainName;
c6cb3634 404 fJobTag = other.fJobTag;
648174cf 405 fOutputSingle = other.fOutputSingle;
5fce53f4 406 fRunPrefix = other.fRunPrefix;
3bdcb562 407 fProofCluster = other.fProofCluster;
408 fProofDataSet = other.fProofDataSet;
409 fFileForTestMode = other.fFileForTestMode;
3bdcb562 410 fAliRootMode = other.fAliRootMode;
d88e4159 411 fProofProcessOpt = other.fProofProcessOpt;
f47d5cb4 412 fMergeDirName = other.fMergeDirName;
d88e4159 413 fDropToShell = other.fDropToShell;
0ad3ea1f 414 fGridJobIDs = other.fGridJobIDs;
415 fGridStages = other.fGridStages;
a41b2882 416 fFriendLibs = other.fFriendLibs;
780ec36d 417 fTreeName = other.fTreeName;
c57f56b7 418 if (other.fInputFiles) {
419 fInputFiles = new TObjArray();
420 TIter next(other.fInputFiles);
421 TObject *obj;
422 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
423 fInputFiles->SetOwner();
424 }
4e5c5506 425 if (other.fPackages) {
426 fPackages = new TObjArray();
427 TIter next(other.fPackages);
428 TObject *obj;
429 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
430 fPackages->SetOwner();
431 }
77f34eae 432 if (other.fModules) {
433 fModules = new TObjArray();
434 fModules->SetOwner();
435 TIter next(other.fModules);
436 AliAnalysisTaskCfg *mod, *crt;
437 while ((crt=(AliAnalysisTaskCfg*)next())) {
438 mod = new AliAnalysisTaskCfg(*crt);
439 fModules->Add(mod);
440 }
441 }
c57f56b7 442 }
443 return *this;
444}
445
37d21c01 446//______________________________________________________________________________
447void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
448{
449// Add a single additional library to be loaded. Extension must be present.
450 TString lib(name);
451 if (!lib.Contains(".")) {
452 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
453 return;
454 }
455 if (fAdditionalLibs.Contains(name)) {
456 Warning("AddAdditionalLibrary", "Library %s already added.", name);
457 return;
458 }
459 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
460 fAdditionalLibs += lib;
461}
462
77f34eae 463//______________________________________________________________________________
464void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
465{
466// Adding a module. Checks if already existing. Becomes owned by this.
467 if (!module) return;
468 if (GetModule(module->GetName())) {
469 Error("AddModule", "A module having the same name %s already added", module->GetName());
470 return;
471 }
472 if (!fModules) {
473 fModules = new TObjArray();
474 fModules->SetOwner();
475 }
476 fModules->Add(module);
477}
478
479//______________________________________________________________________________
480void AliAnalysisAlien::AddModules(TObjArray *list)
481{
482// Adding a list of modules. Checks if already existing. Becomes owned by this.
483 TIter next(list);
484 AliAnalysisTaskCfg *module;
485 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
486}
487
488//______________________________________________________________________________
489Bool_t AliAnalysisAlien::CheckDependencies()
490{
491// Check if all dependencies are satisfied. Reorder modules if needed.
492 Int_t nmodules = GetNmodules();
493 if (!nmodules) {
494 Warning("CheckDependencies", "No modules added yet to check their dependencies");
495 return kTRUE;
496 }
497 AliAnalysisTaskCfg *mod = 0;
498 AliAnalysisTaskCfg *dep = 0;
499 TString depname;
500 Int_t i, j, k;
501 for (i=0; i<nmodules; i++) {
502 mod = (AliAnalysisTaskCfg*) fModules->At(i);
503 Int_t ndeps = mod->GetNdeps();
504 Int_t istart = i;
505 for (j=0; j<ndeps; j++) {
506 depname = mod->GetDependency(j);
507 dep = GetModule(depname);
508 if (!dep) {
509 Error("CheckDependencies","Dependency %s not added for module %s",
510 depname.Data(), mod->GetName());
511 return kFALSE;
512 }
513 if (dep->NeedsDependency(mod->GetName())) {
514 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
515 mod->GetName(), dep->GetName());
516 return kFALSE;
517 }
518 Int_t idep = fModules->IndexOf(dep);
519 // The dependency task must come first
520 if (idep>i) {
521 // Remove at idep and move all objects below up one slot
522 // down to index i included.
523 fModules->RemoveAt(idep);
3e40fd4c 524 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 525 fModules->AddAt(dep, i++);
526 }
527 //Redo from istart if dependencies were inserted
528 if (i>istart) i=istart-1;
529 }
530 }
531 return kTRUE;
532}
533
3e40fd4c 534//______________________________________________________________________________
535AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
536{
537// Create the analysis manager and optionally execute the macro in filename.
538 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
539 if (mgr) return mgr;
540 mgr = new AliAnalysisManager(name);
541 mgr->SetGridHandler((AliAnalysisGrid*)this);
542 if (strlen(filename)) {
543 TString line = gSystem->ExpandPathName(filename);
544 line.Prepend(".x ");
545 gROOT->ProcessLine(line.Data());
546 }
547 return mgr;
548}
549
77f34eae 550//______________________________________________________________________________
551Int_t AliAnalysisAlien::GetNmodules() const
552{
553// Get number of modules.
554 if (!fModules) return 0;
555 return fModules->GetEntries();
556}
557
558//______________________________________________________________________________
559AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
560{
561// Get a module by name.
562 if (!fModules) return 0;
563 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
564}
565
566//______________________________________________________________________________
567Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
568{
569// Load a given module.
570 if (mod->IsLoaded()) return kTRUE;
192ddca2 571 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
572 if (!mgr) {
573 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
574 return kFALSE;
575 }
77f34eae 576 Int_t ndeps = mod->GetNdeps();
577 TString depname;
578 for (Int_t j=0; j<ndeps; j++) {
579 depname = mod->GetDependency(j);
580 AliAnalysisTaskCfg *dep = GetModule(depname);
581 if (!dep) {
582 Error("LoadModule","Dependency %s not existing for module %s",
583 depname.Data(), mod->GetName());
584 return kFALSE;
585 }
586 if (!LoadModule(dep)) {
587 Error("LoadModule","Dependency %s for module %s could not be loaded",
588 depname.Data(), mod->GetName());
589 return kFALSE;
590 }
591 }
592 // Load libraries for the module
593 if (!mod->CheckLoadLibraries()) {
594 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
595 return kFALSE;
596 }
192ddca2 597 // Check if a custom file name was requested
598 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
599
600 // Check if a custom terminate file name was requested
601 if (strlen(mod->GetTerminateFileName())) {
602 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
603 fTerminateFiles += mod->GetTerminateFileName();
604 }
605
77f34eae 606 // Execute the macro
607 if (mod->ExecuteMacro()<0) {
608 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
609 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
610 return kFALSE;
611 }
612 // Configure dependencies
613 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
614 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
615 mod->GetConfigMacro()->GetTitle(), mod->GetName());
616 return kFALSE;
617 }
5e8c7c22 618 // Adjust extra libraries
619 Int_t nlibs = mod->GetNlibs();
620 TString lib;
621 for (Int_t i=0; i<nlibs; i++) {
622 lib = mod->GetLibrary(i);
5e8c7c22 623 lib = Form("lib%s.so", lib.Data());
c9d6bd12 624 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 625 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
626 fAdditionalLibs += lib;
627 }
77f34eae 628 return kTRUE;
629}
630
139fe6ba 631//______________________________________________________________________________
632Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
633{
634// Generate the full train.
635 fAdditionalLibs = "";
636 if (!LoadModules()) return kFALSE;
637 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
638 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 639 mgr->RunLocalInit();
139fe6ba 640 mgr->PrintStatus();
641 Int_t productionMode = fProductionMode;
642 SetProductionMode();
643 TString macro = fAnalysisMacro;
644 TString executable = fExecutable;
645 TString validation = fValidationScript;
646 TString execCommand = fExecutableCommand;
647 SetAnalysisMacro(Form("%s.C", name));
648 SetExecutable(Form("%s.sh", name));
a8e31fb0 649// SetExecutableCommand("aliroot -b -q ");
139fe6ba 650 SetValidationScript(Form("%s_validation.sh", name));
651 StartAnalysis();
652 SetProductionMode(productionMode);
653 fAnalysisMacro = macro;
654 fExecutable = executable;
655 fExecutableCommand = execCommand;
656 fValidationScript = validation;
657 return kTRUE;
658}
659
77f34eae 660//______________________________________________________________________________
21ca8e59 661Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 662{
663// Generate test macros for a single module or for the full train.
5e8c7c22 664 fAdditionalLibs = "";
77f34eae 665 if (strlen(modname)) {
21ca8e59 666 if (!CheckDependencies()) return kFALSE;
667 AliAnalysisTaskCfg *mod = GetModule(modname);
668 if (!mod) {
669 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
670 return kFALSE;
671 }
672 if (!LoadModule(mod)) return kFALSE;
a41b2882 673 if (!LoadFriendLibs()) return kFALSE;
21ca8e59 674 } else if (!LoadModules()) return kFALSE;
675 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
676 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 677 mgr->RunLocalInit();
21ca8e59 678 mgr->PrintStatus();
679 SetLocalTest(kTRUE);
680 Int_t productionMode = fProductionMode;
681 SetProductionMode();
682 TString macro = fAnalysisMacro;
683 TString executable = fExecutable;
684 TString validation = fValidationScript;
685 TString execCommand = fExecutableCommand;
686 SetAnalysisMacro(Form("%s.C", name));
687 SetExecutable(Form("%s.sh", name));
d66aa2a9 688 fOutputFiles = GetListOfFiles("outaod");
689 // Add extra files registered to the analysis manager
690 TString extra = GetListOfFiles("ext");
691 if (!extra.IsNull()) {
692 extra.ReplaceAll(".root", "*.root");
693 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
694 fOutputFiles += extra;
695 }
a8e31fb0 696// SetExecutableCommand("aliroot -b -q ");
21ca8e59 697 SetValidationScript(Form("%s_validation.sh", name));
698 WriteAnalysisFile();
699 WriteAnalysisMacro();
700 WriteExecutable();
701 WriteValidationScript();
ffbe06b9 702 WriteMergingMacro();
703 WriteMergeExecutable();
704 WriteValidationScript(kTRUE);
21ca8e59 705 SetLocalTest(kFALSE);
706 SetProductionMode(productionMode);
707 fAnalysisMacro = macro;
708 fExecutable = executable;
709 fExecutableCommand = execCommand;
710 fValidationScript = validation;
77f34eae 711 return kTRUE;
712}
713
714//______________________________________________________________________________
715Bool_t AliAnalysisAlien::LoadModules()
716{
717// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 718 fAdditionalLibs = "";
21ca8e59 719 Int_t nmodules = GetNmodules();
720 if (!nmodules) {
721 Warning("LoadModules", "No module to be loaded");
722 return kTRUE;
723 }
3e40fd4c 724 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
725 if (!mgr) {
726 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
727 return kFALSE;
728 }
77f34eae 729 if (!CheckDependencies()) return kFALSE;
21ca8e59 730 nmodules = GetNmodules();
77f34eae 731 AliAnalysisTaskCfg *mod;
21ca8e59 732 for (Int_t imod=0; imod<nmodules; imod++) {
733 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 734 if (!LoadModule(mod)) return kFALSE;
735 }
a41b2882 736 // Load additional friend libraries
737 return LoadFriendLibs();
77f34eae 738}
739
a41b2882 740//______________________________________________________________________________
741Bool_t AliAnalysisAlien::LoadFriendLibs() const
742{
743// Load libraries required for reading friends.
744 if (fFriendLibs.Length()) {
745 TObjArray *list = 0;
746 TString lib;
747 if (fFriendLibs.Contains(",")) list = fFriendLibs.Tokenize(",");
748 else list = fFriendLibs.Tokenize(" ");
749 for (Int_t ilib=0; ilib<list->GetEntriesFast(); ilib++) {
750 lib = list->At(ilib)->GetName();
751 lib.ReplaceAll(".so","");
752 lib.ReplaceAll(" ","");
753 if (lib.BeginsWith("lib")) lib.Remove(0, 3);
754 lib.Prepend("lib");
755 Int_t loaded = strlen(gSystem->GetLibraries(lib,"",kFALSE));
756 if (!loaded) loaded = gSystem->Load(lib);
757 if (loaded < 0) {
758 Error("LoadModules", "Cannot load library for friends %s", lib.Data());
759 return kFALSE;
760 }
761 }
762 delete list;
763 }
764 return kTRUE;
765}
766
74013535 767//______________________________________________________________________________
768void AliAnalysisAlien::SetRunPrefix(const char *prefix)
769{
770// Set the run number format. Can be a prefix or a format like "%09d"
771 fRunPrefix = prefix;
772 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
773}
774
f965131e 775//______________________________________________________________________________
776void AliAnalysisAlien::AddIncludePath(const char *path)
777{
778// Add include path in the remote analysis macro.
779 TString p(path);
780 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
781 else fIncludePath += Form("-I%s ", path);
782}
783
c57f56b7 784//______________________________________________________________________________
785void AliAnalysisAlien::AddRunNumber(Int_t run)
786{
787// Add a run number to the list of runs to be processed.
788 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 789 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 790}
791
98690895 792//______________________________________________________________________________
793void AliAnalysisAlien::AddRunList(const char* runList)
794{
795// Add several runs into the list of runs; they are expected to be separated by a blank character.
796 TString sList = runList;
797 TObjArray *list = sList.Tokenize(" ");
798 Int_t n = list->GetEntries();
799 for (Int_t i = 0; i < n; i++) {
800 TObjString *os = (TObjString*)list->At(i);
801 AddRunNumber(os->GetString().Atoi());
802 }
803 delete list;
804}
805
ee75cfc3 806//______________________________________________________________________________
807void AliAnalysisAlien::AddRunNumber(const char* run)
808{
809// Add a run number to the list of runs to be processed.
7aa5d539 810 TString runs = run;
811 TObjString *os;
812 TObjArray *arr = runs.Tokenize(" ");
813 TIter next(arr);
814 TString prefix;
815 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
816 while ((os=(TObjString*)next())){
817 if (fRunNumbers.Length()) fRunNumbers += " ";
818 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
819 }
820 delete arr;
ee75cfc3 821}
822
c57f56b7 823//______________________________________________________________________________
824void AliAnalysisAlien::AddDataFile(const char *lfn)
825{
826// Adds a data file to the input to be analysed. The file should be a valid LFN
827// or point to an existing file in the alien workdir.
828 if (!fInputFiles) fInputFiles = new TObjArray();
829 fInputFiles->Add(new TObjString(lfn));
830}
648174cf 831
832//______________________________________________________________________________
833void AliAnalysisAlien::AddExternalPackage(const char *package)
834{
835// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
836 if (fExternalPackages) fExternalPackages += " ";
837 fExternalPackages += package;
838}
839
c57f56b7 840//______________________________________________________________________________
841Bool_t AliAnalysisAlien::Connect()
842{
843// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
844 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 845 if (fProductionMode) return kTRUE;
c57f56b7 846 if (!gGrid) {
847 Info("Connect", "Trying to connect to AliEn ...");
848 TGrid::Connect("alien://");
849 }
850 if (!gGrid || !gGrid->IsConnected()) {
851 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
852 return kFALSE;
853 }
854 fUser = gGrid->GetUser();
855 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
856 return kTRUE;
857}
858
859//______________________________________________________________________________
860void AliAnalysisAlien::CdWork()
861{
862// Check validity of alien workspace. Create directory if possible.
863 if (!Connect()) {
864 Error("CdWork", "Alien connection required");
865 return;
866 }
867 TString homedir = gGrid->GetHomeDirectory();
868 TString workdir = homedir + fGridWorkingDir;
923e2ca5 869 if (DirectoryExists(workdir)) {
870 gGrid->Cd(workdir);
871 return;
872 }
873 // Work directory not existing - create it
874 gGrid->Cd(homedir);
b93f8109 875 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 876 gGrid->Cd(fGridWorkingDir);
d3b18c4c 877 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 878 } else {
d3b18c4c 879 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 880 workdir.Data(), homedir.Data());
881 fGridWorkingDir = "";
882 }
c57f56b7 883}
884
348be253 885//______________________________________________________________________________
886Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
887{
888// Check if file copying is possible.
d3b18c4c 889 if (fProductionMode) return kTRUE;
b5ae4fe0 890 TString salienpath(alienpath);
891 if (salienpath.Contains(" ")) {
892 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
893 return kFALSE;
894 }
348be253 895 if (!Connect()) {
896 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
897 return kFALSE;
898 }
d3339be3 899 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
900 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 901 // Check if alien_CLOSE_SE is defined
902 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
903 if (!closeSE.IsNull()) {
904 Info("CheckFileCopy", "Your current close storage is pointing to: \
905 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
906 } else {
907 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
908 }
909 // Check if grid directory exists.
910 if (!DirectoryExists(alienpath)) {
911 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
912 return kFALSE;
913 }
b5ae4fe0 914 TString stest = "plugin_test_copy";
915 TFile f(stest, "RECREATE");
348be253 916 // User may not have write permissions to current directory
917 if (f.IsZombie()) {
918 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
919 gSystem->WorkingDirectory());
920 return kFALSE;
921 }
922 f.Close();
b5ae4fe0 923 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
924 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 925 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 926 \n# 1. Make sure you have write permissions there. If this is the case: \
927 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
928 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
929 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
930 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 931 gSystem->Unlink(stest.Data());
348be253 932 return kFALSE;
933 }
b5ae4fe0 934 gSystem->Unlink(stest.Data());
935 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 936 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 937 return kTRUE;
938}
939
c57f56b7 940//______________________________________________________________________________
941Bool_t AliAnalysisAlien::CheckInputData()
942{
943// Check validity of input data. If necessary, create xml files.
d3b18c4c 944 if (fProductionMode) return kTRUE;
a8739e8a 945 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 946 if (!fGridDataDir.Length()) {
947 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
948 return kFALSE;
949 }
9f020952 950 if (fMergeViaJDL) {
951 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
952 return kFALSE;
953 }
d2a409b2 954 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 955 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
956 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 957 return kTRUE;
c57f56b7 958 }
959 // Process declared files
a2f5fc01 960 Bool_t isCollection = kFALSE;
961 Bool_t isXml = kFALSE;
962 Bool_t useTags = kFALSE;
c57f56b7 963 Bool_t checked = kFALSE;
d3b18c4c 964 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 965 TString file;
966 TString workdir = gGrid->GetHomeDirectory();
967 workdir += fGridWorkingDir;
968 if (fInputFiles) {
969 TObjString *objstr;
970 TIter next(fInputFiles);
971 while ((objstr=(TObjString*)next())) {
972 file = workdir;
973 file += "/";
974 file += objstr->GetString();
975 // Store full lfn path
976 if (FileExists(file)) objstr->SetString(file);
977 else {
978 file = objstr->GetName();
979 if (!FileExists(objstr->GetName())) {
980 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
981 objstr->GetName(), workdir.Data());
982 return kFALSE;
983 }
984 }
985 Bool_t iscoll, isxml, usetags;
986 CheckDataType(file, iscoll, isxml, usetags);
987 if (!checked) {
988 checked = kTRUE;
a2f5fc01 989 isCollection = iscoll;
990 isXml = isxml;
991 useTags = usetags;
992 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 993 } else {
a2f5fc01 994 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 995 Error("CheckInputData", "Some conflict was found in the types of inputs");
996 return kFALSE;
997 }
998 }
999 }
1000 }
1001 // Process requested run numbers
a8739e8a 1002 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 1003 // Check validity of alien data directory
1004 if (!fGridDataDir.Length()) {
1005 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
1006 return kFALSE;
1007 }
923e2ca5 1008 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 1009 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
1010 return kFALSE;
1011 }
a2f5fc01 1012 if (isCollection) {
c57f56b7 1013 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
1014 return kFALSE;
1015 }
1016
a2f5fc01 1017 if (checked && !isXml) {
c57f56b7 1018 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
1019 return kFALSE;
1020 }
1021 // Check validity of run number(s)
1022 TObjArray *arr;
1023 TObjString *os;
74013535 1024 TString format;
319593fb 1025 Int_t nruns = 0;
904f9f5f 1026 TString schunk, schunk2;
c57f56b7 1027 TString path;
1028 if (!checked) {
1029 checked = kTRUE;
a2f5fc01 1030 useTags = fDataPattern.Contains("tag");
1031 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1032 }
a2f5fc01 1033 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 1034 Error("CheckInputData", "Cannot mix input files using/not using tags");
1035 return kFALSE;
1036 }
1037 if (fRunNumbers.Length()) {
a8739e8a 1038 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 1039 arr = fRunNumbers.Tokenize(" ");
1040 TIter next(arr);
1041 while ((os=(TObjString*)next())) {
1042 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1043 if (!DirectoryExists(path)) {
1044 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 1045 continue;
c57f56b7 1046 }
1047 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
1048 TString msg = "\n##### file: ";
1049 msg += path;
1050 msg += " type: xml_collection;";
a2f5fc01 1051 if (useTags) msg += " using_tags: Yes";
c57f56b7 1052 else msg += " using_tags: No";
23329835 1053 Info("CheckDataType", "%s", msg.Data());
319593fb 1054 if (fNrunsPerMaster<2) {
d2a409b2 1055 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 1056 } else {
1057 nruns++;
1058 if (((nruns-1)%fNrunsPerMaster) == 0) {
1059 schunk = os->GetString();
1060 }
1061 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1062 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1063 AddDataFile(schunk);
319593fb 1064 }
c57f56b7 1065 }
1066 delete arr;
a8739e8a 1067 } else {
1068 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1069 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1070 format = Form("%%s/%s ", fRunPrefix.Data());
1071 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1072 if (!DirectoryExists(path)) {
a8739e8a 1073 continue;
1074 }
74013535 1075 format = Form("%%s/%s.xml", fRunPrefix.Data());
1076 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1077 TString msg = "\n##### file: ";
1078 msg += path;
1079 msg += " type: xml_collection;";
a2f5fc01 1080 if (useTags) msg += " using_tags: Yes";
a8739e8a 1081 else msg += " using_tags: No";
23329835 1082 Info("CheckDataType", "%s", msg.Data());
319593fb 1083 if (fNrunsPerMaster<2) {
74013535 1084 format = Form("%s.xml", fRunPrefix.Data());
1085 AddDataFile(Form(format.Data(),irun));
319593fb 1086 } else {
1087 nruns++;
1088 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1089 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1090 }
74013535 1091 format = Form("_%s.xml", fRunPrefix.Data());
1092 schunk2 = Form(format.Data(), irun);
319593fb 1093 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1094 schunk += schunk2;
d2a409b2 1095 AddDataFile(schunk);
319593fb 1096 }
a8739e8a 1097 }
904f9f5f 1098 if (!fInputFiles) {
1099 schunk += schunk2;
1100 AddDataFile(schunk);
1101 }
c57f56b7 1102 }
1103 return kTRUE;
1104}
1105
21ca8e59 1106//______________________________________________________________________________
4a8b667c 1107Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1108{
1109// Copy data from the given grid directory according a pattern and make a local
1110// dataset.
2ba4c287 1111// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1112 if (!Connect()) {
1113 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
4a8b667c 1114 return 0;
21ca8e59 1115 }
1116 if (!DirectoryExists(griddir)) {
1117 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
4a8b667c 1118 return 0;
21ca8e59 1119 }
1120 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1121 printf("Running command: %s\n", command.Data());
1122 TGridResult *res = gGrid->Command(command);
1123 Int_t nfound = res->GetEntries();
1124 if (!nfound) {
1125 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
4a8b667c 1126 return 0;
21ca8e59 1127 }
1128 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1129
1130 // archives
1131 TObjArray* additionalArchives = 0;
1132 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1133 additionalArchives = TString(archivefile).Tokenize(";");
1134 archivefile = additionalArchives->At(0)->GetName();
1135 additionalArchives->RemoveAt(0);
1136 additionalArchives->Compress();
1137 }
1138
21ca8e59 1139 // Copy files locally
1140 ofstream out;
1141 out.open(output, ios::out);
1142 TMap *map;
1143 TString turl, dirname, filename, temp;
1144 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1145 gSystem->MakeDirectory(outputdir);
1146 gSystem->ChangeDirectory(outputdir);
4a8b667c 1147 Int_t ncopied = 0;
21ca8e59 1148 for (Int_t i=0; i<nfound; i++) {
1149 map = (TMap*)res->At(i);
1150 turl = map->GetValue("turl")->GetName();
1151 filename = gSystem->BaseName(turl.Data());
1152 dirname = gSystem->DirName(turl.Data());
1153 dirname = gSystem->BaseName(dirname.Data());
1154 gSystem->MakeDirectory(dirname);
43b807f6 1155
1156 TString source(turl);
1157 TString targetFileName(filename);
1158
1159 if (strlen(archivefile) > 0) {
4a8b667c 1160// TODO here the archive in which the file resides should be determined
1161// however whereis returns only a guid, and guid2lfn does not work
1162// Therefore we use the one provided as argument for now
1163 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1164 targetFileName = archivefile;
43b807f6 1165 }
1166 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
4a8b667c 1167 Bool_t success = kTRUE;
1168 if (additionalArchives) {
1169 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1170 TString target;
1171 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1172 gSystem->MakeDirectory(gSystem->DirName(target));
1173 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1174 }
1175 }
2ba4c287 1176
4a8b667c 1177 if (success) {
1178 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1179 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1180 ncopied++;
1181 }
21ca8e59 1182 }
1183 }
1184 gSystem->ChangeDirectory(cdir);
1185 delete res;
4a8b667c 1186 delete additionalArchives;
1187 return ncopied;
21ca8e59 1188}
1189
c57f56b7 1190//______________________________________________________________________________
1191Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1192{
1193// Create dataset for the grid data directory + run number.
3af861ff 1194 const Int_t gMaxEntries = 15000;
d3b18c4c 1195 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1196 if (!Connect()) {
1197 Error("CreateDataset", "Cannot create dataset with no grid connection");
1198 return kFALSE;
1199 }
1200
1201 // Cd workspace
d3b18c4c 1202 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1203 TString workdir = gGrid->GetHomeDirectory();
1204 workdir += fGridWorkingDir;
1205
1206 // Compose the 'find' command arguments
74013535 1207 TString format;
c57f56b7 1208 TString command;
780ec36d 1209 TString delimiter = pattern;
1210 delimiter.Strip();
1211 if (delimiter.Contains(" ")) delimiter = "";
1212 else delimiter = " ";
c57f56b7 1213 TString options = "-x collection ";
bb885a9e 1214 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1215 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1216 TString conditions = "";
3af861ff 1217 Int_t nstart = 0;
1218 Int_t ncount = 0;
1219 Int_t stage = 0;
c57f56b7 1220 TString file;
1221 TString path;
319593fb 1222 Int_t nruns = 0;
904f9f5f 1223 TString schunk, schunk2;
ab254fd1 1224 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1225 if (!fRunNumbers.Length() && !fRunRange[0]) {
1226 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1227 // Make a single data collection from data directory.
1228 path = fGridDataDir;
923e2ca5 1229 if (!DirectoryExists(path)) {
d2a409b2 1230 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1231 return kFALSE;
780ec36d 1232 }
923e2ca5 1233// CdWork();
d2a409b2 1234 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1235 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1236 while (1) {
1237 ncount = 0;
1238 stage++;
1239 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1240 command = "find ";
1241 command += Form("%s -o %d ",options.Data(), nstart);
1242 command += path;
780ec36d 1243 command += delimiter;
3af861ff 1244 command += pattern;
1245 command += conditions;
1246 printf("command: %s\n", command.Data());
1247 TGridResult *res = gGrid->Command(command);
1248 if (res) delete res;
1249 // Write standard output to file
1250 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1251 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1252 Bool_t nullFile = kFALSE;
1253 if (!hasGrep) {
1254 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1255 } else {
1256 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1257 if (nullFile) {
1258 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1259 gSystem->Exec("rm -f __tmp*");
1260 return kFALSE;
1261 }
1262 TString line;
1263 ifstream in;
1264 in.open("__tmp__");
1265 in >> line;
1266 in.close();
1267 gSystem->Exec("rm -f __tmp__");
1268 ncount = line.Atoi();
1269 }
1270 }
1271 if (ncount == gMaxEntries) {
1272 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1273 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1274 if (!cbase) cbase = cadd;
1275 else {
1276 cbase->Add(cadd);
1277 delete cadd;
defd7a3a 1278 }
3af861ff 1279 nstart += ncount;
1280 } else {
1281 if (cbase) {
1282 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1283 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1284 cbase->Add(cadd);
1285 delete cadd;
1286 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1287 delete cbase; cbase = 0;
1288 } else {
1289 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1290 }
1291 gSystem->Exec("rm -f __tmp*");
1292 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1293 break;
1294 }
0f389141 1295 }
1296 Bool_t fileExists = FileExists(file);
1297 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1298 // Copy xml file to alien space
0f389141 1299 if (fileExists) gGrid->Rm(file);
d2a409b2 1300 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1301 if (!FileExists(file)) {
1302 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1303 return kFALSE;
1304 }
1305 // Update list of files to be processed.
1306 }
1307 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1308 return kTRUE;
1309 }
c57f56b7 1310 // Several runs
a2f5fc01 1311 Bool_t nullResult = kTRUE;
a8739e8a 1312 if (fRunNumbers.Length()) {
1313 TObjArray *arr = fRunNumbers.Tokenize(" ");
1314 TObjString *os;
1315 TIter next(arr);
1316 while ((os=(TObjString*)next())) {
3af861ff 1317 nstart = 0;
1318 stage = 0;
0dab9d6d 1319 path = Form("%s/%s", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1320 if (!DirectoryExists(path)) continue;
1321// CdWork();
a8739e8a 1322 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1323 else file = Form("%s.xml", os->GetString().Data());
319593fb 1324 // If local collection file does not exist, create it via 'find' command.
3af861ff 1325 while (1) {
1326 ncount = 0;
1327 stage++;
1328 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1329 command = "find ";
1330 command += Form("%s -o %d ",options.Data(), nstart);
1331 command += path;
780ec36d 1332 command += delimiter;
3af861ff 1333 command += pattern;
1334 command += conditions;
1335 TGridResult *res = gGrid->Command(command);
1336 if (res) delete res;
1337 // Write standard output to file
1338 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1339 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1340 Bool_t nullFile = kFALSE;
1341 if (!hasGrep) {
1342 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1343 } else {
1344 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1345 if (nullFile) {
1346 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1347 gSystem->Exec("rm -f __tmp*");
1348 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1349 break;
3af861ff 1350 }
1351 TString line;
1352 ifstream in;
1353 in.open("__tmp__");
1354 in >> line;
1355 in.close();
1356 gSystem->Exec("rm -f __tmp__");
1357 ncount = line.Atoi();
1358 }
1359 nullResult = kFALSE;
1360 }
1361 if (ncount == gMaxEntries) {
1362 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1363 if (fNrunsPerMaster > 1) {
1364 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1365 file.Data(),gMaxEntries);
1366 return kFALSE;
1367 }
1368 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1369 if (!cbase) cbase = cadd;
1370 else {
1371 cbase->Add(cadd);
1372 delete cadd;
defd7a3a 1373 }
3af861ff 1374 nstart += ncount;
1375 } else {
c89a84bc 1376 if (cbase && fNrunsPerMaster<2) {
3af861ff 1377 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1378 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1379 cbase->Add(cadd);
1380 delete cadd;
1381 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1382 delete cbase; cbase = 0;
1383 } else {
1384 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1385 }
1386 gSystem->Exec("rm -f __tmp*");
1387 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1388 break;
d3339be3 1389 }
3af861ff 1390 }
a8739e8a 1391 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1392 // Check if there is one run per master job.
1393 if (fNrunsPerMaster<2) {
1394 if (FileExists(file)) {
0f389141 1395 if (fOverwriteMode) gGrid->Rm(file);
1396 else {
1397 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1398 continue;
1399 }
319593fb 1400 }
1401 // Copy xml file to alien space
1402 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1403 if (!FileExists(file)) {
1404 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1405 delete arr;
1406 return kFALSE;
1407 }
1408 } else {
1409 nruns++;
1410 if (((nruns-1)%fNrunsPerMaster) == 0) {
1411 schunk = os->GetString();
1412 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1413 } else {
1414 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1415 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1416 cbase->Add(cadd);
1417 delete cadd;
1418 }
1419 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1420 continue;
1421 }
1422 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1423 if (FileExists(schunk)) {
1424 if (fOverwriteMode) gGrid->Rm(file);
1425 else {
1426 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1427 continue;
1428 }
319593fb 1429 }
84fcd93f 1430 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1431 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1432 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1433 if (!FileExists(schunk)) {
1434 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1435 delete arr;
1436 return kFALSE;
1437 }
d3339be3 1438 }
a8739e8a 1439 }
1440 delete arr;
a2f5fc01 1441 if (nullResult) {
d3339be3 1442 Error("CreateDataset", "No valid dataset corresponding to the query!");
1443 return kFALSE;
1444 }
a8739e8a 1445 } else {
1446 // Process a full run range.
1447 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
0dab9d6d 1448 format = Form("%%s/%s", fRunPrefix.Data());
3af861ff 1449 nstart = 0;
1450 stage = 0;
74013535 1451 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1452 if (!DirectoryExists(path)) continue;
1453// CdWork();
74013535 1454 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1455 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1456 else file = Form(format.Data(), irun);
0f389141 1457 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1458 if (fOverwriteMode) gGrid->Rm(file);
1459 else {
1460 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1461 continue;
1462 }
a8739e8a 1463 }
319593fb 1464 // If local collection file does not exist, create it via 'find' command.
3af861ff 1465 while (1) {
1466 ncount = 0;
1467 stage++;
1468 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1469 command = "find ";
1470 command += Form("%s -o %d ",options.Data(), nstart);
1471 command += path;
780ec36d 1472 command += delimiter;
3af861ff 1473 command += pattern;
1474 command += conditions;
1475 TGridResult *res = gGrid->Command(command);
1476 if (res) delete res;
1477 // Write standard output to file
1478 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1479 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1480 Bool_t nullFile = kFALSE;
1481 if (!hasGrep) {
1482 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1483 } else {
1484 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1485 if (nullFile) {
1486 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1487 gSystem->Exec("rm -f __tmp*");
1604c9ed 1488 break;
3af861ff 1489 }
1490 TString line;
1491 ifstream in;
1492 in.open("__tmp__");
1493 in >> line;
1494 in.close();
1495 gSystem->Exec("rm -f __tmp__");
1496 ncount = line.Atoi();
1497 }
1498 nullResult = kFALSE;
1499 }
1500 if (ncount == gMaxEntries) {
1501 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1502 if (fNrunsPerMaster > 1) {
1503 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1504 file.Data(),gMaxEntries);
1505 return kFALSE;
1506 }
1507 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1508 if (!cbase) cbase = cadd;
1509 else {
1510 cbase->Add(cadd);
1511 delete cadd;
defd7a3a 1512 }
3af861ff 1513 nstart += ncount;
1514 } else {
c89a84bc 1515 if (cbase && fNrunsPerMaster<2) {
3af861ff 1516 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1517 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1518 cbase->Add(cadd);
1519 delete cadd;
1520 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1521 delete cbase; cbase = 0;
1522 } else {
1523 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1524 }
1525 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1526 break;
d3339be3 1527 }
319593fb 1528 }
a8739e8a 1529 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1530 // Check if there is one run per master job.
1531 if (fNrunsPerMaster<2) {
1532 if (FileExists(file)) {
0f389141 1533 if (fOverwriteMode) gGrid->Rm(file);
1534 else {
1535 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1536 continue;
1537 }
319593fb 1538 }
1539 // Copy xml file to alien space
1540 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1541 if (!FileExists(file)) {
1542 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1543 return kFALSE;
1544 }
1545 } else {
1546 nruns++;
95e5b448 1547 // Check if the collection for the chunk exist locally.
1548 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1549 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1550 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1551 else continue;
1552 }
84fcd93f 1553 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1554 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1555 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1556 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1557 } else {
1558 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1559 cbase->Add(cadd);
1560 delete cadd;
1561 }
74013535 1562 format = Form("%%s_%s.xml", fRunPrefix.Data());
1563 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1564 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1565 continue;
1566 }
904f9f5f 1567 schunk = schunk2;
319593fb 1568 if (FileExists(schunk)) {
0f389141 1569 if (fOverwriteMode) gGrid->Rm(schunk);
1570 else {
1571 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1572 continue;
1573 }
319593fb 1574 }
84fcd93f 1575 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1576 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1577 if (FileExists(schunk)) {
0f389141 1578 if (fOverwriteMode) gGrid->Rm(schunk);
1579 else {
1580 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1581 continue;
1582 }
95e5b448 1583 }
319593fb 1584 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1585 if (!FileExists(schunk)) {
1586 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1587 return kFALSE;
1588 }
1589 }
c57f56b7 1590 }
a2f5fc01 1591 if (nullResult) {
d3339be3 1592 Error("CreateDataset", "No valid dataset corresponding to the query!");
1593 return kFALSE;
1594 }
a8739e8a 1595 }
c57f56b7 1596 return kTRUE;
1597}
1598
1599//______________________________________________________________________________
1600Bool_t AliAnalysisAlien::CreateJDL()
1601{
1602// Generate a JDL file according to current settings. The name of the file is
1603// specified by fJDLName.
ec5acdce 1604 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
c57f56b7 1605 Bool_t error = kFALSE;
1606 TObjArray *arr = 0;
1607 Bool_t copy = kTRUE;
d3b18c4c 1608 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1609 Bool_t generate = kTRUE;
1610 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1611 if (!Connect()) {
1612 Error("CreateJDL", "Alien connection required");
1613 return kFALSE;
1614 }
1615 // Check validity of alien workspace
d3b18c4c 1616 TString workdir;
1617 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1618 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1619 workdir += fGridWorkingDir;
1620 if (generate) {
1621 TObjString *os;
1622 if (!fInputFiles) {
1623 Error("CreateJDL()", "Define some input files for your analysis.");
1624 error = kTRUE;
1625 }
1626 // Compose list of input files
1627 // Check if output files were defined
1628 if (!fOutputFiles.Length()) {
1629 Error("CreateJDL", "You must define at least one output file");
1630 error = kTRUE;
1631 }
1632 // Check if an output directory was defined and valid
1633 if (!fGridOutputDir.Length()) {
1634 Error("CreateJDL", "You must define AliEn output directory");
1635 error = kTRUE;
1636 } else {
d3b18c4c 1637 if (!fProductionMode) {
1638 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1639 if (!DirectoryExists(fGridOutputDir)) {
1640 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1641 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1642 } else {
1643 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1644 // error = kTRUE;
1645 }
67a3b254 1646 } else {
1647 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1648 }
1649 gGrid->Cd(workdir);
1650 }
c57f56b7 1651 }
1652 // Exit if any error up to now
1653 if (error) return kFALSE;
1654 // Set JDL fields
0f389141 1655 if (!fUser.IsNull()) {
1656 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1657 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1658 }
1659 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1660 TString mergeExec = fExecutable;
1661 mergeExec.ReplaceAll(".sh", "_merge.sh");
1662 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1663 mergeExec.ReplaceAll(".sh", ".C");
1664 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1665 if (!fArguments.IsNull())
1666 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1667 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1668 else {
1669 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1670 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1671 }
b3e07543 1672
1f0d1ca2 1673 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1674 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1675 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1676 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1677
0f389141 1678 if (fMaxInitFailed > 0) {
c57f56b7 1679 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1680 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1681 }
1682 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1683 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1684 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1685 }
1686 if (!IsOneStageMerging()) {
f47d5cb4 1687 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1688 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1689 }
1690 if (fSplitMode.Length()) {
c57f56b7 1691 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1692 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1693 }
1694 fMergingJDL->SetValue("Split", "\"se\"");
1695 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1696 if (!fAliROOTVersion.IsNull()) {
1697 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1698 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1699 }
1700 if (!fROOTVersion.IsNull()) {
c57f56b7 1701 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1702 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1703 }
1704 if (!fAPIVersion.IsNull()) {
c57f56b7 1705 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1706 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1707 }
648174cf 1708 if (!fExternalPackages.IsNull()) {
1709 arr = fExternalPackages.Tokenize(" ");
1710 TIter next(arr);
1711 while ((os=(TObjString*)next())) {
1712 TString pkgname = os->GetString();
1713 Int_t index = pkgname.Index("::");
1714 TString pkgversion = pkgname(index+2, pkgname.Length());
1715 pkgname.Remove(index);
1716 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1717 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1718 }
1719 delete arr;
1720 }
0f389141 1721 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1722 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1723 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1724 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1725 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1726 TString analysisFile = fExecutable;
1727 analysisFile.ReplaceAll(".sh", ".root");
1728 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1729 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1730 if (fAdditionalLibs.Length()) {
1731 arr = fAdditionalLibs.Tokenize(" ");
1732 TIter next(arr);
1733 while ((os=(TObjString*)next())) {
1734 if (os->GetString().Contains(".so")) continue;
1735 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1736 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1737 }
1738 delete arr;
1739 }
4e5c5506 1740 if (fPackages) {
1741 TIter next(fPackages);
1742 TObject *obj;
0f389141 1743 while ((obj=next())) {
4e5c5506 1744 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1745 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1746 }
4e5c5506 1747 }
95e16f01 1748 const char *comment = "List of output files and archives";
c57f56b7 1749 if (fOutputArchive.Length()) {
e4d9a069 1750 TString outputArchive = fOutputArchive;
1751 if (!fRegisterExcludes.IsNull()) {
1752 arr = fRegisterExcludes.Tokenize(" ");
1753 TIter next1(arr);
1754 while ((os=(TObjString*)next1())) {
1755 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1756 outputArchive.ReplaceAll(os->GetString(),"");
1757 }
1758 delete arr;
1759 }
1760 arr = outputArchive.Tokenize(" ");
c57f56b7 1761 TIter next(arr);
0f389141 1762 Bool_t first = kTRUE;
0f389141 1763 while ((os=(TObjString*)next())) {
0f389141 1764 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1765 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
0f389141 1766 else
95e16f01 1767 fGridJDL->AddToSet("Output", os->GetString());
1768 if (first) fGridJDL->AddToSetDescription("Output", comment);
0f389141 1769 first = kFALSE;
1770 }
c57f56b7 1771 delete arr;
f790bc1b 1772 // Output archive for the merging jdl
f790bc1b 1773 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1774 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1775 // Add normal output files, extra files + terminate files
1776 TString files = GetListOfFiles("outextter");
37d21c01 1777 // Do not register files in fRegisterExcludes
1778 if (!fRegisterExcludes.IsNull()) {
1779 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1780 TIter next1(arr);
1781 while ((os=(TObjString*)next1())) {
1782 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1783 files.ReplaceAll(os->GetString(),"");
1784 }
66d71516 1785 delete arr;
0f389141 1786 }
f790bc1b 1787 files.ReplaceAll(".root", "*.root");
ec5acdce 1788
1789 if (mgr->IsCollectThroughput())
95e16f01 1790 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",files.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 1791 else
1792 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1793 } else {
95f84779 1794 TString files = fOutputArchive;
1795 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1796 outputArchive = files;
f790bc1b 1797 }
0f389141 1798 arr = outputArchive.Tokenize(" ");
705adb3e 1799 TIter next2(arr);
0f389141 1800 first = kTRUE;
705adb3e 1801 while ((os=(TObjString*)next2())) {
0cdf65a8 1802 TString currentfile = os->GetString();
0cdf65a8 1803 if (!currentfile.Contains("@") && fCloseSE.Length())
95e16f01 1804 fMergingJDL->AddToSet("Output", Form("%s@%s",currentfile.Data(), fCloseSE.Data()));
0f389141 1805 else
95e16f01 1806 fMergingJDL->AddToSet("Output", currentfile);
1807 if (first) fMergingJDL->AddToSetDescription("Output", comment);
0f389141 1808 first = kFALSE;
1809 }
1810 delete arr;
c57f56b7 1811 }
149d288c 1812 arr = fOutputFiles.Tokenize(",");
c57f56b7 1813 TIter next(arr);
0f389141 1814 Bool_t first = kTRUE;
43da816a 1815 while ((os=(TObjString*)next())) {
e1eaf596 1816 // Ignore ouputs in jdl that are also in outputarchive
1817 TString sout = os->GetString();
f790bc1b 1818 sout.ReplaceAll("*", "");
1819 sout.ReplaceAll(".root", "");
e1eaf596 1820 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1821 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1822 // Ignore fRegisterExcludes
1823 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1824 if (!first) comment = NULL;
43da816a 1825 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1826 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
43da816a 1827 else
95e16f01 1828 fGridJDL->AddToSet("Output", os->GetString());
1829 if (first) fGridJDL->AddToSetDescription("Output", comment);
f790bc1b 1830 if (fMergeExcludes.Contains(sout)) continue;
1831 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1832 fMergingJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
f790bc1b 1833 else
95e16f01 1834 fMergingJDL->AddToSet("Output", os->GetString());
1835 if (first) fMergingJDL->AddToSetDescription("Output", comment);
1836 first = kFALSE;
43da816a 1837 }
c57f56b7 1838 delete arr;
0f389141 1839 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1840 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1841 TString validationScript = fValidationScript;
0f389141 1842 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1843 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1844 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1845 if (fMasterResubmitThreshold) {
1846 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1847 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1848 }
d2a409b2 1849 // Write a jdl with 2 input parameters: collection name and output dir name.
1850 WriteJDL(copy);
c57f56b7 1851 }
1852 // Copy jdl to grid workspace
a8739e8a 1853 if (copy) {
b5e4aaa7 1854 // Check if an output directory was defined and valid
1855 if (!fGridOutputDir.Length()) {
1856 Error("CreateJDL", "You must define AliEn output directory");
1857 return kFALSE;
1858 } else {
1859 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1860 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1861 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1862 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1863 } else {
1864 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1865 return kFALSE;
1866 }
1867 }
1868 gGrid->Cd(workdir);
1869 }
648174cf 1870 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1871 TString mergeJDLName = fExecutable;
1872 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1873 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1874 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1875 if (fProductionMode) {
648174cf 1876 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1877 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1878 }
648174cf 1879 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1880 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1881 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1882 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1883 Fatal("","Terminating");
1884// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1885 if (fMergeViaJDL) {
1886 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1887// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1888 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1889 Fatal("","Terminating");
0f389141 1890 }
648174cf 1891 }
c57f56b7 1892 if (fAdditionalLibs.Length()) {
e7c71df0 1893 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1894 TObjString *os;
1895 TIter next(arr);
1896 while ((os=(TObjString*)next())) {
c57f56b7 1897 if (os->GetString().Contains(".so")) continue;
4e5c5506 1898 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1899 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1900// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1901 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1902 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1903 Fatal("","Terminating");
c57f56b7 1904 }
1905 delete arr;
1906 }
4e5c5506 1907 if (fPackages) {
1908 TIter next(fPackages);
1909 TObject *obj;
1910 while ((obj=next())) {
fdbbc7be 1911 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1912 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1913// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1914 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1915 Form("%s/%s", workdir.Data(), obj->GetName())))
1916 Fatal("","Terminating");
4e5c5506 1917 }
1918 }
c57f56b7 1919 }
1920 return kTRUE;
1921}
1922
a8739e8a 1923//______________________________________________________________________________
d2a409b2 1924Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1925{
1926// Writes one or more JDL's corresponding to findex. If findex is negative,
1927// all run numbers are considered in one go (jdl). For non-negative indices
1928// they correspond to the indices in the array fInputFiles.
1929 if (!fInputFiles) return kFALSE;
d3b18c4c 1930 TObject *os;
1931 TString workdir;
1932 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1933 workdir += fGridWorkingDir;
f47d5cb4 1934 TString stageName = "$2";
1935 if (fProductionMode) stageName = "$4";
1936 if (!fMergeDirName.IsNull()) {
1937 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1938 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1939 } else {
1940 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1941 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1942 }
d3b18c4c 1943 if (fProductionMode) {
a8739e8a 1944 TIter next(fInputFiles);
b3e07543 1945 while ((os=next())) {
d3b18c4c 1946 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1947 }
56754bf6 1948 if (!fOutputToRunNo)
1949 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1950 else
1951 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1952 } else {
1953 if (!fRunNumbers.Length() && !fRunRange[0]) {
1954 // One jdl with no parameters in case input data is specified by name.
1955 TIter next(fInputFiles);
1956 while ((os=next()))
1957 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1958 if (!fOutputSingle.IsNull())
1959 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1960 else {
1961 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1962 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1963 }
1964 } else {
1965 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1966 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1967 if (!fOutputSingle.IsNull()) {
1968 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1969 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1970 } else {
1971 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1972 }
1973 }
a8739e8a 1974 }
1975
a8739e8a 1976 // Generate the JDL as a string
1977 TString sjdl = fGridJDL->Generate();
0f389141 1978 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1979 // Final merge jdl
1980 if (!fMergeDirName.IsNull()) {
1981 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1982 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1983 } else {
1984 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1985 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1986 }
b3e07543 1987 TString sjdl2 = fMergingJDL->Generate();
1988 Int_t index, index1;
95e16f01 1989 sjdl.ReplaceAll("\",\"", "\",\n \"");
a8739e8a 1990 sjdl.ReplaceAll("(member", "\n (member");
1991 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1992 sjdl.ReplaceAll("{", "{\n ");
1993 sjdl.ReplaceAll("};", "\n};");
1994 sjdl.ReplaceAll("{\n \n", "{\n");
1995 sjdl.ReplaceAll("\n\n", "\n");
1996 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 1997 sjdl1.ReplaceAll("\",\"", "\",\n \"");
0f389141 1998 sjdl1.ReplaceAll("(member", "\n (member");
1999 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2000 sjdl1.ReplaceAll("{", "{\n ");
2001 sjdl1.ReplaceAll("};", "\n};");
2002 sjdl1.ReplaceAll("{\n \n", "{\n");
2003 sjdl1.ReplaceAll("\n\n", "\n");
2004 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2005 sjdl2.ReplaceAll("\",\"", "\",\n \"");
b3e07543 2006 sjdl2.ReplaceAll("(member", "\n (member");
2007 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2008 sjdl2.ReplaceAll("{", "{\n ");
2009 sjdl2.ReplaceAll("};", "\n};");
2010 sjdl2.ReplaceAll("{\n \n", "{\n");
2011 sjdl2.ReplaceAll("\n\n", "\n");
2012 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 2013 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 2014 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 2015 index = sjdl.Index("JDLVariables");
2016 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 2017 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2018 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 2019 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 2020 index = fJobTag.Index(":");
2021 if (index < 0) index = fJobTag.Length();
2022 TString jobTag = fJobTag;
f47d5cb4 2023 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 2024 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 2025 if (fProductionMode) {
2026 sjdl1.Prepend("# Generated merging jdl (production mode) \
2027 \n# $1 = full alien path to output directory to be merged \
2028 \n# $2 = train number \
2029 \n# $3 = production (like LHC10b) \
2030 \n# $4 = merging stage \
2031 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2032 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2033 sjdl2.Prepend("# Generated merging jdl \
2034 \n# $1 = full alien path to output directory to be merged \
2035 \n# $2 = train number \
2036 \n# $3 = production (like LHC10b) \
2037 \n# $4 = merging stage \
2038 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2039 } else {
2040 sjdl1.Prepend("# Generated merging jdl \
2041 \n# $1 = full alien path to output directory to be merged \
2042 \n# $2 = merging stage \
03335324 2043 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2044 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2045 sjdl2.Prepend("# Generated merging jdl \
2046 \n# $1 = full alien path to output directory to be merged \
2047 \n# $2 = merging stage \
03335324 2048 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2049 }
0f389141 2050 index = sjdl1.Index("JDLVariables");
2051 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 2052 index = sjdl2.Index("JDLVariables");
2053 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 2054 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2055 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
2056 index = sjdl2.Index("Split =");
2057 if (index>=0) {
2058 index1 = sjdl2.Index("\n", index);
2059 sjdl2.Remove(index, index1-index+1);
2060 }
2061 index = sjdl2.Index("SplitMaxInputFileNumber");
2062 if (index>=0) {
2063 index1 = sjdl2.Index("\n", index);
2064 sjdl2.Remove(index, index1-index+1);
2065 }
2066 index = sjdl2.Index("InputDataCollection");
2067 if (index>=0) {
2068 index1 = sjdl2.Index(";", index);
2069 sjdl2.Remove(index, index1-index+1);
2070 }
2071 index = sjdl2.Index("InputDataListFormat");
2072 if (index>=0) {
2073 index1 = sjdl2.Index("\n", index);
2074 sjdl2.Remove(index, index1-index+1);
2075 }
2076 index = sjdl2.Index("InputDataList");
2077 if (index>=0) {
2078 index1 = sjdl2.Index("\n", index);
2079 sjdl2.Remove(index, index1-index+1);
2080 }
f47d5cb4 2081 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2082 // Write jdl to file
a8739e8a 2083 ofstream out;
d2a409b2 2084 out.open(fJDLName.Data(), ios::out);
a8739e8a 2085 if (out.bad()) {
d3b18c4c 2086 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2087 return kFALSE;
2088 }
2089 out << sjdl << endl;
b3e07543 2090 out.close();
0f389141 2091 TString mergeJDLName = fExecutable;
2092 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2093 if (fMergeViaJDL) {
2094 ofstream out1;
2095 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2096 if (out1.bad()) {
d3b18c4c 2097 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2098 return kFALSE;
2099 }
2100 out1 << sjdl1 << endl;
b3e07543 2101 out1.close();
2102 ofstream out2;
2103 TString finalJDL = mergeJDLName;
2104 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2105 out2.open(finalJDL.Data(), ios::out);
2106 if (out2.bad()) {
2107 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2108 return kFALSE;
2109 }
2110 out2 << sjdl2 << endl;
2111 out2.close();
0f389141 2112 }
a8739e8a 2113
2114 // Copy jdl to grid workspace
2115 if (!copy) {
d3b18c4c 2116 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2117 } else {
d2a409b2 2118 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2119 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2120 TString finalJDL = mergeJDLName;
2121 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2122 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2123 if (fProductionMode) {
b5fe9cba 2124 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2125 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2126 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2127 }
d2a409b2 2128 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2129 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2130 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2131 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2132// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2133 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2134 Fatal("","Terminating");
0f389141 2135 if (fMergeViaJDL) {
b3e07543 2136 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2137// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2138// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2139 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2140 Fatal("","Terminating");
2141 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2142 Fatal("","Terminating");
0f389141 2143 }
a8739e8a 2144 }
2145 return kTRUE;
2146}
2147
c57f56b7 2148//______________________________________________________________________________
5513444a 2149Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2150{
2151// Returns true if file exists.
5513444a 2152 if (!gGrid) return kFALSE;
2ea55496 2153 TString slfn = lfn;
2154 slfn.ReplaceAll("alien://","");
2155 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2156 if (!res) return kFALSE;
2157 TMap *map = dynamic_cast<TMap*>(res->At(0));
2158 if (!map) {
2159 delete res;
2160 return kFALSE;
2161 }
2162 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2163 if (!objs || !objs->GetString().Length()) {
2164 delete res;
2165 return kFALSE;
2166 }
2167 delete res;
2168 return kTRUE;
2169}
2170
923e2ca5 2171//______________________________________________________________________________
2172Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2173{
2174// Returns true if directory exists. Can be also a path.
2175 if (!gGrid) return kFALSE;
2176 // Check if dirname is a path
2177 TString dirstripped = dirname;
2178 dirstripped = dirstripped.Strip();
2179 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2180 TString dir = gSystem->BaseName(dirstripped);
2181 dir += "/";
2182 TString path = gSystem->DirName(dirstripped);
2183 TGridResult *res = gGrid->Ls(path, "-F");
2184 if (!res) return kFALSE;
2185 TIter next(res);
2186 TMap *map;
2187 TObject *obj;
2188 while ((map=dynamic_cast<TMap*>(next()))) {
2189 obj = map->GetValue("name");
2190 if (!obj) break;
2191 if (dir == obj->GetName()) {
2192 delete res;
2193 return kTRUE;
2194 }
2195 }
2196 delete res;
2197 return kFALSE;
2198}
2199
c57f56b7 2200//______________________________________________________________________________
a2f5fc01 2201void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2202{
2203// Check input data type.
a2f5fc01 2204 isCollection = kFALSE;
2205 isXml = kFALSE;
2206 useTags = kFALSE;
c57f56b7 2207 if (!gGrid) {
2208 Error("CheckDataType", "No connection to grid");
2209 return;
2210 }
a2f5fc01 2211 isCollection = IsCollection(lfn);
c57f56b7 2212 TString msg = "\n##### file: ";
2213 msg += lfn;
a2f5fc01 2214 if (isCollection) {
c57f56b7 2215 msg += " type: raw_collection;";
2216 // special treatment for collections
a2f5fc01 2217 isXml = kFALSE;
c57f56b7 2218 // check for tag files in the collection
2219 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2220 if (!res) {
2221 msg += " using_tags: No (unknown)";
23329835 2222 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2223 return;
2224 }
2225 const char* typeStr = res->GetKey(0, "origLFN");
2226 if (!typeStr || !strlen(typeStr)) {
2227 msg += " using_tags: No (unknown)";
23329835 2228 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2229 return;
2230 }
2231 TString file = typeStr;
a2f5fc01 2232 useTags = file.Contains(".tag");
2233 if (useTags) msg += " using_tags: Yes";
c57f56b7 2234 else msg += " using_tags: No";
23329835 2235 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2236 return;
2237 }
2238 TString slfn(lfn);
2239 slfn.ToLower();
a2f5fc01 2240 isXml = slfn.Contains(".xml");
2241 if (isXml) {
c57f56b7 2242 // Open xml collection and check if there are tag files inside
2243 msg += " type: xml_collection;";
2244 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2245 if (!coll) {
2246 msg += " using_tags: No (unknown)";
23329835 2247 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2248 return;
2249 }
2250 TMap *map = coll->Next();
2251 if (!map) {
2252 msg += " using_tags: No (unknown)";
23329835 2253 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2254 return;
2255 }
2256 map = (TMap*)map->GetValue("");
2257 TString file;
2258 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2259 useTags = file.Contains(".tag");
c57f56b7 2260 delete coll;
a2f5fc01 2261 if (useTags) msg += " using_tags: Yes";
c57f56b7 2262 else msg += " using_tags: No";
23329835 2263 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2264 return;
2265 }
a2f5fc01 2266 useTags = slfn.Contains(".tag");
c57f56b7 2267 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2268 else msg += " type: unknown file;";
a2f5fc01 2269 if (useTags) msg += " using_tags: Yes";
c57f56b7 2270 else msg += " using_tags: No";
23329835 2271 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2272}
2273
4e5c5506 2274//______________________________________________________________________________
2275void AliAnalysisAlien::EnablePackage(const char *package)
2276{
2277// Enables a par file supposed to exist in the current directory.
2278 TString pkg(package);
2279 pkg.ReplaceAll(".par", "");
2280 pkg += ".par";
2281 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2282 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2283 return;
2284 }
2285 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2286 Info("EnablePackage", "AliEn plugin will use .par packages");
2287 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2288 if (!fPackages) {
2289 fPackages = new TObjArray();
2290 fPackages->SetOwner();
2291 }
2292 fPackages->Add(new TObjString(pkg));
2293}
2294
3bdcb562 2295//______________________________________________________________________________
2296TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2297{
2298// Make a tree from files having the location specified in fFileForTestMode.
2299// Inspired from JF's CreateESDChain.
2300 if (fFileForTestMode.IsNull()) {
2301 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2302 return NULL;
2303 }
2304 if (gSystem->AccessPathName(fFileForTestMode)) {
2305 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2306 return NULL;
2307 }
2308 // Open the file
2309 ifstream in;
2310 in.open(fFileForTestMode);
2311 Int_t count = 0;
2312 // Read the input list of files and add them to the chain
b505f1f8 2313 TString line;
d6c60197 2314 TString streeName(treeName);
2315 if (IsUseMCchain()) streeName = "TE";
2316 TChain *chain = new TChain(streeName);
06f67114 2317 TList *friends = new TList();
2318 TChain *cfriend = 0;
2319 if (!fFriendChainName.IsNull()) {
2320 TObjArray *list = fFriendChainName.Tokenize(" ");
2321 TIter next(list);
2322 TObjString *str;
2323 while((str=(TObjString*)next())) {
2324 cfriend = new TChain(streeName, str->GetName());
2325 friends->Add(cfriend);
2326 chain->AddFriend(cfriend);
2327 }
2328 delete list;
2329 }
2330 TString bpath;
2331 TIter nextfriend(friends);
b505f1f8 2332 while (in.good())
2333 {
3bdcb562 2334 in >> line;
ffbe06b9 2335 if (line.IsNull() || line.BeginsWith("#")) continue;
3bdcb562 2336 if (count++ == fNtestFiles) break;
2337 TString esdFile(line);
2338 TFile *file = TFile::Open(esdFile);
b505f1f8 2339 if (file && !file->IsZombie()) {
2340 chain->Add(esdFile);
3bdcb562 2341 file->Close();
b505f1f8 2342 if (!fFriendChainName.IsNull()) {
4b5d9e21 2343 if (esdFile.Index("#") > -1)
2344 esdFile.Remove(esdFile.Index("#"));
06f67114 2345 bpath = gSystem->DirName(esdFile);
2346 bpath += "/";
2347 TString fileFriend;
2348 nextfriend.Reset();
2349 while ((cfriend=(TChain*)nextfriend())) {
2350 fileFriend = bpath;
2351 fileFriend += cfriend->GetTitle();
2352 file = TFile::Open(fileFriend);
2353 if (file && !file->IsZombie()) {
2354 file->Close();
2355 cfriend->Add(fileFriend);
2356 } else {
2357 Fatal("GetChainForTestMode", "Cannot open friend file: %s", fileFriend.Data());
2358 return 0;
2359 }
2360 }
4b5d9e21 2361 }
3bdcb562 2362 } else {
2363 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2364 }
b505f1f8 2365 }
2366 in.close();
2367 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2368 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2369 delete chain;
06f67114 2370 friends->Delete();
2371 delete friends;
3bdcb562 2372 return NULL;
b505f1f8 2373 }
b505f1f8 2374 return chain;
3bdcb562 2375}
2376
d2a409b2 2377//______________________________________________________________________________
2378const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2379{
2380// Get job status for all jobs with jobid>jobidstart.
2381 static char mstatus[20];
2382 mstatus[0] = '\0';
2383 nrunning = 0;
2384 nwaiting = 0;
2385 nerror = 0;
2386 ndone = 0;
2387 TGridJobStatusList *list = gGrid->Ps("");
2388 if (!list) return mstatus;
2389 Int_t nentries = list->GetSize();
2390 TGridJobStatus *status;
2391 Int_t pid;
2392 for (Int_t ijob=0; ijob<nentries; ijob++) {
2393 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2394 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2395 if (pid<jobidstart) continue;
2396 if (pid == lastid) {
8ddad121 2397 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2398 }
2399 switch (status->GetStatus()) {
2400 case TGridJobStatus::kWAITING:
2401 nwaiting++; break;
2402 case TGridJobStatus::kRUNNING:
2403 nrunning++; break;
2404 case TGridJobStatus::kABORTED:
2405 case TGridJobStatus::kFAIL:
2406 case TGridJobStatus::kUNKNOWN:
2407 nerror++; break;
2408 case TGridJobStatus::kDONE:
2409 ndone++;
2410 }
2411 }
2412 list->Delete();
2413 delete list;
2414 return mstatus;
2415}
2416
c57f56b7 2417//______________________________________________________________________________
2418Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2419{
2420// Returns true if file is a collection. Functionality duplicated from
2421// TAlien::Type() because we don't want to directly depend on TAlien.
2422 if (!gGrid) {
2423 Error("IsCollection", "No connection to grid");
2424 return kFALSE;
2425 }
2426 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2427 if (!res) return kFALSE;
2428 const char* typeStr = res->GetKey(0, "type");
2429 if (!typeStr || !strlen(typeStr)) return kFALSE;
2430 if (!strcmp(typeStr, "collection")) return kTRUE;
2431 delete res;
2432 return kFALSE;
2433}
2434
fe2d7fc2 2435//______________________________________________________________________________
2436Bool_t AliAnalysisAlien::IsSingleOutput() const
2437{
2438// Check if single-ouput option is on.
2439 return (!fOutputSingle.IsNull());
2440}
2441
16a4353c 2442//______________________________________________________________________________
2443void AliAnalysisAlien::Print(Option_t *) const
2444{
2445// Print current plugin settings.
84fcd93f 2446 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2447 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2448 if (mgr && mgr->IsProofMode()) {
2449 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2450 if (TestBit(AliAnalysisGrid::kTest))
2451 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2452 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2453 if (!fProofDataSet.IsNull())
2454 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2455 if (fProofReset==1)
2456 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2457 if (fProofReset>1)
2458 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
874de7ea 2459 if (!fROOTVersion.IsNull())
2460 printf("= ROOT version requested________________________ %s\n", fROOTVersion.Data());
d3b18c4c 2461 else
2462 printf("= ROOT version requested________________________ default\n");
2463 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2464 if (!fAliRootMode.IsNull())
2465 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2466 if (fNproofWorkers)
2467 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2468 if (fNproofWorkersPerSlave)
2469 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2470 if (TestSpecialBit(kClearPackages))
2471 printf("= ClearPackages requested...\n");
2472 if (fIncludePath.Data())
2473 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2474 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2475 if (fPackages && fPackages->GetEntries()) {
2476 TIter next(fPackages);
2477 TObject *obj;
2478 TString list;
2479 while ((obj=next())) list += obj->GetName();
2480 printf("= Par files to be used: ________________________ %s\n", list.Data());
2481 }
2482 if (TestSpecialBit(kProofConnectGrid))
2483 printf("= Requested PROOF connection to grid\n");
2484 return;
2485 }
e1c22e21 2486 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2487 if (fOverwriteMode) {
2488 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2489 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2490 }
348be253 2491 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
780ec36d 2492 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO:Print");
84fcd93f 2493 printf("= Production mode:______________________________ %d\n", fProductionMode);
2494 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2495 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2496 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2497 if (fUser.Length())
84fcd93f 2498 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2499 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2500 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
780ec36d 2501 TString basedatadir = fGridDataDir;
2502 TString pattern = fDataPattern;
2503 pattern.Strip();
2504 Int_t ind = pattern.Index(" ");
2505 if (ind>=0) {
2506 basedatadir += "/%run%/";
2507 basedatadir += pattern(0, ind);
2508 pattern = pattern(ind+1, pattern.Length());
2509 }
2510 printf("= Data base directory path requested: __________ %s\n", basedatadir.Data());
2511 printf("= Data search pattern: _________________________ %s\n", pattern.Data());
84fcd93f 2512 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2513 if (fRunNumbers.Length())
84fcd93f 2514 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2515 if (fRunRange[0])
74013535 2516 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2517 if (!fRunRange[0] && !fRunNumbers.Length()) {
2518 TIter next(fInputFiles);
2519 TObject *obj;
2520 TString list;
2521 while ((obj=next())) list += obj->GetName();
84fcd93f 2522 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2523 }
2524 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2525 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2526 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2527 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2528 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2529 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2530 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2531 printf("=====================================================================\n");
2532 printf("= Job price: ___________________________________ %d\n", fPrice);
2533 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2534 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2535 if (fMaxInitFailed>0)
84fcd93f 2536 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2537 if (fMasterResubmitThreshold>0)
84fcd93f 2538 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2539 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2540 if (fNrunsPerMaster>0)
84fcd93f 2541 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2542 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2543 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2544 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2545 if (fArguments.Length())
84fcd93f 2546 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2547 if (fExecutableArgs.Length())
2548 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2549 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2550 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2551 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2552 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2553 if (fDatasetName)
84fcd93f 2554 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2555 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2556 if (fIncludePath.Data())
84fcd93f 2557 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2558 if (fCloseSE.Length())
84fcd93f 2559 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2560 if (fFriendChainName.Length())
84fcd93f 2561 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2562 if (fPackages && fPackages->GetEntries()) {
16a4353c 2563 TIter next(fPackages);
2564 TObject *obj;
2565 TString list;
2566 while ((obj=next())) list += obj->GetName();
84fcd93f 2567 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2568 }
2569}
2570
c57f56b7 2571//______________________________________________________________________________
2572void AliAnalysisAlien::SetDefaults()
2573{
2574// Set default values for everything. What cannot be filled will be left empty.
2575 if (fGridJDL) delete fGridJDL;
2576 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2577 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2578 fPrice = 1;
2579 fTTL = 30000;
2580 fSplitMaxInputFileNumber = 100;
2581 fMaxInitFailed = 0;
2582 fMasterResubmitThreshold = 0;
bb885a9e 2583 fNtestFiles = 10;
149d288c 2584 fNreplicas = 2;
a8739e8a 2585 fRunRange[0] = 0;
2586 fRunRange[1] = 0;
8e3bd06a 2587 fRunPrefix = "%d";
319593fb 2588 fNrunsPerMaster = 1;
16a4353c 2589 fMaxMergeFiles = 100;
c57f56b7 2590 fRunNumbers = "";
2591 fExecutable = "analysis.sh";
5aa94dc3 2592 fExecutableCommand = "root -b -q -x";
c57f56b7 2593 fArguments = "";
631c0b05 2594 fExecutableArgs = "";
c57f56b7 2595 fAnalysisMacro = "myAnalysis.C";
2596 fAnalysisSource = "";
2597 fAdditionalLibs = "";
2598 fSplitMode = "se";
2599 fAPIVersion = "";
2600 fROOTVersion = "";
2601 fAliROOTVersion = "";
2602 fUser = ""; // Your alien user name
2603 fGridWorkingDir = "";
2604 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2605 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2606 fFriendChainName = "";
c57f56b7 2607 fGridOutputDir = "output";
b83f84f0 2608 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2609 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2610 fInputFormat = "xml-single";
2611 fJDLName = "analysis.jdl";
c6cb3634 2612 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2613 fMergeExcludes = "";
0f389141 2614 fMergeViaJDL = 0;
348be253 2615 SetUseCopy(kTRUE);
2616 SetCheckCopy(kTRUE);
149d288c 2617 SetDefaultOutputs(kTRUE);
e1c22e21 2618 fOverwriteMode = 1;
c57f56b7 2619}
2620
a41b2882 2621//______________________________________________________________________________
2622void AliAnalysisAlien::SetFriendChainName(const char *name, const char *libnames)
2623{
2624 // Set file name for the chain of friends and optionally additional libs to be loaded.
2625 // Libs should be separated by blancs.
2626 fFriendChainName = name;
06f67114 2627 fFriendChainName.ReplaceAll(",", " ");
2628 fFriendChainName.Strip();
2629 fFriendChainName.ReplaceAll(" ", " ");
2630
a41b2882 2631 fFriendLibs = libnames;
1a3f255c 2632 if (fFriendLibs.Length()) {
2633 if(!fFriendLibs.Contains(".so"))
2634 Fatal("SetFriendChainName()", "You should provide explicit library names (with extension)");
2635 fFriendLibs.ReplaceAll(",", " ");
2636 fFriendLibs.Strip();
2637 fFriendLibs.ReplaceAll(" ", " ");
a41b2882 2638 }
2639}
2640
874de7ea 2641//______________________________________________________________________________
2642void AliAnalysisAlien::SetRootVersionForProof(const char *version)
2643{
2644// Obsolete method. Use SetROOTVersion instead
2645 Warning("SetRootVersionForProof", "Obsolete. Use SetROOTVersion instead");
95e16f01 2646 if (fROOTVersion.IsNull()) SetROOTVersion(version);
2647 else Error("SetRootVersionForProof", "ROOT version already set to %s", fROOTVersion.Data());
874de7ea 2648}
2649
0f389141 2650//______________________________________________________________________________
b3e07543 2651Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2652{
b3e07543 2653// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2654 // First check if the result is already in the output directory.
2655 if (FileExists(Form("%s/%s",aliendir,filename))) {
2656 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2657 return kFALSE;
2658 }
b3e07543 2659 // Now check the last stage done.
2660 Int_t stage = 0;
2661 while (1) {
2662 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2663 stage++;
2664 }
2665 // Next stage of merging
2666 stage++;
2667 TString pattern = "*root_archive.zip";
2668 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2669 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2670 if (res) delete res;
2671 // Write standard output to file
2672 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2673 // Count the number of files inside
2674 ifstream ifile;
2675 ifile.open(Form("Stage_%d.xml",stage));
2676 if (!ifile.good()) {
2677 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2678 return kFALSE;
b3e07543 2679 }
2680 TString line;
2681 Int_t nfiles = 0;
2682 while (!ifile.eof()) {
2683 ifile >> line;
2684 if (line.Contains("/event")) nfiles++;
7c2cd90a 2685 }
b3e07543 2686 ifile.close();
2687 if (!nfiles) {
2688 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2689 return kFALSE;
2690 } else {
2691 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2692 }
b3e07543 2693 // Copy the file in the output directory
2694 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2695// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2696 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2697 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2698 // Check if this is the last stage to be done.
2699 Bool_t laststage = (nfiles<nperchunk);
2700 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
0ad3ea1f 2701 Int_t jobId = 0;
b3e07543 2702 if (laststage) {
2703 printf("### Submiting final merging stage %d\n", stage);
2704 TString finalJDL = jdl;
2705 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2706 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
0ad3ea1f 2707 jobId = SubmitSingleJob(query);
b3e07543 2708 } else {
2709 printf("### Submiting merging stage %d\n", stage);
03335324 2710 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
0ad3ea1f 2711 jobId = SubmitSingleJob(query);
7c2cd90a 2712 }
0ad3ea1f 2713 if (!jobId) return kFALSE;
2714
2715 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
2716 fGridJobIDs.Append(Form("%d", jobId));
2717 if (!fGridStages.IsNull()) fGridStages.Append(" ");
2718 fGridStages.Append(Form("%s_merge_stage%d",
2719 laststage ? "final" : "partial", stage));
2720
b3e07543 2721 return kTRUE;
2722}
7c2cd90a 2723
105631b5 2724//______________________________________________________________________________
2725AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2726{
2727// Loat the analysis manager from a file.
2728 TFile *file = TFile::Open(fname);
2729 if (!file) {
2730 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2731 return 0;
2732 }
2733 TIter nextkey(file->GetListOfKeys());
2734 AliAnalysisManager *mgr = 0;
2735 TKey *key;
2736 while ((key=(TKey*)nextkey())) {
2737 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2738 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2739 }
2740 if (!mgr)
2741 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2742 return mgr;
2743}
2744
7c2cd90a 2745//______________________________________________________________________________
2746Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2747{
2748// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2749 if (!gGrid) return 0;
2750 printf("=> %s ------> ",query);
2751 TGridResult *res = gGrid->Command(query);
2752 if (!res) return 0;
2753 TString jobId = res->GetKey(0,"jobId");
2754 delete res;
2755 if (jobId.IsNull()) {
2756 printf("submission failed. Reason:\n");
2757 gGrid->Stdout();
2758 gGrid->Stderr();
2759 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2760 return 0;
2761 }
0ad3ea1f 2762 Int_t ijobId = jobId.Atoi();
2763 printf(" Job id: '%s' (%d)\n", jobId.Data(), ijobId);
2764 return ijobId;
7c2cd90a 2765}
2766
ec5acdce 2767//______________________________________________________________________________
2768Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2769{
2770// Merges a collection of output files using concatenation.
d66aa2a9 2771 TString scoll(collection);
2772 if (!scoll.Contains(".xml")) return kFALSE;
ec5acdce 2773 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2774 if (!coll) {
2775 ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2776 return kFALSE;
2777 }
2778 // Iterate grid collection
2779 TString outtmp;
2780 Bool_t merged = kFALSE;
2781 Int_t ifile = 0;
2782 while (coll->Next()) {
2783 TString fname = gSystem->DirName(coll->GetTURL());
2784 fname += "/";
2785 fname += output;
2786 outtmp = Form("%d_%s", ifile, output);
2787 if (!TFile::Cp(fname, outtmp)) {
2788 ::Error("MergeInfo", "Could not copy %s", fname.Data());
2789 continue;
2790 }
2791 ifile++;
2792 if (ifile<2) {
2793 gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2794 continue;
2795 }
2796 gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2797 gSystem->Exec("cp tempmerged lastmerged");
2798 }
2799 if (ifile) {
2800 gSystem->Exec(Form("cp lastmerged %s", output));
2801 gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2802 merged = kTRUE;
2803 }
2804 return merged;
2805}
2806
7c2cd90a 2807//______________________________________________________________________________
b3e07543 2808Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2809{
b3e07543 2810// Merge given output files from basedir. Basedir can be an alien output directory
2811// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2812// files in a group (ignored for xml input). Merging can be done in stages:
2813// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2814// stage=1 : works with an xml of all root_archive.zip in the output directory
2815// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2816 TString outputFile = output;
0f389141 2817 TString command;
a2f5fc01 2818 TString outputChunk;
2819 TString previousChunk = "";
b3e07543 2820 TObjArray *listoffiles = new TObjArray();
2821// listoffiles->SetOwner();
a2f5fc01 2822 Int_t countChunk = 0;
2823 Int_t countZero = nmaxmerge;
0f389141 2824 Bool_t merged = kTRUE;
d66aa2a9 2825 Bool_t isGrid = kTRUE;
a2f5fc01 2826 Int_t index = outputFile.Index("@");
2827 if (index > 0) outputFile.Remove(index);
7c2cd90a 2828 TString inputFile = outputFile;
b3e07543 2829 TString sbasedir = basedir;
2830 if (sbasedir.Contains(".xml")) {
2831 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2832 nmaxmerge = 9999999;
2833 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2834 if (!coll) {
2835 ::Error("MergeOutput", "Input XML collection empty.");
2836 return kFALSE;
2837 }
2838 // Iterate grid collection
2839 while (coll->Next()) {
2840 TString fname = gSystem->DirName(coll->GetTURL());
2841 fname += "/";
2842 fname += inputFile;
2843 listoffiles->Add(new TNamed(fname.Data(),""));
2844 }
ffbe06b9 2845 } else if (sbasedir.Contains(".txt")) {
d66aa2a9 2846 // The file having the .txt extension is expected to contain a list of
2847 // folders where the output files will be looked. For alien folders,
2848 // the full folder LFN is expected (starting with alien://)
ffbe06b9 2849 // Assume lfn's on each line
2850 TString line;
2851 ifstream in;
2852 in.open(sbasedir);
d66aa2a9 2853 if (in.fail()) {
2854 ::Error("MergeOutput", "File %s cannot be opened. Merging stopped." ,sbasedir.Data());
2855 return kTRUE;
2856 }
ffbe06b9 2857 Int_t nfiles = 0;
2858 while (in.good()) {
2859 in >> line;
2860 if (line.IsNull() || line.BeginsWith("#")) continue;
d66aa2a9 2861 line.Strip();
2862 if (!line.Contains("alien:")) isGrid = kFALSE;
2863 line += "/";
2864 line += outputFile;
ffbe06b9 2865 nfiles++;
2866 listoffiles->Add(new TNamed(line.Data(),""));
2867 }
d66aa2a9 2868 in.close();
ffbe06b9 2869 if (!nfiles) {
2870 ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data());
2871 delete listoffiles;
2872 return kFALSE;
2873 }
b3e07543 2874 } else {
2875 command = Form("find %s/ *%s", basedir, inputFile.Data());
2876 printf("command: %s\n", command.Data());
2877 TGridResult *res = gGrid->Command(command);
2878 if (!res) {
2879 ::Error("MergeOutput","No result for the find command\n");
2880 delete listoffiles;
2881 return kFALSE;
2882 }
2883 TIter nextmap(res);
2884 TMap *map = 0;
2885 while ((map=(TMap*)nextmap())) {
2886 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2887 if (!objs || !objs->GetString().Length()) {
2888 // Nothing found - skip this output
2889 delete res;
2890 delete listoffiles;
2891 return kFALSE;
2892 }
2893 listoffiles->Add(new TNamed(objs->GetName(),""));
2894 }
2895 delete res;
2896 }
2897 if (!listoffiles->GetEntries()) {
7c2cd90a 2898 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2899 delete listoffiles;
0f389141 2900 return kFALSE;
2901 }
2902
2903 TFileMerger *fm = 0;
b3e07543 2904 TIter next0(listoffiles);
2905 TObjArray *listoffilestmp = new TObjArray();
2906 listoffilestmp->SetOwner();
2907 TObject *nextfile;
2908 TString snextfile;
2909 // Keep only the files at upper level
2910 Int_t countChar = 0;
2911 while ((nextfile=next0())) {
2912 snextfile = nextfile->GetName();
2913 Int_t crtCount = snextfile.CountChar('/');
2914 if (nextfile == listoffiles->First()) countChar = crtCount;
2915 if (crtCount < countChar) countChar = crtCount;
2916 }
2917 next0.Reset();
2918 while ((nextfile=next0())) {
2919 snextfile = nextfile->GetName();
2920 Int_t crtCount = snextfile.CountChar('/');
2921 if (crtCount > countChar) {
2922 delete nextfile;
2923 continue;
2924 }
2925 listoffilestmp->Add(nextfile);
2926 }
2927 delete listoffiles;
2928 listoffiles = listoffilestmp; // Now contains 'good' files
2929 listoffiles->Print();
2930 TIter next(listoffiles);
7c2cd90a 2931 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2932 outputChunk = outputFile;
2933 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2934 // Check for existent temporary merge files
e1c22e21 2935 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2936 // Preserve old merging functionality for stage 0.
2937 if (stage==0) {
2938 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2939 while (1) {
2940 // Skip as many input files as in a chunk
2ea55496 2941 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2942 nextfile = next();
2943 if (!nextfile) {
2ea55496 2944 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2945 delete listoffiles;
2ea55496 2946 return kFALSE;
2947 }
b3e07543 2948 snextfile = nextfile->GetName();
7c2cd90a 2949 }
2950 outputChunk = outputFile;
2951 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2952 countChunk++;
2953 if (gSystem->AccessPathName(outputChunk)) continue;
2954 // Merged file with chunks up to <countChunk> found
2955 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2956 previousChunk = outputChunk;
2957 break;
2958 }
2959 }
2960 countZero = nmaxmerge;
2961
b3e07543 2962 while ((nextfile=next())) {
2963 snextfile = nextfile->GetName();
2ea55496 2964 // Loop 'find' results and get next LFN
7c2cd90a 2965 if (countZero == nmaxmerge) {
2966 // First file in chunk - create file merger and add previous chunk if any.
d66aa2a9 2967 fm = new TFileMerger(isGrid);
7c2cd90a 2968 fm->SetFastMethod(kTRUE);
2969 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2970 outputChunk = outputFile;
2971 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2972 }
2973 // If last file found, put merged results in the output file
b3e07543 2974 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2975 // Add file to be merged and decrement chunk counter.
b3e07543 2976 fm->AddFile(snextfile);
7c2cd90a 2977 countZero--;
b3e07543 2978 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2979 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2980 // Nothing found - skip this output
2981 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2982 merged = kFALSE;
2983 break;
7c2cd90a 2984 }
2985 fm->OutputFile(outputChunk);
2986 // Merge the outputs, then go to next chunk
2987 if (!fm->Merge()) {
2988 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2989 merged = kFALSE;
2990 break;
7c2cd90a 2991 } else {
2992 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2993 gSystem->Unlink(previousChunk);
2994 }
b3e07543 2995 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2996 countChunk++;
2997 countZero = nmaxmerge;
2998 previousChunk = outputChunk;
0f389141 2999 }
0f389141 3000 }
b3e07543 3001 delete listoffiles;
3002 delete fm;
7c2cd90a 3003 return merged;
3004 }
3005 // Merging stage different than 0.
3006 // Move to the begining of the requested chunk.
d66aa2a9 3007 fm = new TFileMerger(isGrid);
7c2cd90a 3008 fm->SetFastMethod(kTRUE);
b3e07543 3009 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
3010 delete listoffiles;
7c2cd90a 3011 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3012 // Nothing found - skip this output
3013 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
3014 delete fm;
3015 return kFALSE;
3016 }
b3e07543 3017 fm->OutputFile(outputFile);
7c2cd90a 3018 // Merge the outputs
3019 if (!fm->Merge()) {
3020 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
3021 delete fm;
3022 return kFALSE;
3023 } else {
b3e07543 3024 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 3025 }
7c2cd90a 3026 delete fm;
3027 return kTRUE;
0f389141 3028}
3029
c57f56b7 3030//______________________________________________________________________________
3031Bool_t AliAnalysisAlien::MergeOutputs()
3032{
3033// Merge analysis outputs existing in the AliEn space.
3034 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
3035 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3036 if (!Connect()) {
3037 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
3038 return kFALSE;
0f389141 3039 }
c9e8f7fd 3040 if (fMergeViaJDL) {
3041 if (!TestBit(AliAnalysisGrid::kMerge)) {
3042 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
3043 return kFALSE;
3044 }
3045 if (fProductionMode) {
3046 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
3047 return kFALSE;
3048 }
0f389141 3049 Info("MergeOutputs", "Submitting merging JDL");
a03be957 3050 if (!SubmitMerging()) return kFALSE;
0f389141 3051 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 3052 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 3053 return kFALSE;
c57f56b7 3054 }
3055 // Get the output path
dc030a24 3056 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 3057 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 3058 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
3059 return kFALSE;
3060 }
3061 if (!fOutputFiles.Length()) {
3062 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3063 return kFALSE;
0f389141 3064 }
3065 // Check if fast read option was requested
e1c22e21 3066 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
3067 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 3068 if (fFastReadOption) {
d3339be3 3069 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
3070 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 3071 gEnv->SetValue("XNet.ConnectTimeout",50);
3072 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 3073 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 3074 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 3075 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 3076 }
e8b839ab 3077 // Make sure we change the temporary directory
3078 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 3079 // Set temporary compilation directory to current one
3080 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 3081 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3082 TIter next(list);
3083 TObjString *str;
a2f5fc01 3084 TString outputFile;
c57f56b7 3085 Bool_t merged = kTRUE;
3086 while((str=(TObjString*)next())) {
a2f5fc01 3087 outputFile = str->GetString();
3088 Int_t index = outputFile.Index("@");
3089 if (index > 0) outputFile.Remove(index);
3090 TString outputChunk = outputFile;
3091 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 3092 // Skip already merged outputs
a2f5fc01 3093 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 3094 if (fOverwriteMode) {
a2f5fc01 3095 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
3096 gSystem->Unlink(outputFile);
3097 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3098 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3099 outputChunk.Data());
3100 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3101 }
3102 } else {
a2f5fc01 3103 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 3104 continue;
3105 }
3106 } else {
a2f5fc01 3107 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3108 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3109 outputChunk.Data());
3110 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3111 }
3112 }
37d21c01 3113 if (fMergeExcludes.Contains(outputFile.Data()) ||
3114 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 3115 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 3116 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 3117 if (!merged) {
3118 Error("MergeOutputs", "Terminate() will NOT be executed");
09d5920f 3119 delete list;
3120 return kFALSE;
ff07ec61 3121 }
3122 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
3123 if (fileOpened) fileOpened->Close();
c57f56b7 3124 }
09d5920f 3125 delete list;
0f389141 3126 return kTRUE;
c57f56b7 3127}
3128
bb885a9e 3129//______________________________________________________________________________
3130void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
3131{
3132// Use the output files connected to output containers from the analysis manager
3133// rather than the files defined by SetOutputFiles
3134 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 3135 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 3136 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
3137}
3138
149d288c 3139//______________________________________________________________________________
3140void AliAnalysisAlien::SetOutputFiles(const char *list)
3141{
3142// Manually set the output files list.
3143// Removes duplicates. Not allowed if default outputs are not disabled.
3144 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3145 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
3146 return;
3147 }
3148 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
3149 fOutputFiles = "";
3150 TString slist = list;
3151 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
3152 TObjArray *arr = slist.Tokenize(" ");
3153 TObjString *os;
3154 TIter next(arr);
3155 TString sout;
3156 while ((os=(TObjString*)next())) {
3157 sout = os->GetString();
3158 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
3159 if (fOutputFiles.Contains(sout)) continue;
3160 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
3161 fOutputFiles += sout;
3162 }
3163 delete arr;
f790bc1b 3164}
149d288c 3165
3166//______________________________________________________________________________
3167void AliAnalysisAlien::SetOutputArchive(const char *list)
3168{
3169// Manually set the output archive list. Free text - you are on your own...
3170// Not allowed if default outputs are not disabled.
3171 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3172 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
3173 return;
3174 }
3175 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
3176 fOutputArchive = list;
3177}
3178
3179//______________________________________________________________________________
3180void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
3181{
3182// Setting a prefered output SE is not allowed anymore.
3183 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
3184}
3185
28ce6c9c 3186//______________________________________________________________________________
3187void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
3188{
3189// Set some PROOF special parameter.
3190 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3191 if (pair) {
3192 TObject *old = pair->Key();
3193 TObject *val = pair->Value();
3194 fProofParam.Remove(old);
3195 delete old;
3196 delete val;
3197 }
3198 fProofParam.Add(new TObjString(pname), new TObjString(value));
3199}
3200
3201//______________________________________________________________________________
3202const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
3203{
3204// Returns a special PROOF parameter.
3205 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3206 if (!pair) return 0;
3207 return pair->Value()->GetName();
3208}
3209
5513444a 3210//______________________________________________________________________________
3211Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 3212{
3213// Start remote grid analysis.
3bdcb562 3214 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3215 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
3216 if (!mgr || !mgr->IsInitialized()) {
3217 Error("StartAnalysis", "You need an initialized analysis manager for this");
3218 return kFALSE;
3219 }
3220 // Are we in PROOF mode ?
3221 if (mgr->IsProofMode()) {
baa34407 3222 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3223 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3224 if (fProofCluster.IsNull()) {
3225 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3226 return kFALSE;
3227 }
3228 if (fProofDataSet.IsNull() && !testMode) {
3229 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3230 return kFALSE;
3231 }
3232 // Set the needed environment
3233 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3234 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3235 if (fProofReset && !testMode) {
3236 if (fProofReset==1) {
3237 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3238 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3239 } else {
3240 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3241 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3242 }
3243 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3244 return kFALSE;
3245 }
baa34407 3246
3247 if (!testMode) {
3248 // Check if there is an old active session
3249 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3250 if (nsessions) {
3251 Error("StartAnalysis","You have to reset your old session first\n");
3252 return kFALSE;
3253 }
53911ca6 3254 }
3bdcb562 3255 // Do we need to change the ROOT version ? The success of this cannot be checked.
874de7ea 3256 if (!fROOTVersion.IsNull() && !testMode) {
cbccef47 3257 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"VO_ALICE@ROOT::%s\");",
874de7ea 3258 fProofCluster.Data(), fROOTVersion.Data()));
3bdcb562 3259 }
3260 // Connect to PROOF and check the status
3261 Long_t proof = 0;
27734f0e 3262 TString sworkers;
3263 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3264 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3265 if (!testMode) {
27734f0e 3266 if (!sworkers.IsNull())
3267 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3268 else
3269 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3270 } else {
3271 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3272 if (!proof) {
3273 Error("StartAnalysis", "Could not start PROOF in test mode");
3274 return kFALSE;
3275 }
3276 }
3277 if (!proof) {
3278 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3279 return kFALSE;
3280 }
27734f0e 3281 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3282 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3283 // Set proof special parameters if any
3284 TIter nextpp(&fProofParam);
3285 TObject *proofparam;
3286 while ((proofparam=nextpp())) {
3287 TString svalue = GetProofParameter(proofparam->GetName());
3288 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3289 }
3bdcb562 3290 // Is dataset existing ?
3291 if (!testMode) {
3292 TString dataset = fProofDataSet;
3293 Int_t index = dataset.Index("#");
3294 if (index>=0) dataset.Remove(index);
830acc4c 3295// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3296// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3297// return kFALSE;
3298// }
3299// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3300 }
3301 // Is ClearPackages() needed ?
3302 if (TestSpecialBit(kClearPackages)) {
3303 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3304 gROOT->ProcessLine("gProof->ClearPackages();");
3305 }
3306 // Is a given aliroot mode requested ?
3307 TList optionsList;
1ac846e9 3308 TString parLibs;
830acc4c 3309 if (!fAliRootMode.IsNull()) {
3bdcb562 3310 TString alirootMode = fAliRootMode;
3311 if (alirootMode == "default") alirootMode = "";
3312 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3313 optionsList.SetOwner();
3314 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3315 // Check the additional libs to be loaded
3316 TString extraLibs;
1ac846e9 3317 Bool_t parMode = kFALSE;
dc1ffbcb 3318 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3319 // Parse the extra libs for .so
3320 if (fAdditionalLibs.Length()) {
a41b2882 3321 TString additionalLibs = fAdditionalLibs;
3322 additionalLibs.Strip();
3323 if (additionalLibs.Length() && fFriendLibs.Length())
3324 additionalLibs += " ";
3325 additionalLibs += fFriendLibs;
3326 TObjArray *list = additionalLibs.Tokenize(" ");
3bdcb562 3327 TIter next(list);
3328 TObjString *str;
1ac846e9 3329 while((str=(TObjString*)next())) {
3330 if (str->GetString().Contains(".so")) {
3331 if (parMode) {
3332 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3333 break;
3334 }
3335 TString stmp = str->GetName();
3336 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3337 stmp.ReplaceAll(".so","");
3338 if (!extraLibs.IsNull()) extraLibs += ":";
3339 extraLibs += stmp;
3340 continue;
3341 }
3342 if (str->GetString().Contains(".par")) {
3343 // The first par file found in the list will not allow any further .so
3344 parMode = kTRUE;
3345 if (!parLibs.IsNull()) parLibs += ":";
3346 parLibs += str->GetName();
3347 continue;
3348 }
3bdcb562 3349 }
3350 if (list) delete list;
3351 }
a41b2882 3352 if (!extraLibs.IsNull()) {
3353 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3354 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3355 }
830acc4c 3356 // Check extra includes
3357 if (!fIncludePath.IsNull()) {
3358 TString includePath = fIncludePath;
3359 includePath.ReplaceAll(" ",":");
dc030a24 3360 includePath.ReplaceAll("$ALICE_ROOT/","");
3361 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3362 includePath.ReplaceAll("-I","");
dc030a24 3363 includePath.Remove(TString::kTrailing, ':');
830acc4c 3364 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3365 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3366 }
27734f0e 3367 // Check if connection to grid is requested
3368 if (TestSpecialBit(kProofConnectGrid))
3369 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3370 // Enable AliRoot par
3371 if (testMode) {
3372 // Enable proof lite package
3373 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3374 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3375 TNamed *obj = (TNamed*)optionsList.At(i);
3376 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3377 }
3378 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3379 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3380 Info("StartAnalysis", "AliRootProofLite enabled");
3381 } else {
3382 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3383 return kFALSE;
3384 }
3385 } else {
dc030a24 3386 if ( ! fAliROOTVersion.IsNull() ) {
3387 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3388 fAliROOTVersion.Data(), &optionsList))) {
3389 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3390 return kFALSE;
3391 }
3392 }
27734f0e 3393 }
1ac846e9 3394 // Enable first par files from fAdditionalLibs
3395 if (!parLibs.IsNull()) {
3396 TObjArray *list = parLibs.Tokenize(":");
3397 TIter next(list);
3398 TObjString *package;
3399 while((package=(TObjString*)next())) {
40501bb7 3400 TString spkg = package->GetName();
3401 spkg.ReplaceAll(".par", "");
3402 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3403 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3404 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3405 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3406 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3407 return kFALSE;
3408 }
3409 } else {
3410 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3411 return kFALSE;
3412 }
3413 }
3414 if (list) delete list;
3415 }
3bdcb562 3416 } else {
3417 if (fAdditionalLibs.Contains(".so") && !testMode) {
3418 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3419 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3420 return kFALSE;
3421 }
3422 }
3423 // Enable par files if requested
3424 if (fPackages && fPackages->GetEntries()) {
3425 TIter next(fPackages);
3426 TObject *package;
3427 while ((package=next())) {
1ac846e9 3428 // Skip packages already enabled
3429 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3430 TString spkg = package->GetName();
3431 spkg.ReplaceAll(".par", "");
3432 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3433 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3434 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3435 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3436 return kFALSE;
3437 }
3438 } else {
3439 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3440 return kFALSE;
3441 }
3442 }
3443 }
3444 // Do we need to load analysis source files ?
3445 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3446 if (fAnalysisSource.Length()) {
3447 TObjArray *list = fAnalysisSource.Tokenize(" ");
3448 TIter next(list);
3449 TObjString *str;
3450 while((str=(TObjString*)next())) {
3451 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3452 }
3bdcb562 3453 if (list) delete list;
3454 }
830acc4c 3455 if (testMode) {
3456 // Register dataset to proof lite.
3457 if (fFileForTestMode.IsNull()) {
3458 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3459 return kFALSE;
3460 }
3461 if (gSystem->AccessPathName(fFileForTestMode)) {
3462 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3463 return kFALSE;
3464 }
3465 TFileCollection *coll = new TFileCollection();
3466 coll->AddFromFile(fFileForTestMode);
8ddad121 3467 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3468 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3469 }
3bdcb562 3470 return kTRUE;
3471 }
c57f56b7 3472
43da816a 3473 // Check if output files have to be taken from the analysis manager
3474 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3475 // Add output files and AOD files
3476 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3477 // Add extra files registered to the analysis manager
f790bc1b 3478 TString extra = GetListOfFiles("ext");
3479 if (!extra.IsNull()) {
149d288c 3480 extra.ReplaceAll(".root", "*.root");
f790bc1b 3481 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3482 fOutputFiles += extra;
84fcd93f 3483 }
149d288c 3484 // Compose the output archive.
b83f84f0 3485 fOutputArchive = "log_archive.zip:std*@disk=1 ";
ec5acdce 3486 if (mgr->IsCollectThroughput())
95e16f01 3487 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",fOutputFiles.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 3488 else
3489 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3490 }
f7b1cbc2 3491// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3492 if (TestBit(AliAnalysisGrid::kOffline)) {
3493 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3494 \n there nor any job run. You can revise the JDL and analysis \
3495 \n macro then run the same in \"submit\" mode.");
3496 } else if (TestBit(AliAnalysisGrid::kTest)) {
3497 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3498 \n dataset.");
3499 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3500 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3501 \n space and job submitted.");
3502 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3503 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3504 if (fMergeViaJDL) CheckInputData();
5513444a 3505 return kTRUE;
c57f56b7 3506 } else {
3507 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3508 }
3509
348be253 3510 Print();
c57f56b7 3511 if (!Connect()) {
3512 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3513 return kFALSE;
16a4353c 3514 }
d3b18c4c 3515 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3516 if (!CheckInputData()) {
3517 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3518 return kFALSE;
c57f56b7 3519 }
d3339be3 3520 if (!CreateDataset(fDataPattern)) {
3521 TString serror;
3522 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3523 if (fRunNumbers.Length()) serror = "run numbers";
3524 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3525 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3526 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3527 return kFALSE;
3528 }
21ca8e59 3529 WriteAnalysisFile();
c57f56b7 3530 WriteAnalysisMacro();
3531 WriteExecutable();
3532 WriteValidationScript();
0f389141 3533 if (fMergeViaJDL) {
3534 WriteMergingMacro();
3535 WriteMergeExecutable();
3536 WriteValidationScript(kTRUE);
3537 }
5513444a 3538 if (!CreateJDL()) return kFALSE;
3539 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3540 if (testMode) {
c57f56b7 3541 // Locally testing the analysis
3542 Info("StartAnalysis", "\n_______________________________________________________________________ \
3543 \n Running analysis script in a daughter shell as on a worker node \
3544 \n_______________________________________________________________________");
149d288c 3545 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3546 TIter next(list);
3547 TObjString *str;
a2f5fc01 3548 TString outputFile;
c57f56b7 3549 while((str=(TObjString*)next())) {
a2f5fc01 3550 outputFile = str->GetString();
3551 Int_t index = outputFile.Index("@");
3552 if (index > 0) outputFile.Remove(index);
3553 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3554 }
3555 delete list;
3556 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3557 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3558// gSystem->Exec("cat stdout");
5513444a 3559 return kFALSE;
c57f56b7 3560 }
5513444a 3561 // Check if submitting is managed by LPM manager
a3e84053 3562 if (fProductionMode) {
7ac61826 3563 //TString prodfile = fJDLName;
3564 //prodfile.ReplaceAll(".jdl", ".prod");
3565 //WriteProductionFile(prodfile);
5513444a 3566 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3567 return kFALSE;
3568 }
a8739e8a 3569 // Submit AliEn job(s)
d2a409b2 3570 gGrid->Cd(fGridOutputDir);
a8739e8a 3571 TGridResult *res;
c57f56b7 3572 TString jobID = "";
0ad3ea1f 3573 fGridJobIDs = "";
3574 fGridStages = "";
d2a409b2 3575 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3576 // Submit a given xml or a set of runs
a8739e8a 3577 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3578 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3579 if (res) {
3580 const char *cjobId = res->GetKey(0,"jobId");
3581 if (!cjobId) {
a03be957 3582 gGrid->Stdout();
3583 gGrid->Stderr();
a8739e8a 3584 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3585 return kFALSE;
a8739e8a 3586 } else {
3587 Info("StartAnalysis", "\n_______________________________________________________________________ \
3588 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3589 \n_______________________________________________________________________",
3590 fJDLName.Data(), cjobId);
3591 jobID = cjobId;
0ad3ea1f 3592 if (jobID.Atoi()) {
3593 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3594 fGridJobIDs.Append(jobID);
3595 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3596 fGridStages.Append("full");
3597 }
a8739e8a 3598 }
3599 delete res;
a03be957 3600 } else {
3601 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3602 return kFALSE;
a8739e8a 3603 }
3604 } else {
d2a409b2 3605 // Submit for a range of enumeration of runs.
a03be957 3606 if (!Submit()) return kFALSE;
0ad3ea1f 3607 jobID = fGridJobIDs;
c57f56b7 3608 }
a8739e8a 3609
d88e4159 3610 if (fDropToShell) {
3611 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3612 \n You may exit at any time and terminate the job later using the option <terminate> \
3613 \n ##################################################################################", jobID.Data());
3614 gSystem->Exec("aliensh");
3615 } else {
3616 Info("StartAnalysis", "\n#### SUBMITTED JOB %s TO ALIEN QUEUE #### \
3617 \n Remember to terminate the job later using the option <terminate> \
3618 \n ##################################################################################", jobID.Data());
3619 }
5513444a 3620 return kTRUE;
c57f56b7 3621}
3622
f790bc1b 3623//______________________________________________________________________________
3624const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3625{
3626// Get a comma-separated list of output files of the requested type.
3627// Type can be (case unsensitive):
3628// aod - list of aod files (std, extensions and filters)
3629// out - list of output files connected to containers (but not aod's or extras)
3630// ext - list of extra files registered to the manager
3631// ter - list of files produced in terminate
3632 static TString files;
3633 files = "";
3634 TString stype = type;
3635 stype.ToLower();
3636 TString aodfiles, extra;
3637 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3638 if (!mgr) {
3639 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3640 return files.Data();
3641 }
3642 if (mgr->GetOutputEventHandler()) {
3643 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3644 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3645 if (!extraaod.IsNull()) {
3646 aodfiles += ",";
3647 aodfiles += extraaod;
3648 }
3649 }
3650 if (stype.Contains("aod")) {
3651 files = aodfiles;
3652 if (stype == "aod") return files.Data();
3653 }
3654 // Add output files that are not in the list of AOD files
3655 TString outputfiles = "";
3656 TIter next(mgr->GetOutputs());
3657 AliAnalysisDataContainer *output;
3658 const char *filename = 0;
3659 while ((output=(AliAnalysisDataContainer*)next())) {
3660 filename = output->GetFileName();
3661 if (!(strcmp(filename, "default"))) continue;
3662 if (outputfiles.Contains(filename)) continue;
3663 if (aodfiles.Contains(filename)) continue;
3664 if (!outputfiles.IsNull()) outputfiles += ",";
3665 outputfiles += filename;
3666 }
3667 if (stype.Contains("out")) {
3668 if (!files.IsNull()) files += ",";
3669 files += outputfiles;
3670 if (stype == "out") return files.Data();
3671 }
3672 // Add extra files registered to the analysis manager
3673 TString sextra;
3674 extra = mgr->GetExtraFiles();
3675 if (!extra.IsNull()) {
3676 extra.Strip();
3677 extra.ReplaceAll(" ", ",");
3678 TObjArray *fextra = extra.Tokenize(",");
3679 TIter nextx(fextra);
3680 TObject *obj;
3681 while ((obj=nextx())) {
3682 if (aodfiles.Contains(obj->GetName())) continue;
3683 if (outputfiles.Contains(obj->GetName())) continue;
3684 if (sextra.Contains(obj->GetName())) continue;
3685 if (!sextra.IsNull()) sextra += ",";
3686 sextra += obj->GetName();
3687 }
3688 delete fextra;
3689 if (stype.Contains("ext")) {
3690 if (!files.IsNull()) files += ",";
3691 files += sextra;
3692 }
3693 }
3694 if (stype == "ext") return files.Data();
3695 TString termfiles;
3696 if (!fTerminateFiles.IsNull()) {
3697 fTerminateFiles.Strip();
3698 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3699 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3700 TIter nextx(fextra);
3701 TObject *obj;
3702 while ((obj=nextx())) {
3703 if (aodfiles.Contains(obj->GetName())) continue;
3704 if (outputfiles.Contains(obj->GetName())) continue;
3705 if (termfiles.Contains(obj->GetName())) continue;
3706 if (sextra.Contains(obj->GetName())) continue;
3707 if (!termfiles.IsNull()) termfiles += ",";
3708 termfiles += obj->GetName();
3709 }
3710 delete fextra;
3711 }
3712 if (stype.Contains("ter")) {
3713 if (!files.IsNull() && !termfiles.IsNull()) {
3714 files += ",";
3715 files += termfiles;
3716 }
3717 }
3718 return files.Data();
3719}
3720
d2a409b2 3721//______________________________________________________________________________
a03be957 3722Bool_t AliAnalysisAlien::Submit()
d2a409b2 3723{
3724// Submit all master jobs.
3725 Int_t nmasterjobs = fInputFiles->GetEntries();
3726 Long_t tshoot = gSystem->Now();
a03be957 3727 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3728 while (fNsubmitted < nmasterjobs) {
3729 Long_t now = gSystem->Now();
3730 if ((now-tshoot)>30000) {
3731 tshoot = now;
a03be957 3732 if (!SubmitNext()) return kFALSE;
d2a409b2 3733 }
3734 }
a03be957 3735 return kTRUE;
d2a409b2 3736}
3737
0f389141 3738//______________________________________________________________________________
a03be957 3739Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3740{
3741// Submit all merging jobs.
dc030a24 3742 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3743 gGrid->Cd(fGridOutputDir);
3744 TString mergeJDLName = fExecutable;
3745 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3746 if (!fInputFiles) {
3747 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3748 return kFALSE;
3749 }
0f389141 3750 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3751 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3752 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3753 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3754 if (fOutputToRunNo) {
3755 // The output directory is the run number
3756 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3757 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3758 } else {
17b4e9a4 3759 if (!fRunNumbers.Length() && !fRunRange[0]) {
3760 // The output directory is the grid outdir
3761 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3762 runOutDir = fGridOutputDir;
3763 } else {
3764 // The output directory is the master number in 3 digits format
3765 printf("### Submitting merging job for master <%03d>\n", i);
3766 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3767 }
7c2cd90a 3768 }
3769 // Check now the number of merging stages.
2b222f7c 3770 TObjArray *list = fOutputFiles.Tokenize(",");
3771 TIter next(list);
3772 TObjString *str;
3773 TString outputFile;
3774 while((str=(TObjString*)next())) {
3775 outputFile = str->GetString();
3776 Int_t index = outputFile.Index("@");
3777 if (index > 0) outputFile.Remove(index);
37d21c01 3778 if (!fMergeExcludes.Contains(outputFile) &&
3779 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3780 }
3781 delete list;
b3e07543 3782 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3783 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3784 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3785 }
a03be957 3786 if (!ntosubmit) return kTRUE;
d88e4159 3787 if (fDropToShell) {
3788 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3789 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3790 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3791 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3792 \n ################################################################################################################");
3793 gSystem->Exec("aliensh");
3794 } else {
3795 Info("StartAnalysis", "\n #### STARTED MERGING JOBS FOR YOU #### \
3796 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL') \
3797 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3798 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3799 \n ################################################################################################################");
3800 }
a03be957 3801 return kTRUE;
0f389141 3802}
3803
d2a409b2 3804//______________________________________________________________________________
a03be957 3805Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3806{
08d5b699 3807// Submit next bunch of master jobs if the queue is free. The first master job is
3808// submitted right away, while the next will not be unless the previous was split.
3809// The plugin will not submit new master jobs if there are more that 500 jobs in
3810// waiting phase.
d2a409b2 3811 static Bool_t iscalled = kFALSE;
3812 static Int_t firstmaster = 0;
3813 static Int_t lastmaster = 0;
3814 static Int_t npermaster = 0;
a03be957 3815 if (iscalled) return kTRUE;
d2a409b2 3816 iscalled = kTRUE;
3817 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3818 Int_t ntosubmit = 0;
3819 TGridResult *res;
3820 TString jobID = "";
3bdcb562 3821 Int_t nmasterjobs = fInputFiles->GetEntries();
3822 if (!fNsubmitted) {
3823 ntosubmit = 1;
3824 if (!IsUseSubmitPolicy()) {
90d50a8c 3825 if (nmasterjobs>5)
3bdcb562 3826 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3827 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3828 ntosubmit = nmasterjobs;
3829 }
3830 } else {
d2a409b2 3831 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3832 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3833 // If last master not split, just return
a03be957 3834 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3835 // No more than 100 waiting jobs
08d5b699 3836 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3837 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3838 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3839 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3840 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3841 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3842 }
d2a409b2 3843 for (Int_t i=0; i<ntosubmit; i++) {
3844 // Submit for a range of enumeration of runs.
a03be957 3845 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3846 TString query;
cd11251e 3847 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3848 runOutDir.ReplaceAll(".xml", "");
3849 if (fOutputToRunNo)
3850 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3851 else
3852 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3853 printf("********* %s\n",query.Data());
d2a409b2 3854 res = gGrid->Command(query);
3855 if (res) {
98ca124f 3856 TString cjobId1 = res->GetKey(0,"jobId");
3857 if (!cjobId1.Length()) {
d2a409b2 3858 iscalled = kFALSE;
a03be957 3859 gGrid->Stdout();
3860 gGrid->Stderr();
3861 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3862 return kFALSE;
d2a409b2 3863 } else {
3864 Info("StartAnalysis", "\n_______________________________________________________________________ \
3865 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3866 \n_______________________________________________________________________",
98ca124f 3867 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
0ad3ea1f 3868 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3869 fGridJobIDs.Append(cjobId1);
3870 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3871 fGridStages.Append("full");
d2a409b2 3872 jobID += cjobId1;
3873 jobID += " ";
98ca124f 3874 lastmaster = cjobId1.Atoi();
d2a409b2 3875 if (!firstmaster) firstmaster = lastmaster;
3876 fNsubmitted++;
3877 }
3878 delete res;
a03be957 3879 } else {
3880 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3881 return kFALSE;
d2a409b2 3882 }
3883 }
3884 iscalled = kFALSE;
a03be957 3885 return kTRUE;
d2a409b2 3886}
3887
c57f56b7 3888//______________________________________________________________________________
3889void AliAnalysisAlien::WriteAnalysisFile()
3890{
f10e8481 3891// Write current analysis manager into the file <analysisFile>
3892 TString analysisFile = fExecutable;
3893 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3894 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3895 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3896 if (!mgr || !mgr->IsInitialized()) {
3897 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3898 return;
3899 }
3900 // Check analysis type
3901 TObject *handler;
3902 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3903 handler = (TObject*)mgr->GetInputEventHandler();
3904 if (handler) {
65eb22f8 3905 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3906 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3907 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3908 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3909 } else {
3910 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3911 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3912 }
c57f56b7 3913 }
3914 TDirectory *cdir = gDirectory;
f10e8481 3915 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3916 if (file) {
e16a394c 3917 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3918 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3919 // Unless merging makes no sense
3920 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3921 mgr->Write();
3922 delete file;
fe2d7fc2 3923 // Enable termination for local jobs
3924 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3925 }
3926 if (cdir) cdir->cd();
f10e8481 3927 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3928 }
3929 Bool_t copy = kTRUE;
d3b18c4c 3930 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3931 if (copy) {
3932 CdWork();
3933 TString workdir = gGrid->GetHomeDirectory();
3934 workdir += fGridWorkingDir;
d3b18c4c 3935 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3936 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3937 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3938 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3939 }
3940}
3941
3942//______________________________________________________________________________
3943void AliAnalysisAlien::WriteAnalysisMacro()
3944{
3945// Write the analysis macro that will steer the analysis in grid mode.
3946 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3947 ofstream out;
3948 out.open(fAnalysisMacro.Data(), ios::out);
3949 if (!out.good()) {
3950 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3951 return;
3952 }
5b273635 3953 Bool_t hasSTEERBase = kFALSE;
3954 Bool_t hasESD = kFALSE;
3955 Bool_t hasAOD = kFALSE;
3956 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3957 Bool_t hasOADB = kFALSE;
5b273635 3958 Bool_t hasANALYSISalice = kFALSE;
3959 Bool_t hasCORRFW = kFALSE;
c57f56b7 3960 TString func = fAnalysisMacro;
3961 TString type = "ESD";
3962 TString comment = "// Analysis using ";
c85cfc0f 3963 if (IsUseMCchain()) {
3964 type = "MC";
3965 comment += "MC";
3966 } else {
3967 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3968 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3969 type = "AOD";
3970 comment += "AOD";
3971 }
3972 }
0df6ccf2 3973 if (type!="AOD" && fFriendChainName!="") {
3974 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3975 return;
3976 }
c57f56b7 3977 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3978 else comment += " data";
3979 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3980 func.ReplaceAll(".C", "");
3981 out << "void " << func.Data() << "()" << endl;
3982 out << "{" << endl;
3983 out << comment.Data() << endl;
3984 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3985 out << " TStopwatch timer;" << endl;
3986 out << " timer.Start();" << endl << endl;
13f0c00b 3987 // Change temp directory to current one
fdee57f1 3988 if (!IsLocalTest()) {
3989 out << "// connect to AliEn and make the chain" << endl;
3990 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
3991 }
13f0c00b 3992 out << "// Set temporary merging directory to current one" << endl;
3993 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3994 out << "// Set temporary compilation directory to current one" << endl;
3995 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3996 // Reset existing include path
3997 out << "// Reset existing include path and add current directory first in the search" << endl;
3998 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3999 if (!fExecutableCommand.Contains("aliroot")) {
4000 out << "// load base root libraries" << endl;
4001 out << " gSystem->Load(\"libTree\");" << endl;
4002 out << " gSystem->Load(\"libGeom\");" << endl;
4003 out << " gSystem->Load(\"libVMC\");" << endl;
4004 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4005 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4006 }
d5c6455a 4007 if (fAdditionalRootLibs.Length()) {
47a4137d 4008 // in principle libtree /lib geom libvmc etc. can go into this list, too
4009 out << "// Add aditional libraries" << endl;
4010 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4011 TIter next(list);
4012 TObjString *str;
4013 while((str=(TObjString*)next())) {
4014 if (str->GetString().Contains(".so"))
4015 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 4016 }
47a4137d 4017 if (list) delete list;
d5c6455a 4018 }
57377eb5 4019 out << "// Load analysis framework libraries" << endl;
652561d5 4020 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 4021 if (!fPackages) {
f790bc1b 4022 if (!fExecutableCommand.Contains("aliroot")) {
4023 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4024 out << " gSystem->Load(\"libESD\");" << endl;
4025 out << " gSystem->Load(\"libAOD\");" << endl;
4026 }
4e5c5506 4027 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4028 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 4029 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4030 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 4031 } else {
4e5c5506 4032 TIter next(fPackages);
4033 TObject *obj;
57377eb5 4034 TString pkgname;
57377eb5 4035 while ((obj=next())) {
4036 pkgname = obj->GetName();
4478e6f1 4037 if (pkgname == "STEERBase" ||
4038 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4039 if (pkgname == "ESD" ||
4040 pkgname == "ESD.par") hasESD = kTRUE;
4041 if (pkgname == "AOD" ||
4042 pkgname == "AOD.par") hasAOD = kTRUE;
4043 if (pkgname == "ANALYSIS" ||
4044 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4045 if (pkgname == "OADB" ||
4046 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 4047 if (pkgname == "ANALYSISalice" ||
4048 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4049 if (pkgname == "CORRFW" ||
4050 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 4051 }
4052 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 4053 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4054 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 4055 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4056 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 4057 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4058 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 4059 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4060 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4061 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4062 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 4063 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4064 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 4065 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4066 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 4067 out << "// Compile other par packages" << endl;
4068 next.Reset();
fcc9bb6f 4069 while ((obj=next())) {
4070 pkgname = obj->GetName();
4478e6f1 4071 if (pkgname == "STEERBase" ||
4072 pkgname == "STEERBase.par" ||
4073 pkgname == "ESD" ||
4074 pkgname == "ESD.par" ||
4075 pkgname == "AOD" ||
4076 pkgname == "AOD.par" ||
4077 pkgname == "ANALYSIS" ||
4078 pkgname == "ANALYSIS.par" ||
7ea40ac8 4079 pkgname == "OADB" ||
4080 pkgname == "OADB.par" ||
4478e6f1 4081 pkgname == "ANALYSISalice" ||
4082 pkgname == "ANALYSISalice.par" ||
4083 pkgname == "CORRFW" ||
4084 pkgname == "CORRFW.par") continue;
5b273635 4085 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 4086 }
4e5c5506 4087 }
b3e07543 4088 out << "// include path" << endl;
4089 // Get the include path from the interpreter and remove entries pointing to AliRoot
4090 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4091 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4092 out << " TIter nextpath(listpaths);" << endl;
4093 out << " TObjString *pname;" << endl;
4094 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4095 out << " TString current = pname->GetName();" << endl;
4096 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4097 out << " gSystem->AddIncludePath(current);" << endl;
4098 out << " }" << endl;
4099 out << " if (listpaths) delete listpaths;" << endl;
4100 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4101 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4102 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 4103 if (fAdditionalLibs.Length()) {
4104 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4105 TString additionalLibs = fAdditionalLibs;
4106 additionalLibs.Strip();
4107 if (additionalLibs.Length() && fFriendLibs.Length())
4108 additionalLibs += " ";
4109 additionalLibs += fFriendLibs;
4110 TObjArray *list = additionalLibs.Tokenize(" ");
6da75e0b 4111 TIter next(list);
4112 TObjString *str;
4113 while((str=(TObjString*)next())) {
4114 if (str->GetString().Contains(".so"))
4115 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 4116 if (str->GetString().Contains(".par"))
4117 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 4118 }
4119 if (list) delete list;
4120 }
4121 out << endl;
c57f56b7 4122 out << "// analysis source to be compiled at runtime (if any)" << endl;
4123 if (fAnalysisSource.Length()) {
4124 TObjArray *list = fAnalysisSource.Tokenize(" ");
4125 TIter next(list);
4126 TObjString *str;
4127 while((str=(TObjString*)next())) {
4128 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4129 }
4130 if (list) delete list;
4131 }
4132 out << endl;
b3e07543 4133// out << " printf(\"Currently load libraries:\\n\");" << endl;
4134// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 4135 if (fFastReadOption) {
d3339be3 4136 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
4137 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 4138 out << "// fast xrootd reading enabled" << endl;
4139 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4140 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4141 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4142 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4143 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4144 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 4145 }
c57f56b7 4146 out << "// read the analysis manager from file" << endl;
f10e8481 4147 TString analysisFile = fExecutable;
4148 analysisFile.ReplaceAll(".sh", ".root");
105631b5 4149 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4150 << analysisFile << "\");" << endl;
4151 out << " if (!mgr) return;" << endl;
21ca8e59 4152 if (IsLocalTest()) {
4153 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
4154 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 4155 if (fFileForTestMode.IsNull())
4156 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
4157 else
4158 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 4159 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4b5d9e21 4160 if (!fFriendChainName.IsNull())
b9f1efa5 4161 out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\",\"" << fFriendLibs << "\");" << endl;
024dfa52 4162 if (IsUseMCchain())
4163 out << " plugin->SetUseMCchain();" << endl;
21ca8e59 4164 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 4165 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 4166 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 4167 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
4168 } else {
7a366e0b 4169 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 4170 out << " mgr->SetNSysInfo(100);" << endl;
4171 }
21ca8e59 4172 }
c57f56b7 4173 out << " mgr->PrintStatus();" << endl;
52b6a92b 4174 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 4175 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 4176 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 4177 } else {
9a8288ff 4178 if (TestBit(AliAnalysisGrid::kTest))
4179 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4180 else
4181 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 4182 }
4183 }
21ca8e59 4184 if (!IsLocalTest()) {
242accb2 4185 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 4186 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
4187 } else {
4188 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 4189 }
f7498086 4190 out << " timer.Stop();" << endl;
4191 out << " timer.Print();" << endl;
c57f56b7 4192 out << "}" << endl << endl;
21ca8e59 4193 if (!IsLocalTest()) {
fcc9bb6f 4194 out <<"//________________________________________________________________________________" << endl;
c57f56b7 4195 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
4196 out << "{" << endl;
4197 out << "// Create a chain using url's from xml file" << endl;
242accb2 4198 out << " TString filename;" << endl;
4199 out << " Int_t run = 0;" << endl;
c85cfc0f 4200 if (IsUseMCchain()) {
4201 out << " TString treename = \"TE\";" << endl;
4202 } else {
780ec36d 4203 if (!fTreeName.IsNull()) {
4204 out << " TString treename = \"" << fTreeName << "\";" << endl;
4205 } else {
4206 out << " TString treename = type;" << endl;
4207 out << " treename.ToLower();" << endl;
4208 out << " treename += \"Tree\";" << endl;
4209 }
c85cfc0f 4210 }
e02fee64 4211 out << " printf(\"***************************************\\n\");" << endl;
4212 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
4213 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 4214 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
4215 out << " if (!coll) {" << endl;
4216 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
4217 out << " return NULL;" << endl;
4218 out << " }" << endl;
242accb2 4219 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 4220 out << " TChain *chain = new TChain(treename);" << endl;
06f67114 4221 if(!fFriendChainName.IsNull()) {
4222 out << " TList *friends = new TList();" << endl;
4223 out << " TIter nextfriend(friends);" << endl;
4224 out << " TChain *cfriend = 0;" << endl;
4225 TObjArray *list = fFriendChainName.Tokenize(" ");
4226 TIter next(list);
4227 TObjString *str;
4228 while((str=(TObjString*)next())) {
4229 out << " cfriend = new TChain(treename, \"" << str->GetName() << "\");" << endl;
4230 out << " friends->Add(cfriend);" << endl;
4231 out << " chain->AddFriend(cfriend);" << endl;
4232 }
4233 delete list;
4234// out << " TChain *chainFriend = new TChain(treename);" << endl;
0df6ccf2 4235 }
c57f56b7 4236 out << " coll->Reset();" << endl;
0df6ccf2 4237 out << " while (coll->Next()) {" << endl;
242accb2 4238 out << " filename = coll->GetTURL("");" << endl;
4239 out << " if (mgr) {" << endl;
4240 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
4241 out << " if (nrun && nrun != run) {" << endl;
4242 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
4243 out << " mgr->SetRunFromPath(nrun);" << endl;
4244 out << " run = nrun;" << endl;
4245 out << " }" << endl;
4246 out << " }" << endl;
4247 out << " chain->Add(filename);" << endl;
06f67114 4248 if(!fFriendChainName.IsNull()) {
4249 out << " TString bpath=coll->GetTURL(\"\");" << endl;
4250 out << " if (bpath.Index(\"#\") > -1) bpath.Remove(bpath.Index(\"#\"));" << endl;
4251 out << " bpath = gSystem->DirName(bpath);" << endl;
4252 out << " bpath += \"/\";" << endl;
4253 out << " TString fileFriend;" << endl;
4254 out << " nextfriend.Reset();" << endl;
4255 out << " while ((cfriend=(TChain*)nextfriend())) {" << endl;
4256 out << " fileFriend = bpath;" << endl;
4257 out << " fileFriend += cfriend->GetTitle();" << endl;
4258 out << " TFile *file = TFile::Open(fileFriend);" << endl;
4259 out << " if (file) {" << endl;
4260 out << " file->Close();" << endl;
4261 out << " cfriend->Add(fileFriend.Data());" << endl;
4262 out << " } else {" << endl;
4263 out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
4264 out << " return 0;" << endl;
4265 out << " }" << endl;
4b5d9e21 4266 out << " }" << endl;
0df6ccf2 4267 }
4268 out << " }" << endl;
c57f56b7 4269 out << " if (!chain->GetNtrees()) {" << endl;
4270 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
4271 out << " return NULL;" << endl;
4272 out << " }" << endl;
4273 out << " return chain;" << endl;
fcc9bb6f 4274 out << "}" << endl << endl;
c57f56b7 4275 }
5b273635 4276 if (hasANALYSISalice) {
4277 out <<"//________________________________________________________________________________" << endl;
4278 out << "Bool_t SetupPar(const char *package) {" << endl;
4279 out << "// Compile the package and set it up." << endl;
4280 out << " TString pkgdir = package;" << endl;
4281 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4282 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4283 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4284 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4285 out << " // Check for BUILD.sh and execute" << endl;
4286 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4287 out << " printf(\"*******************************\\n\");" << endl;
4288 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4289 out << " printf(\"*******************************\\n\");" << endl;
4290 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4291 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4292 out << " gSystem->ChangeDirectory(cdir);" << endl;
4293 out << " return kFALSE;" << endl;
4294 out << " }" << endl;
4295 out << " } else {" << endl;
4296 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4297 out << " gSystem->ChangeDirectory(cdir);" << endl;
4298 out << " return kFALSE;" << endl;
4299 out << " }" << endl;
4300 out << " // Check for SETUP.C and execute" << endl;
4301 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4302 out << " printf(\"*******************************\\n\");" << endl;
4303 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4304 out << " printf(\"*******************************\\n\");" << endl;
4305 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4306 out << " } else {" << endl;
4307 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4308 out << " gSystem->ChangeDirectory(cdir);" << endl;
4309 out << " return kFALSE;" << endl;
4310 out << " }" << endl;
4311 out << " // Restore original workdir" << endl;
4312 out << " gSystem->ChangeDirectory(cdir);" << endl;
4313 out << " return kTRUE;" << endl;
4314 out << "}" << endl;
4315 }
c57f56b7 4316 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4317 }
4318 Bool_t copy = kTRUE;
d3b18c4c 4319 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4320 if (copy) {
4321 CdWork();
4322 TString workdir = gGrid->GetHomeDirectory();
4323 workdir += fGridWorkingDir;
4324 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4325 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4326// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4327 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4328 Form("alien://%s/%s", workdir.Data(),
4329 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4330 }
4331}
4332
0f389141 4333//______________________________________________________________________________
4334void AliAnalysisAlien::WriteMergingMacro()
4335{
4336// Write a macro to merge the outputs per master job.
4337 if (!fMergeViaJDL) return;
4338 if (!fOutputFiles.Length()) {
4339 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4340 return;
4341 }
ec5acdce 4342 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
0f389141 4343 TString mergingMacro = fExecutable;
4344 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4345 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4346 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4347 ofstream out;
4348 out.open(mergingMacro.Data(), ios::out);
4349 if (!out.good()) {
4350 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4351 return;
4352 }
5b273635 4353 Bool_t hasSTEERBase = kFALSE;
4354 Bool_t hasESD = kFALSE;
4355 Bool_t hasAOD = kFALSE;
4356 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4357 Bool_t hasOADB = kFALSE;
5b273635 4358 Bool_t hasANALYSISalice = kFALSE;
4359 Bool_t hasCORRFW = kFALSE;
0f389141 4360 TString func = mergingMacro;
4361 TString comment;
4362 func.ReplaceAll(".C", "");
f47d5cb4 4363 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4364 out << "{" << endl;
4365 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4366 out << " TStopwatch timer;" << endl;
4367 out << " timer.Start();" << endl << endl;
b3e07543 4368 // Reset existing include path
4369 out << "// Reset existing include path and add current directory first in the search" << endl;
4370 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4371 if (!fExecutableCommand.Contains("aliroot")) {
4372 out << "// load base root libraries" << endl;
4373 out << " gSystem->Load(\"libTree\");" << endl;
4374 out << " gSystem->Load(\"libGeom\");" << endl;
4375 out << " gSystem->Load(\"libVMC\");" << endl;
4376 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4377 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4378 }
0f389141 4379 if (fAdditionalRootLibs.Length()) {
4380 // in principle libtree /lib geom libvmc etc. can go into this list, too
4381 out << "// Add aditional libraries" << endl;
4382 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4383 TIter next(list);
4384 TObjString *str;
4385 while((str=(TObjString*)next())) {
4386 if (str->GetString().Contains(".so"))
4387 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4388 }
4389 if (list) delete list;
4390 }
0f389141 4391 out << "// Load analysis framework libraries" << endl;
4392 if (!fPackages) {
7c2cd90a 4393 if (!fExecutableCommand.Contains("aliroot")) {
4394 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4395 out << " gSystem->Load(\"libESD\");" << endl;
4396 out << " gSystem->Load(\"libAOD\");" << endl;
4397 }
0f389141 4398 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4399 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4400 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4401 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4402 } else {
4403 TIter next(fPackages);
4404 TObject *obj;
4405 TString pkgname;
5b273635 4406 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4407 while ((obj=next())) {
4408 pkgname = obj->GetName();
4409 if (pkgname == "STEERBase" ||
4410 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4411 if (pkgname == "ESD" ||
4412 pkgname == "ESD.par") hasESD = kTRUE;
4413 if (pkgname == "AOD" ||
4414 pkgname == "AOD.par") hasAOD = kTRUE;
4415 if (pkgname == "ANALYSIS" ||
4416 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4417 if (pkgname == "OADB" ||
4418 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4419 if (pkgname == "ANALYSISalice" ||
4420 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4421 if (pkgname == "CORRFW" ||
4422 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4423 }
5b273635 4424 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4425 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4426 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4427 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4428 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4429 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4430 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4431 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4432 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4433 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4434 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4435 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4436 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4437 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4438 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4439 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4440 out << "// Compile other par packages" << endl;
4441 next.Reset();
4442 while ((obj=next())) {
4443 pkgname = obj->GetName();
4444 if (pkgname == "STEERBase" ||
4445 pkgname == "STEERBase.par" ||
4446 pkgname == "ESD" ||
4447 pkgname == "ESD.par" ||
4448 pkgname == "AOD" ||
4449 pkgname == "AOD.par" ||
4450 pkgname == "ANALYSIS" ||
4451 pkgname == "ANALYSIS.par" ||
7ea40ac8 4452 pkgname == "OADB" ||
4453 pkgname == "OADB.par" ||
0f389141 4454 pkgname == "ANALYSISalice" ||
4455 pkgname == "ANALYSISalice.par" ||
4456 pkgname == "CORRFW" ||
4457 pkgname == "CORRFW.par") continue;
5b273635 4458 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4459 }
4460 }
b3e07543 4461 out << "// include path" << endl;
4462 // Get the include path from the interpreter and remove entries pointing to AliRoot
4463 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4464 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4465 out << " TIter nextpath(listpaths);" << endl;
4466 out << " TObjString *pname;" << endl;
4467 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4468 out << " TString current = pname->GetName();" << endl;
4469 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4470 out << " gSystem->AddIncludePath(current);" << endl;
4471 out << " }" << endl;
4472 out << " if (listpaths) delete listpaths;" << endl;
4473 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4474 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4475 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4476 if (fAdditionalLibs.Length()) {
4477 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4478 TString additionalLibs = fAdditionalLibs;
4479 additionalLibs.Strip();
4480 if (additionalLibs.Length() && fFriendLibs.Length())
4481 additionalLibs += " ";
4482 additionalLibs += fFriendLibs;
4483 TObjArray *list = additionalLibs.Tokenize(" ");
0f389141 4484 TIter next(list);
4485 TObjString *str;
4486 while((str=(TObjString*)next())) {
4487 if (str->GetString().Contains(".so"))
4488 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4489 }
4490 if (list) delete list;
4491 }
4492 out << endl;
4493 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4494 if (fAnalysisSource.Length()) {
4495 TObjArray *list = fAnalysisSource.Tokenize(" ");
4496 TIter next(list);
4497 TObjString *str;
4498 while((str=(TObjString*)next())) {
4499 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4500 }
4501 if (list) delete list;
4502 }
149d288c 4503 out << endl;
4504
0f389141 4505 if (fFastReadOption) {
4506 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4507 out << "// fast xrootd reading enabled" << endl;
4508 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4509 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4510 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4511 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4512 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4513 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4514 }
4515 // Change temp directory to current one
fdee57f1 4516 out << "// Connect to AliEn" << endl;
4517 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
e8b839ab 4518 out << "// Set temporary merging directory to current one" << endl;
4519 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4520 out << "// Set temporary compilation directory to current one" << endl;
4521 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
7c2cd90a 4522 out << " TString outputDir = dir;" << endl;
f790bc1b 4523 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4524 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4525 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4526 out << " TIter *iter = new TIter(list);" << endl;
4527 out << " TObjString *str;" << endl;
a2f5fc01 4528 out << " TString outputFile;" << endl;
0f389141 4529 out << " Bool_t merged = kTRUE;" << endl;
ec5acdce 4530 TString analysisFile = fExecutable;
4531 analysisFile.ReplaceAll(".sh", ".root");
4532 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4533 << analysisFile << "\");" << endl;
4534 out << " if (!mgr) {" << endl;
4535 out << " printf(\"ERROR: Analysis manager could not be extracted from file \");" << endl;
4536 out << " return;" << endl;
4537 out << " }" << endl;
d66aa2a9 4538 if (IsLocalTest()) {
4539 out << " printf(\"===================================\n\");" << endl;
4540 out << " printf(\"Testing merging...\\n\");" << endl;
4541 out << " printf(\"===================================\n\");" << endl;
4542 }
0f389141 4543 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4544 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4545 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4546 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4547 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4548 out << " // Skip already merged outputs" << endl;
a2f5fc01 4549 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
d66aa2a9 4550 out << " printf(\"Output file <%s> found. Not merging again.\\n\",outputFile.Data());" << endl;
0f389141 4551 out << " continue;" << endl;
4552 out << " }" << endl;
a2f5fc01 4553 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4554 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4555 out << " if (!merged) {" << endl;
a2f5fc01 4556 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4557 out << " return;" << endl;
0f389141 4558 out << " }" << endl;
4559 out << " }" << endl;
d66aa2a9 4560 if (mgr && mgr->IsCollectThroughput() && !IsLocalTest()) {
ec5acdce 4561 out << " TString infolog = \"" << mgr->GetFileInfoLog() << "\";" << endl;
4562 out << " AliAnalysisAlien::MergeInfo(infolog, outputDir);" << endl;
4563 }
7c2cd90a 4564 out << " // all outputs merged, validate" << endl;
4565 out << " ofstream out;" << endl;
4566 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4567 out << " out.close();" << endl;
4568 out << " // read the analysis manager from file" << endl;
d66aa2a9 4569 if (IsLocalTest()) {
4570 out << " printf(\"===================================\n\");" << endl;
4571 out << " printf(\"Testing Terminate()...\\n\");" << endl;
4572 out << " printf(\"===================================\n\");" << endl;
4573 } else {
4574 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
4575 }
5c4250fc 4576 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4577 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4578 out << " mgr->PrintStatus();" << endl;
ec5acdce 4579 if (mgr) {
4580 if (mgr->GetDebugLevel()>3) {
f866cba5 4581 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4582 } else {
9a8288ff 4583 if (TestBit(AliAnalysisGrid::kTest))
4584 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4585 else
4586 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4587 }
4588 }
9a8288ff 4589 out << " TTree *tree = NULL;" << endl;
4590 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4591 out << "}" << endl << endl;
5b273635 4592 if (hasANALYSISalice) {
4593 out <<"//________________________________________________________________________________" << endl;
4594 out << "Bool_t SetupPar(const char *package) {" << endl;
4595 out << "// Compile the package and set it up." << endl;
4596 out << " TString pkgdir = package;" << endl;
4597 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4598 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4599 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4600 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4601 out << " // Check for BUILD.sh and execute" << endl;
4602 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4603 out << " printf(\"*******************************\\n\");" << endl;
4604 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4605 out << " printf(\"*******************************\\n\");" << endl;
4606 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4607 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4608 out << " gSystem->ChangeDirectory(cdir);" << endl;
4609 out << " return kFALSE;" << endl;
4610 out << " }" << endl;
4611 out << " } else {" << endl;
4612 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4613 out << " gSystem->ChangeDirectory(cdir);" << endl;
4614 out << " return kFALSE;" << endl;
4615 out << " }" << endl;
4616 out << " // Check for SETUP.C and execute" << endl;
4617 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4618 out << " printf(\"*******************************\\n\");" << endl;
4619 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4620 out << " printf(\"*******************************\\n\");" << endl;
4621 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4622 out << " } else {" << endl;
4623 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4624 out << " gSystem->ChangeDirectory(cdir);" << endl;
4625 out << " return kFALSE;" << endl;
4626 out << " }" << endl;
4627 out << " // Restore original workdir" << endl;
4628 out << " gSystem->ChangeDirectory(cdir);" << endl;
4629 out << " return kTRUE;" << endl;
4630 out << "}" << endl;
4631 }
0f389141 4632 }
4633 Bool_t copy = kTRUE;
d3b18c4c 4634 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4635 if (copy) {
4636 CdWork();
4637 TString workdir = gGrid->GetHomeDirectory();
4638 workdir += fGridWorkingDir;
4639 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4640 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4641// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4642 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4643 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4644 }
4645}
4646
4647//______________________________________________________________________________
4648Bool_t AliAnalysisAlien::SetupPar(const char *package)
4649{
205b201f 4650// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4651// Note that for loading the compiled library. The current directory should have precedence in
4652// LD_LIBRARY_PATH
4653 TString pkgdir = package;
4654 pkgdir.ReplaceAll(".par","");
b50a41c3 4655 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4656 TString cdir = gSystem->WorkingDirectory();
4657 gSystem->ChangeDirectory(pkgdir);
4658 // Check for BUILD.sh and execute
4659 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4660 printf("**************************************************\n");
4661 printf("*** Building PAR archive %s\n", package);
4662 printf("**************************************************\n");
4663 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4664 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4665 gSystem->ChangeDirectory(cdir);
4666 return kFALSE;
4667 }
4668 } else {
4669 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4670 gSystem->ChangeDirectory(cdir);
4671 return kFALSE;
4672 }
4673 // Check for SETUP.C and execute
4674 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4675 printf("**************************************************\n");
4676 printf("*** Setup PAR archive %s\n", package);
4677 printf("**************************************************\n");
4678 gROOT->Macro("PROOF-INF/SETUP.C");
4679 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4680 } else {
4681 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4682 gSystem->ChangeDirectory(cdir);
4683 return kFALSE;
4684 }
4685 // Restore original workdir
4686 gSystem->ChangeDirectory(cdir);
4687 return kTRUE;
4688}
4689
c57f56b7 4690//______________________________________________________________________________
4691void AliAnalysisAlien::WriteExecutable()
4692{
4693// Generate the alien executable script.
5aa94dc3 4694 // Patch executable with -x to catch error code
4695 if (fExecutableCommand.Contains("root") &&
4696 fExecutableCommand.Contains("-q") &&
4697 !fExecutableCommand.Contains("-x")) fExecutableCommand += " -x";
c57f56b7 4698 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4699 ofstream out;
4700 out.open(fExecutable.Data(), ios::out);
4701 if (out.bad()) {
5513444a 4702 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4703 return;
4704 }
4705 out << "#!/bin/bash" << endl;
2a4d5166 4706 // Make sure we can properly compile par files
4707 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4708 out << "echo \"=========================================\"" << endl;
4709 out << "echo \"############## PATH : ##############\"" << endl;
4710 out << "echo $PATH" << endl;
4711 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4712 out << "echo $LD_LIBRARY_PATH" << endl;
4713 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4714 out << "echo $ROOTSYS" << endl;
4715 out << "echo \"############## which root : ##############\"" << endl;
4716 out << "which root" << endl;
4717 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4718 out << "echo $ALICE_ROOT" << endl;
4719 out << "echo \"############## which aliroot : ##############\"" << endl;
4720 out << "which aliroot" << endl;
9c5ddadc 4721 out << "echo \"############## system limits : ##############\"" << endl;
4722 out << "ulimit -a" << endl;
4723 out << "echo \"############## memory : ##############\"" << endl;
4724 out << "free -m" << endl;
c57f56b7 4725 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4726 out << fExecutableCommand << " ";
370c9880 4727 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4728 out << "RET=$?" << endl;
4729 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4730 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4731 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4732 out << " let sig=\"$RET - 128\""<<endl;
4733 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4734 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4735 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4736 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4737 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4738 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4739 out << " fi"<<endl;
4740 out << " exit $RET"<< endl;
4741 out << "fi" << endl << endl ;
4742 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4743 out << "echo \"############## memory after: ##############\"" << endl;
4744 out << "free -m" << endl;
c57f56b7 4745 }
4746 Bool_t copy = kTRUE;
d3b18c4c 4747 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4748 if (copy) {
4749 CdWork();
4750 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4751 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4752 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4753 workdir += fGridWorkingDir;
4754 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4755 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4756 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4757// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4758 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4759 executable.Data())) Fatal("","Terminating");
c57f56b7 4760 }
4761}
4762
0f389141 4763//______________________________________________________________________________
4764void AliAnalysisAlien::WriteMergeExecutable()
4765{
4766// Generate the alien executable script for the merging job.
4767 if (!fMergeViaJDL) return;
4768 TString mergeExec = fExecutable;
4769 mergeExec.ReplaceAll(".sh", "_merge.sh");
4770 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4771 ofstream out;
4772 out.open(mergeExec.Data(), ios::out);
4773 if (out.bad()) {
4774 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4775 return;
4776 }
4777 out << "#!/bin/bash" << endl;
2a4d5166 4778 // Make sure we can properly compile par files
4779 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4780 out << "echo \"=========================================\"" << endl;
4781 out << "echo \"############## PATH : ##############\"" << endl;
4782 out << "echo $PATH" << endl;
4783 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4784 out << "echo $LD_LIBRARY_PATH" << endl;
4785 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4786 out << "echo $ROOTSYS" << endl;
4787 out << "echo \"############## which root : ##############\"" << endl;
4788 out << "which root" << endl;
4789 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4790 out << "echo $ALICE_ROOT" << endl;
4791 out << "echo \"############## which aliroot : ##############\"" << endl;
4792 out << "which aliroot" << endl;
4793 out << "echo \"############## system limits : ##############\"" << endl;
4794 out << "ulimit -a" << endl;
4795 out << "echo \"############## memory : ##############\"" << endl;
4796 out << "free -m" << endl;
4797 out << "echo \"=========================================\"" << endl << endl;
0f389141 4798 TString mergeMacro = fExecutable;
4799 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4800 if (IsOneStageMerging())
4801 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4802 else
f47d5cb4 4803 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4804 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4805 out << "RET=$?" << endl;
4806 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4807 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4808 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4809 out << " let sig=\"$RET - 128\""<<endl;
4810 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4811 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4812 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4813 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4814 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4815 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4816 out << " fi"<<endl;
4817 out << " exit $RET"<< endl;
4818 out << "fi" << endl << endl ;
0f389141 4819 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4820 out << "echo \"############## memory after: ##############\"" << endl;
4821 out << "free -m" << endl;
0f389141 4822 }
4823 Bool_t copy = kTRUE;
d3b18c4c 4824 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4825 if (copy) {
4826 CdWork();
4827 TString workdir = gGrid->GetHomeDirectory();
4828 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4829 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4830 workdir += fGridWorkingDir;
4831 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4832 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4833 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4834// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4835 if (!copyLocal2Alien("WriteMergeExecutable",
4836 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4837 }
4838}
4839
c57f56b7 4840//______________________________________________________________________________
5513444a 4841void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4842{
4843// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4844// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4845// Next lines: full_path_to_dataset XXX (XXX is a string)
4846// To submit, one has to: submit jdl XXX for all lines
4847 ofstream out;
4848 out.open(filename, ios::out);
4849 if (out.bad()) {
4850 Error("WriteProductionFile", "Bad file name: %s", filename);
4851 return;
4852 }
d3b18c4c 4853 TString workdir;
4854 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4855 workdir = gGrid->GetHomeDirectory();
5513444a 4856 workdir += fGridWorkingDir;
f5e8c702 4857 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4858 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4859 out << locjdl << " " << njobspermaster << endl;
5513444a 4860 Int_t nmasterjobs = fInputFiles->GetEntries();
4861 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4862 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4863 runOutDir.ReplaceAll(".xml", "");
4864 if (fOutputToRunNo)
4865 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4866 else
4867 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4868 }
d3b18c4c 4869 if (gGrid) {
4870 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4871 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4872// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4873 if (!copyLocal2Alien("WriteProductionFile", filename,
4874 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4875 }
5513444a 4876}
4877
4878//______________________________________________________________________________
0f389141 4879void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4880{
4881// Generate the alien validation script.
4882 // Generate the validation script
4883 TObjString *os;
d3b18c4c 4884 if (fValidationScript.IsNull()) {
4885 fValidationScript = fExecutable;
4886 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4887 }
4888 TString validationScript = fValidationScript;
4889 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4890 if (!Connect()) {
4891 Error("WriteValidationScript", "Alien connection required");
4892 return;
4893 }
d3b18c4c 4894 if (!fTerminateFiles.IsNull()) {
4895 fTerminateFiles.Strip();
4896 fTerminateFiles.ReplaceAll(" ",",");
4897 }
a2f5fc01 4898 TString outStream = "";
4899 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4900 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4901 ofstream out;
0d5d317c 4902 out.open(validationScript, ios::out);
c57f56b7 4903 out << "#!/bin/bash" << endl;
4904 out << "##################################################" << endl;
4905 out << "validateout=`dirname $0`" << endl;
4906 out << "validatetime=`date`" << endl;
4907 out << "validated=\"0\";" << endl;
4908 out << "error=0" << endl;
4909 out << "if [ -z $validateout ]" << endl;
4910 out << "then" << endl;
4911 out << " validateout=\".\"" << endl;
4912 out << "fi" << endl << endl;
4913 out << "cd $validateout;" << endl;
4914 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4915 out << "echo \"*******************************************************\"" << outStream << endl;
4916 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4917 out << "" << endl;
a2f5fc01 4918 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4919 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4920 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4921 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4922 out << "ls -la ./" << outStream << endl;
4923 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4924 out << "##################################################" << endl;
ebec370a 4925 out << "" << endl;
4926
4927 out << "if [ ! -f stderr ] ; then" << endl;
4928 out << " error=1" << endl;
a2f5fc01 4929 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4930 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4931 out << "fi" << endl;
4932
b34c9f51 4933 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4934 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4935 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4936 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4937 out << "" << endl;
4938
ebec370a 4939 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4940 out << " error=1" << endl;
a2f5fc01 4941 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4942 out << " echo \"$parArch\" " << outStream << endl;
4943 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4944 out << "fi" << endl;
4945
4946 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4947 out << " error=1" << endl;
a2f5fc01 4948 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4949 out << " echo \"$segViol\" " << outStream << endl;
4950 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4951 out << "fi" << endl;
4952
4953 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4954 out << " error=1" << endl;
a2f5fc01 4955 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4956 out << " echo \"$segFault\" " << outStream << endl;
4957 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4958 out << "fi" << endl;
4959
b34c9f51 4960 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4961 out << " error=1" << endl;
a2f5fc01 4962 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4963 out << " echo \"$glibcErr\" " << outStream << endl;
4964 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4965 out << "fi" << endl;
4966
ebec370a 4967 // Part dedicated to the specific analyses running into the train
4968
d3b18c4c 4969 TString outputFiles = fOutputFiles;
4970 if (merge && !fTerminateFiles.IsNull()) {
4971 outputFiles += ",";
4972 outputFiles += fTerminateFiles;
4973 }
4974 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4975 TIter next1(arr);
a2f5fc01 4976 TString outputFile;
bbd88db8 4977 while (!merge && (os=(TObjString*)next1())) {
4978 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4979 outputFile = os->GetString();
4980 Int_t index = outputFile.Index("@");
4981 if (index > 0) outputFile.Remove(index);
bbd88db8 4982 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4983 if (outputFile.Contains("*")) continue;
4984 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4985 out << " error=1" << endl;
7c2cd90a 4986 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4987 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4988 out << "fi" << endl;
4989 }
4990 delete arr;
7c2cd90a 4991 out << "if ! [ -f outputs_valid ] ; then" << endl;
4992 out << " error=1" << endl;
4993 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4994 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4995 out << "fi" << endl;
923e2ca5 4996
c57f56b7 4997 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4998 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4999 if (!IsKeepLogs()) {
5000 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 5001 outStream = "";
149d288c 5002 out << " rm -f std*" << endl;
5003 }
c57f56b7 5004 out << "fi" << endl;
5005
a2f5fc01 5006 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5007 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 5008 out << "cd -" << endl;
5009 out << "exit $error" << endl;
5010 }
5011 Bool_t copy = kTRUE;
d3b18c4c 5012 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 5013 if (copy) {
5014 CdWork();
5015 TString workdir = gGrid->GetHomeDirectory();
5016 workdir += fGridWorkingDir;
d3b18c4c 5017 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 5018 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 5019// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
5020 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
5021 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 5022 }
5023}