]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Fix for coverity warning 21784. (Misha Veldhoen <Misha.Veldhoen@cern.ch>)
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
c82bb898 51using std::ofstream;
52using std::ifstream;
53using std::ios;
54using std::endl;
c57f56b7 55ClassImp(AliAnalysisAlien)
6c395669 56#if 0
57;
58#endif
c57f56b7 59
6c395669 60namespace {
61 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
62 {
63 TString sl(Form("file:%s", loc));
64 TString sr(Form("alien://%s", rem));
65 Bool_t ret = TFile::Cp(sl, sr);
66 if (!ret) {
67 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
68 }
69 return ret;
70 }
71}
72
c57f56b7 73//______________________________________________________________________________
74AliAnalysisAlien::AliAnalysisAlien()
75 :AliAnalysisGrid(),
76 fGridJDL(NULL),
0f389141 77 fMergingJDL(NULL),
c57f56b7 78 fPrice(0),
79 fTTL(0),
80 fSplitMaxInputFileNumber(0),
81 fMaxInitFailed(0),
82 fMasterResubmitThreshold(0),
bb885a9e 83 fNtestFiles(0),
319593fb 84 fNrunsPerMaster(0),
16a4353c 85 fMaxMergeFiles(0),
b3e07543 86 fMaxMergeStages(0),
d2a409b2 87 fNsubmitted(0),
a3e84053 88 fProductionMode(0),
cd11251e 89 fOutputToRunNo(0),
0f389141 90 fMergeViaJDL(0),
91 fFastReadOption(0),
e1c22e21 92 fOverwriteMode(1),
149d288c 93 fNreplicas(2),
3bdcb562 94 fNproofWorkers(0),
27734f0e 95 fNproofWorkersPerSlave(0),
3bdcb562 96 fProofReset(0),
c57f56b7 97 fRunNumbers(),
98 fExecutable(),
0a1c1f7f 99 fExecutableCommand(),
c57f56b7 100 fArguments(),
631c0b05 101 fExecutableArgs(),
c57f56b7 102 fAnalysisMacro(),
103 fAnalysisSource(),
d3b18c4c 104 fValidationScript(),
d5c6455a 105 fAdditionalRootLibs(),
c57f56b7 106 fAdditionalLibs(),
107 fSplitMode(),
108 fAPIVersion(),
109 fROOTVersion(),
110 fAliROOTVersion(),
648174cf 111 fExternalPackages(),
c57f56b7 112 fUser(),
113 fGridWorkingDir(),
114 fGridDataDir(),
115 fDataPattern(),
116 fGridOutputDir(),
117 fOutputArchive(),
118 fOutputFiles(),
119 fInputFormat(),
e7c71df0 120 fDatasetName(),
c57f56b7 121 fJDLName(),
d3b18c4c 122 fTerminateFiles(),
bb885a9e 123 fMergeExcludes(),
37d21c01 124 fRegisterExcludes(),
f965131e 125 fIncludePath(),
bb885a9e 126 fCloseSE(),
0df6ccf2 127 fFriendChainName(),
c6cb3634 128 fJobTag(),
648174cf 129 fOutputSingle(),
5fce53f4 130 fRunPrefix(),
3bdcb562 131 fProofCluster(),
132 fProofDataSet(),
133 fFileForTestMode(),
3bdcb562 134 fAliRootMode(),
d88e4159 135 fProofProcessOpt(),
f47d5cb4 136 fMergeDirName(),
4e5c5506 137 fInputFiles(0),
28ce6c9c 138 fPackages(0),
77f34eae 139 fModules(0),
d88e4159 140 fProofParam(),
a41b2882 141 fDropToShell(true),
142 fGridJobIDs(""),
143 fGridStages(""),
144 fFriendLibs("")
c57f56b7 145{
146// Dummy ctor.
147 SetDefaults();
148}
149
150//______________________________________________________________________________
151AliAnalysisAlien::AliAnalysisAlien(const char *name)
152 :AliAnalysisGrid(name),
153 fGridJDL(NULL),
0f389141 154 fMergingJDL(NULL),
c57f56b7 155 fPrice(0),
156 fTTL(0),
157 fSplitMaxInputFileNumber(0),
158 fMaxInitFailed(0),
159 fMasterResubmitThreshold(0),
bb885a9e 160 fNtestFiles(0),
319593fb 161 fNrunsPerMaster(0),
16a4353c 162 fMaxMergeFiles(0),
b3e07543 163 fMaxMergeStages(0),
d2a409b2 164 fNsubmitted(0),
a3e84053 165 fProductionMode(0),
cd11251e 166 fOutputToRunNo(0),
0f389141 167 fMergeViaJDL(0),
168 fFastReadOption(0),
e1c22e21 169 fOverwriteMode(1),
149d288c 170 fNreplicas(2),
3bdcb562 171 fNproofWorkers(0),
27734f0e 172 fNproofWorkersPerSlave(0),
3bdcb562 173 fProofReset(0),
c57f56b7 174 fRunNumbers(),
175 fExecutable(),
0a1c1f7f 176 fExecutableCommand(),
c57f56b7 177 fArguments(),
631c0b05 178 fExecutableArgs(),
c57f56b7 179 fAnalysisMacro(),
180 fAnalysisSource(),
d3b18c4c 181 fValidationScript(),
d5c6455a 182 fAdditionalRootLibs(),
c57f56b7 183 fAdditionalLibs(),
184 fSplitMode(),
185 fAPIVersion(),
186 fROOTVersion(),
187 fAliROOTVersion(),
648174cf 188 fExternalPackages(),
c57f56b7 189 fUser(),
190 fGridWorkingDir(),
191 fGridDataDir(),
192 fDataPattern(),
193 fGridOutputDir(),
194 fOutputArchive(),
195 fOutputFiles(),
196 fInputFormat(),
e7c71df0 197 fDatasetName(),
c57f56b7 198 fJDLName(),
d3b18c4c 199 fTerminateFiles(),
bb885a9e 200 fMergeExcludes(),
37d21c01 201 fRegisterExcludes(),
f965131e 202 fIncludePath(),
bb885a9e 203 fCloseSE(),
0df6ccf2 204 fFriendChainName(),
c6cb3634 205 fJobTag(),
648174cf 206 fOutputSingle(),
5fce53f4 207 fRunPrefix(),
3bdcb562 208 fProofCluster(),
209 fProofDataSet(),
210 fFileForTestMode(),
3bdcb562 211 fAliRootMode(),
d88e4159 212 fProofProcessOpt(),
f47d5cb4 213 fMergeDirName(),
4e5c5506 214 fInputFiles(0),
28ce6c9c 215 fPackages(0),
77f34eae 216 fModules(0),
d88e4159 217 fProofParam(),
0ad3ea1f 218 fDropToShell(true),
219 fGridJobIDs(""),
a41b2882 220 fGridStages(""),
221 fFriendLibs("")
c57f56b7 222{
223// Default ctor.
224 SetDefaults();
225}
226
227//______________________________________________________________________________
228AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
229 :AliAnalysisGrid(other),
230 fGridJDL(NULL),
0f389141 231 fMergingJDL(NULL),
c57f56b7 232 fPrice(other.fPrice),
233 fTTL(other.fTTL),
234 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
235 fMaxInitFailed(other.fMaxInitFailed),
236 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 237 fNtestFiles(other.fNtestFiles),
319593fb 238 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 239 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 240 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 241 fNsubmitted(other.fNsubmitted),
a3e84053 242 fProductionMode(other.fProductionMode),
cd11251e 243 fOutputToRunNo(other.fOutputToRunNo),
0f389141 244 fMergeViaJDL(other.fMergeViaJDL),
245 fFastReadOption(other.fFastReadOption),
246 fOverwriteMode(other.fOverwriteMode),
149d288c 247 fNreplicas(other.fNreplicas),
3bdcb562 248 fNproofWorkers(other.fNproofWorkers),
27734f0e 249 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 250 fProofReset(other.fProofReset),
c57f56b7 251 fRunNumbers(other.fRunNumbers),
252 fExecutable(other.fExecutable),
0a1c1f7f 253 fExecutableCommand(other.fExecutableCommand),
c57f56b7 254 fArguments(other.fArguments),
631c0b05 255 fExecutableArgs(other.fExecutableArgs),
c57f56b7 256 fAnalysisMacro(other.fAnalysisMacro),
257 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 258 fValidationScript(other.fValidationScript),
d5c6455a 259 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 260 fAdditionalLibs(other.fAdditionalLibs),
261 fSplitMode(other.fSplitMode),
262 fAPIVersion(other.fAPIVersion),
263 fROOTVersion(other.fROOTVersion),
264 fAliROOTVersion(other.fAliROOTVersion),
648174cf 265 fExternalPackages(other.fExternalPackages),
c57f56b7 266 fUser(other.fUser),
267 fGridWorkingDir(other.fGridWorkingDir),
268 fGridDataDir(other.fGridDataDir),
269 fDataPattern(other.fDataPattern),
270 fGridOutputDir(other.fGridOutputDir),
271 fOutputArchive(other.fOutputArchive),
272 fOutputFiles(other.fOutputFiles),
273 fInputFormat(other.fInputFormat),
e7c71df0 274 fDatasetName(other.fDatasetName),
c57f56b7 275 fJDLName(other.fJDLName),
d3b18c4c 276 fTerminateFiles(other.fTerminateFiles),
bb885a9e 277 fMergeExcludes(other.fMergeExcludes),
37d21c01 278 fRegisterExcludes(other.fRegisterExcludes),
f965131e 279 fIncludePath(other.fIncludePath),
bb885a9e 280 fCloseSE(other.fCloseSE),
0df6ccf2 281 fFriendChainName(other.fFriendChainName),
c6cb3634 282 fJobTag(other.fJobTag),
648174cf 283 fOutputSingle(other.fOutputSingle),
5fce53f4 284 fRunPrefix(other.fRunPrefix),
3bdcb562 285 fProofCluster(other.fProofCluster),
286 fProofDataSet(other.fProofDataSet),
287 fFileForTestMode(other.fFileForTestMode),
3bdcb562 288 fAliRootMode(other.fAliRootMode),
d88e4159 289 fProofProcessOpt(other.fProofProcessOpt),
f47d5cb4 290 fMergeDirName(other.fMergeDirName),
4e5c5506 291 fInputFiles(0),
28ce6c9c 292 fPackages(0),
77f34eae 293 fModules(0),
d88e4159 294 fProofParam(),
0ad3ea1f 295 fDropToShell(other.fDropToShell),
296 fGridJobIDs(other.fGridJobIDs),
a41b2882 297 fGridStages(other.fGridStages),
298 fFriendLibs(other.fFriendLibs)
c57f56b7 299{
300// Copy ctor.
301 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 302 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 303 fRunRange[0] = other.fRunRange[0];
304 fRunRange[1] = other.fRunRange[1];
c57f56b7 305 if (other.fInputFiles) {
306 fInputFiles = new TObjArray();
307 TIter next(other.fInputFiles);
308 TObject *obj;
309 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
310 fInputFiles->SetOwner();
311 }
4e5c5506 312 if (other.fPackages) {
313 fPackages = new TObjArray();
314 TIter next(other.fPackages);
315 TObject *obj;
316 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
317 fPackages->SetOwner();
318 }
77f34eae 319 if (other.fModules) {
320 fModules = new TObjArray();
321 fModules->SetOwner();
322 TIter next(other.fModules);
323 AliAnalysisTaskCfg *mod, *crt;
324 while ((crt=(AliAnalysisTaskCfg*)next())) {
325 mod = new AliAnalysisTaskCfg(*crt);
326 fModules->Add(mod);
327 }
328 }
c57f56b7 329}
330
331//______________________________________________________________________________
332AliAnalysisAlien::~AliAnalysisAlien()
333{
334// Destructor.
77f34eae 335 delete fGridJDL;
336 delete fMergingJDL;
337 delete fInputFiles;
338 delete fPackages;
339 delete fModules;
28ce6c9c 340 fProofParam.DeleteAll();
c57f56b7 341}
342
343//______________________________________________________________________________
344AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
345{
346// Assignment.
347 if (this != &other) {
348 AliAnalysisGrid::operator=(other);
349 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 350 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 351 fPrice = other.fPrice;
352 fTTL = other.fTTL;
353 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
354 fMaxInitFailed = other.fMaxInitFailed;
355 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 356 fNtestFiles = other.fNtestFiles;
a3e84053 357 fNrunsPerMaster = other.fNrunsPerMaster;
358 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 359 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 360 fNsubmitted = other.fNsubmitted;
361 fProductionMode = other.fProductionMode;
cd11251e 362 fOutputToRunNo = other.fOutputToRunNo;
0f389141 363 fMergeViaJDL = other.fMergeViaJDL;
364 fFastReadOption = other.fFastReadOption;
365 fOverwriteMode = other.fOverwriteMode;
149d288c 366 fNreplicas = other.fNreplicas;
3bdcb562 367 fNproofWorkers = other.fNproofWorkers;
27734f0e 368 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 369 fProofReset = other.fProofReset;
c57f56b7 370 fRunNumbers = other.fRunNumbers;
371 fExecutable = other.fExecutable;
0a1c1f7f 372 fExecutableCommand = other.fExecutableCommand;
c57f56b7 373 fArguments = other.fArguments;
631c0b05 374 fExecutableArgs = other.fExecutableArgs;
c57f56b7 375 fAnalysisMacro = other.fAnalysisMacro;
376 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 377 fValidationScript = other.fValidationScript;
d5c6455a 378 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 379 fAdditionalLibs = other.fAdditionalLibs;
380 fSplitMode = other.fSplitMode;
381 fAPIVersion = other.fAPIVersion;
382 fROOTVersion = other.fROOTVersion;
383 fAliROOTVersion = other.fAliROOTVersion;
648174cf 384 fExternalPackages = other.fExternalPackages;
c57f56b7 385 fUser = other.fUser;
386 fGridWorkingDir = other.fGridWorkingDir;
387 fGridDataDir = other.fGridDataDir;
388 fDataPattern = other.fDataPattern;
389 fGridOutputDir = other.fGridOutputDir;
390 fOutputArchive = other.fOutputArchive;
391 fOutputFiles = other.fOutputFiles;
392 fInputFormat = other.fInputFormat;
e7c71df0 393 fDatasetName = other.fDatasetName;
c57f56b7 394 fJDLName = other.fJDLName;
d3b18c4c 395 fTerminateFiles = other.fTerminateFiles;
bb885a9e 396 fMergeExcludes = other.fMergeExcludes;
37d21c01 397 fRegisterExcludes = other.fRegisterExcludes;
f965131e 398 fIncludePath = other.fIncludePath;
bb885a9e 399 fCloseSE = other.fCloseSE;
0df6ccf2 400 fFriendChainName = other.fFriendChainName;
c6cb3634 401 fJobTag = other.fJobTag;
648174cf 402 fOutputSingle = other.fOutputSingle;
5fce53f4 403 fRunPrefix = other.fRunPrefix;
3bdcb562 404 fProofCluster = other.fProofCluster;
405 fProofDataSet = other.fProofDataSet;
406 fFileForTestMode = other.fFileForTestMode;
3bdcb562 407 fAliRootMode = other.fAliRootMode;
d88e4159 408 fProofProcessOpt = other.fProofProcessOpt;
f47d5cb4 409 fMergeDirName = other.fMergeDirName;
d88e4159 410 fDropToShell = other.fDropToShell;
0ad3ea1f 411 fGridJobIDs = other.fGridJobIDs;
412 fGridStages = other.fGridStages;
a41b2882 413 fFriendLibs = other.fFriendLibs;
c57f56b7 414 if (other.fInputFiles) {
415 fInputFiles = new TObjArray();
416 TIter next(other.fInputFiles);
417 TObject *obj;
418 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
419 fInputFiles->SetOwner();
420 }
4e5c5506 421 if (other.fPackages) {
422 fPackages = new TObjArray();
423 TIter next(other.fPackages);
424 TObject *obj;
425 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
426 fPackages->SetOwner();
427 }
77f34eae 428 if (other.fModules) {
429 fModules = new TObjArray();
430 fModules->SetOwner();
431 TIter next(other.fModules);
432 AliAnalysisTaskCfg *mod, *crt;
433 while ((crt=(AliAnalysisTaskCfg*)next())) {
434 mod = new AliAnalysisTaskCfg(*crt);
435 fModules->Add(mod);
436 }
437 }
c57f56b7 438 }
439 return *this;
440}
441
37d21c01 442//______________________________________________________________________________
443void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
444{
445// Add a single additional library to be loaded. Extension must be present.
446 TString lib(name);
447 if (!lib.Contains(".")) {
448 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
449 return;
450 }
451 if (fAdditionalLibs.Contains(name)) {
452 Warning("AddAdditionalLibrary", "Library %s already added.", name);
453 return;
454 }
455 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
456 fAdditionalLibs += lib;
457}
458
77f34eae 459//______________________________________________________________________________
460void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
461{
462// Adding a module. Checks if already existing. Becomes owned by this.
463 if (!module) return;
464 if (GetModule(module->GetName())) {
465 Error("AddModule", "A module having the same name %s already added", module->GetName());
466 return;
467 }
468 if (!fModules) {
469 fModules = new TObjArray();
470 fModules->SetOwner();
471 }
472 fModules->Add(module);
473}
474
475//______________________________________________________________________________
476void AliAnalysisAlien::AddModules(TObjArray *list)
477{
478// Adding a list of modules. Checks if already existing. Becomes owned by this.
479 TIter next(list);
480 AliAnalysisTaskCfg *module;
481 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
482}
483
484//______________________________________________________________________________
485Bool_t AliAnalysisAlien::CheckDependencies()
486{
487// Check if all dependencies are satisfied. Reorder modules if needed.
488 Int_t nmodules = GetNmodules();
489 if (!nmodules) {
490 Warning("CheckDependencies", "No modules added yet to check their dependencies");
491 return kTRUE;
492 }
493 AliAnalysisTaskCfg *mod = 0;
494 AliAnalysisTaskCfg *dep = 0;
495 TString depname;
496 Int_t i, j, k;
497 for (i=0; i<nmodules; i++) {
498 mod = (AliAnalysisTaskCfg*) fModules->At(i);
499 Int_t ndeps = mod->GetNdeps();
500 Int_t istart = i;
501 for (j=0; j<ndeps; j++) {
502 depname = mod->GetDependency(j);
503 dep = GetModule(depname);
504 if (!dep) {
505 Error("CheckDependencies","Dependency %s not added for module %s",
506 depname.Data(), mod->GetName());
507 return kFALSE;
508 }
509 if (dep->NeedsDependency(mod->GetName())) {
510 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
511 mod->GetName(), dep->GetName());
512 return kFALSE;
513 }
514 Int_t idep = fModules->IndexOf(dep);
515 // The dependency task must come first
516 if (idep>i) {
517 // Remove at idep and move all objects below up one slot
518 // down to index i included.
519 fModules->RemoveAt(idep);
3e40fd4c 520 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 521 fModules->AddAt(dep, i++);
522 }
523 //Redo from istart if dependencies were inserted
524 if (i>istart) i=istart-1;
525 }
526 }
527 return kTRUE;
528}
529
3e40fd4c 530//______________________________________________________________________________
531AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
532{
533// Create the analysis manager and optionally execute the macro in filename.
534 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
535 if (mgr) return mgr;
536 mgr = new AliAnalysisManager(name);
537 mgr->SetGridHandler((AliAnalysisGrid*)this);
538 if (strlen(filename)) {
539 TString line = gSystem->ExpandPathName(filename);
540 line.Prepend(".x ");
541 gROOT->ProcessLine(line.Data());
542 }
543 return mgr;
544}
545
77f34eae 546//______________________________________________________________________________
547Int_t AliAnalysisAlien::GetNmodules() const
548{
549// Get number of modules.
550 if (!fModules) return 0;
551 return fModules->GetEntries();
552}
553
554//______________________________________________________________________________
555AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
556{
557// Get a module by name.
558 if (!fModules) return 0;
559 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
560}
561
562//______________________________________________________________________________
563Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
564{
565// Load a given module.
566 if (mod->IsLoaded()) return kTRUE;
192ddca2 567 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
568 if (!mgr) {
569 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
570 return kFALSE;
571 }
77f34eae 572 Int_t ndeps = mod->GetNdeps();
573 TString depname;
574 for (Int_t j=0; j<ndeps; j++) {
575 depname = mod->GetDependency(j);
576 AliAnalysisTaskCfg *dep = GetModule(depname);
577 if (!dep) {
578 Error("LoadModule","Dependency %s not existing for module %s",
579 depname.Data(), mod->GetName());
580 return kFALSE;
581 }
582 if (!LoadModule(dep)) {
583 Error("LoadModule","Dependency %s for module %s could not be loaded",
584 depname.Data(), mod->GetName());
585 return kFALSE;
586 }
587 }
588 // Load libraries for the module
589 if (!mod->CheckLoadLibraries()) {
590 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
591 return kFALSE;
592 }
192ddca2 593 // Check if a custom file name was requested
594 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
595
596 // Check if a custom terminate file name was requested
597 if (strlen(mod->GetTerminateFileName())) {
598 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
599 fTerminateFiles += mod->GetTerminateFileName();
600 }
601
77f34eae 602 // Execute the macro
603 if (mod->ExecuteMacro()<0) {
604 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
605 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
606 return kFALSE;
607 }
608 // Configure dependencies
609 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
610 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
611 mod->GetConfigMacro()->GetTitle(), mod->GetName());
612 return kFALSE;
613 }
5e8c7c22 614 // Adjust extra libraries
615 Int_t nlibs = mod->GetNlibs();
616 TString lib;
617 for (Int_t i=0; i<nlibs; i++) {
618 lib = mod->GetLibrary(i);
5e8c7c22 619 lib = Form("lib%s.so", lib.Data());
c9d6bd12 620 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 621 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
622 fAdditionalLibs += lib;
623 }
77f34eae 624 return kTRUE;
625}
626
139fe6ba 627//______________________________________________________________________________
628Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
629{
630// Generate the full train.
631 fAdditionalLibs = "";
632 if (!LoadModules()) return kFALSE;
633 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
634 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 635 mgr->RunLocalInit();
139fe6ba 636 mgr->PrintStatus();
637 Int_t productionMode = fProductionMode;
638 SetProductionMode();
639 TString macro = fAnalysisMacro;
640 TString executable = fExecutable;
641 TString validation = fValidationScript;
642 TString execCommand = fExecutableCommand;
643 SetAnalysisMacro(Form("%s.C", name));
644 SetExecutable(Form("%s.sh", name));
a8e31fb0 645// SetExecutableCommand("aliroot -b -q ");
139fe6ba 646 SetValidationScript(Form("%s_validation.sh", name));
647 StartAnalysis();
648 SetProductionMode(productionMode);
649 fAnalysisMacro = macro;
650 fExecutable = executable;
651 fExecutableCommand = execCommand;
652 fValidationScript = validation;
653 return kTRUE;
654}
655
77f34eae 656//______________________________________________________________________________
21ca8e59 657Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 658{
659// Generate test macros for a single module or for the full train.
5e8c7c22 660 fAdditionalLibs = "";
77f34eae 661 if (strlen(modname)) {
21ca8e59 662 if (!CheckDependencies()) return kFALSE;
663 AliAnalysisTaskCfg *mod = GetModule(modname);
664 if (!mod) {
665 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
666 return kFALSE;
667 }
668 if (!LoadModule(mod)) return kFALSE;
a41b2882 669 if (!LoadFriendLibs()) return kFALSE;
21ca8e59 670 } else if (!LoadModules()) return kFALSE;
671 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
672 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 673 mgr->RunLocalInit();
21ca8e59 674 mgr->PrintStatus();
675 SetLocalTest(kTRUE);
676 Int_t productionMode = fProductionMode;
677 SetProductionMode();
678 TString macro = fAnalysisMacro;
679 TString executable = fExecutable;
680 TString validation = fValidationScript;
681 TString execCommand = fExecutableCommand;
682 SetAnalysisMacro(Form("%s.C", name));
683 SetExecutable(Form("%s.sh", name));
d66aa2a9 684 fOutputFiles = GetListOfFiles("outaod");
685 // Add extra files registered to the analysis manager
686 TString extra = GetListOfFiles("ext");
687 if (!extra.IsNull()) {
688 extra.ReplaceAll(".root", "*.root");
689 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
690 fOutputFiles += extra;
691 }
a8e31fb0 692// SetExecutableCommand("aliroot -b -q ");
21ca8e59 693 SetValidationScript(Form("%s_validation.sh", name));
694 WriteAnalysisFile();
695 WriteAnalysisMacro();
696 WriteExecutable();
697 WriteValidationScript();
ffbe06b9 698 WriteMergingMacro();
699 WriteMergeExecutable();
700 WriteValidationScript(kTRUE);
21ca8e59 701 SetLocalTest(kFALSE);
702 SetProductionMode(productionMode);
703 fAnalysisMacro = macro;
704 fExecutable = executable;
705 fExecutableCommand = execCommand;
706 fValidationScript = validation;
77f34eae 707 return kTRUE;
708}
709
710//______________________________________________________________________________
711Bool_t AliAnalysisAlien::LoadModules()
712{
713// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 714 fAdditionalLibs = "";
21ca8e59 715 Int_t nmodules = GetNmodules();
716 if (!nmodules) {
717 Warning("LoadModules", "No module to be loaded");
718 return kTRUE;
719 }
3e40fd4c 720 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
721 if (!mgr) {
722 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
723 return kFALSE;
724 }
77f34eae 725 if (!CheckDependencies()) return kFALSE;
21ca8e59 726 nmodules = GetNmodules();
77f34eae 727 AliAnalysisTaskCfg *mod;
21ca8e59 728 for (Int_t imod=0; imod<nmodules; imod++) {
729 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 730 if (!LoadModule(mod)) return kFALSE;
731 }
a41b2882 732 // Load additional friend libraries
733 return LoadFriendLibs();
77f34eae 734}
735
a41b2882 736//______________________________________________________________________________
737Bool_t AliAnalysisAlien::LoadFriendLibs() const
738{
739// Load libraries required for reading friends.
740 if (fFriendLibs.Length()) {
741 TObjArray *list = 0;
742 TString lib;
743 if (fFriendLibs.Contains(",")) list = fFriendLibs.Tokenize(",");
744 else list = fFriendLibs.Tokenize(" ");
745 for (Int_t ilib=0; ilib<list->GetEntriesFast(); ilib++) {
746 lib = list->At(ilib)->GetName();
747 lib.ReplaceAll(".so","");
748 lib.ReplaceAll(" ","");
749 if (lib.BeginsWith("lib")) lib.Remove(0, 3);
750 lib.Prepend("lib");
751 Int_t loaded = strlen(gSystem->GetLibraries(lib,"",kFALSE));
752 if (!loaded) loaded = gSystem->Load(lib);
753 if (loaded < 0) {
754 Error("LoadModules", "Cannot load library for friends %s", lib.Data());
755 return kFALSE;
756 }
757 }
758 delete list;
759 }
760 return kTRUE;
761}
762
74013535 763//______________________________________________________________________________
764void AliAnalysisAlien::SetRunPrefix(const char *prefix)
765{
766// Set the run number format. Can be a prefix or a format like "%09d"
767 fRunPrefix = prefix;
768 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
769}
770
f965131e 771//______________________________________________________________________________
772void AliAnalysisAlien::AddIncludePath(const char *path)
773{
774// Add include path in the remote analysis macro.
775 TString p(path);
776 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
777 else fIncludePath += Form("-I%s ", path);
778}
779
c57f56b7 780//______________________________________________________________________________
781void AliAnalysisAlien::AddRunNumber(Int_t run)
782{
783// Add a run number to the list of runs to be processed.
784 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 785 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 786}
787
98690895 788//______________________________________________________________________________
789void AliAnalysisAlien::AddRunList(const char* runList)
790{
791// Add several runs into the list of runs; they are expected to be separated by a blank character.
792 TString sList = runList;
793 TObjArray *list = sList.Tokenize(" ");
794 Int_t n = list->GetEntries();
795 for (Int_t i = 0; i < n; i++) {
796 TObjString *os = (TObjString*)list->At(i);
797 AddRunNumber(os->GetString().Atoi());
798 }
799 delete list;
800}
801
ee75cfc3 802//______________________________________________________________________________
803void AliAnalysisAlien::AddRunNumber(const char* run)
804{
805// Add a run number to the list of runs to be processed.
7aa5d539 806 TString runs = run;
807 TObjString *os;
808 TObjArray *arr = runs.Tokenize(" ");
809 TIter next(arr);
810 TString prefix;
811 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
812 while ((os=(TObjString*)next())){
813 if (fRunNumbers.Length()) fRunNumbers += " ";
814 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
815 }
816 delete arr;
ee75cfc3 817}
818
c57f56b7 819//______________________________________________________________________________
820void AliAnalysisAlien::AddDataFile(const char *lfn)
821{
822// Adds a data file to the input to be analysed. The file should be a valid LFN
823// or point to an existing file in the alien workdir.
824 if (!fInputFiles) fInputFiles = new TObjArray();
825 fInputFiles->Add(new TObjString(lfn));
826}
648174cf 827
828//______________________________________________________________________________
829void AliAnalysisAlien::AddExternalPackage(const char *package)
830{
831// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
832 if (fExternalPackages) fExternalPackages += " ";
833 fExternalPackages += package;
834}
835
c57f56b7 836//______________________________________________________________________________
837Bool_t AliAnalysisAlien::Connect()
838{
839// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
840 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 841 if (fProductionMode) return kTRUE;
c57f56b7 842 if (!gGrid) {
843 Info("Connect", "Trying to connect to AliEn ...");
844 TGrid::Connect("alien://");
845 }
846 if (!gGrid || !gGrid->IsConnected()) {
847 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
848 return kFALSE;
849 }
850 fUser = gGrid->GetUser();
851 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
852 return kTRUE;
853}
854
855//______________________________________________________________________________
856void AliAnalysisAlien::CdWork()
857{
858// Check validity of alien workspace. Create directory if possible.
859 if (!Connect()) {
860 Error("CdWork", "Alien connection required");
861 return;
862 }
863 TString homedir = gGrid->GetHomeDirectory();
864 TString workdir = homedir + fGridWorkingDir;
923e2ca5 865 if (DirectoryExists(workdir)) {
866 gGrid->Cd(workdir);
867 return;
868 }
869 // Work directory not existing - create it
870 gGrid->Cd(homedir);
b93f8109 871 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 872 gGrid->Cd(fGridWorkingDir);
d3b18c4c 873 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 874 } else {
d3b18c4c 875 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 876 workdir.Data(), homedir.Data());
877 fGridWorkingDir = "";
878 }
c57f56b7 879}
880
348be253 881//______________________________________________________________________________
882Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
883{
884// Check if file copying is possible.
d3b18c4c 885 if (fProductionMode) return kTRUE;
b5ae4fe0 886 TString salienpath(alienpath);
887 if (salienpath.Contains(" ")) {
888 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
889 return kFALSE;
890 }
348be253 891 if (!Connect()) {
892 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
893 return kFALSE;
894 }
d3339be3 895 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
896 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 897 // Check if alien_CLOSE_SE is defined
898 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
899 if (!closeSE.IsNull()) {
900 Info("CheckFileCopy", "Your current close storage is pointing to: \
901 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
902 } else {
903 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
904 }
905 // Check if grid directory exists.
906 if (!DirectoryExists(alienpath)) {
907 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
908 return kFALSE;
909 }
b5ae4fe0 910 TString stest = "plugin_test_copy";
911 TFile f(stest, "RECREATE");
348be253 912 // User may not have write permissions to current directory
913 if (f.IsZombie()) {
914 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
915 gSystem->WorkingDirectory());
916 return kFALSE;
917 }
918 f.Close();
b5ae4fe0 919 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
920 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 921 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 922 \n# 1. Make sure you have write permissions there. If this is the case: \
923 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
924 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
925 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
926 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 927 gSystem->Unlink(stest.Data());
348be253 928 return kFALSE;
929 }
b5ae4fe0 930 gSystem->Unlink(stest.Data());
931 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 932 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 933 return kTRUE;
934}
935
c57f56b7 936//______________________________________________________________________________
937Bool_t AliAnalysisAlien::CheckInputData()
938{
939// Check validity of input data. If necessary, create xml files.
d3b18c4c 940 if (fProductionMode) return kTRUE;
a8739e8a 941 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 942 if (!fGridDataDir.Length()) {
943 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
944 return kFALSE;
945 }
9f020952 946 if (fMergeViaJDL) {
947 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
948 return kFALSE;
949 }
d2a409b2 950 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 951 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
952 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 953 return kTRUE;
c57f56b7 954 }
955 // Process declared files
a2f5fc01 956 Bool_t isCollection = kFALSE;
957 Bool_t isXml = kFALSE;
958 Bool_t useTags = kFALSE;
c57f56b7 959 Bool_t checked = kFALSE;
d3b18c4c 960 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 961 TString file;
962 TString workdir = gGrid->GetHomeDirectory();
963 workdir += fGridWorkingDir;
964 if (fInputFiles) {
965 TObjString *objstr;
966 TIter next(fInputFiles);
967 while ((objstr=(TObjString*)next())) {
968 file = workdir;
969 file += "/";
970 file += objstr->GetString();
971 // Store full lfn path
972 if (FileExists(file)) objstr->SetString(file);
973 else {
974 file = objstr->GetName();
975 if (!FileExists(objstr->GetName())) {
976 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
977 objstr->GetName(), workdir.Data());
978 return kFALSE;
979 }
980 }
981 Bool_t iscoll, isxml, usetags;
982 CheckDataType(file, iscoll, isxml, usetags);
983 if (!checked) {
984 checked = kTRUE;
a2f5fc01 985 isCollection = iscoll;
986 isXml = isxml;
987 useTags = usetags;
988 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 989 } else {
a2f5fc01 990 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 991 Error("CheckInputData", "Some conflict was found in the types of inputs");
992 return kFALSE;
993 }
994 }
995 }
996 }
997 // Process requested run numbers
a8739e8a 998 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 999 // Check validity of alien data directory
1000 if (!fGridDataDir.Length()) {
1001 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
1002 return kFALSE;
1003 }
923e2ca5 1004 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 1005 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
1006 return kFALSE;
1007 }
a2f5fc01 1008 if (isCollection) {
c57f56b7 1009 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
1010 return kFALSE;
1011 }
1012
a2f5fc01 1013 if (checked && !isXml) {
c57f56b7 1014 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
1015 return kFALSE;
1016 }
1017 // Check validity of run number(s)
1018 TObjArray *arr;
1019 TObjString *os;
74013535 1020 TString format;
319593fb 1021 Int_t nruns = 0;
904f9f5f 1022 TString schunk, schunk2;
c57f56b7 1023 TString path;
1024 if (!checked) {
1025 checked = kTRUE;
a2f5fc01 1026 useTags = fDataPattern.Contains("tag");
1027 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1028 }
a2f5fc01 1029 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 1030 Error("CheckInputData", "Cannot mix input files using/not using tags");
1031 return kFALSE;
1032 }
1033 if (fRunNumbers.Length()) {
a8739e8a 1034 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 1035 arr = fRunNumbers.Tokenize(" ");
1036 TIter next(arr);
1037 while ((os=(TObjString*)next())) {
1038 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1039 if (!DirectoryExists(path)) {
1040 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 1041 continue;
c57f56b7 1042 }
1043 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
1044 TString msg = "\n##### file: ";
1045 msg += path;
1046 msg += " type: xml_collection;";
a2f5fc01 1047 if (useTags) msg += " using_tags: Yes";
c57f56b7 1048 else msg += " using_tags: No";
23329835 1049 Info("CheckDataType", "%s", msg.Data());
319593fb 1050 if (fNrunsPerMaster<2) {
d2a409b2 1051 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 1052 } else {
1053 nruns++;
1054 if (((nruns-1)%fNrunsPerMaster) == 0) {
1055 schunk = os->GetString();
1056 }
1057 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1058 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1059 AddDataFile(schunk);
319593fb 1060 }
c57f56b7 1061 }
1062 delete arr;
a8739e8a 1063 } else {
1064 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1065 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1066 format = Form("%%s/%s ", fRunPrefix.Data());
1067 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1068 if (!DirectoryExists(path)) {
a8739e8a 1069 continue;
1070 }
74013535 1071 format = Form("%%s/%s.xml", fRunPrefix.Data());
1072 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1073 TString msg = "\n##### file: ";
1074 msg += path;
1075 msg += " type: xml_collection;";
a2f5fc01 1076 if (useTags) msg += " using_tags: Yes";
a8739e8a 1077 else msg += " using_tags: No";
23329835 1078 Info("CheckDataType", "%s", msg.Data());
319593fb 1079 if (fNrunsPerMaster<2) {
74013535 1080 format = Form("%s.xml", fRunPrefix.Data());
1081 AddDataFile(Form(format.Data(),irun));
319593fb 1082 } else {
1083 nruns++;
1084 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1085 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1086 }
74013535 1087 format = Form("_%s.xml", fRunPrefix.Data());
1088 schunk2 = Form(format.Data(), irun);
319593fb 1089 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1090 schunk += schunk2;
d2a409b2 1091 AddDataFile(schunk);
319593fb 1092 }
a8739e8a 1093 }
904f9f5f 1094 if (!fInputFiles) {
1095 schunk += schunk2;
1096 AddDataFile(schunk);
1097 }
c57f56b7 1098 }
1099 return kTRUE;
1100}
1101
21ca8e59 1102//______________________________________________________________________________
4a8b667c 1103Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1104{
1105// Copy data from the given grid directory according a pattern and make a local
1106// dataset.
2ba4c287 1107// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1108 if (!Connect()) {
1109 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
4a8b667c 1110 return 0;
21ca8e59 1111 }
1112 if (!DirectoryExists(griddir)) {
1113 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
4a8b667c 1114 return 0;
21ca8e59 1115 }
1116 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1117 printf("Running command: %s\n", command.Data());
1118 TGridResult *res = gGrid->Command(command);
1119 Int_t nfound = res->GetEntries();
1120 if (!nfound) {
1121 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
4a8b667c 1122 return 0;
21ca8e59 1123 }
1124 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1125
1126 // archives
1127 TObjArray* additionalArchives = 0;
1128 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1129 additionalArchives = TString(archivefile).Tokenize(";");
1130 archivefile = additionalArchives->At(0)->GetName();
1131 additionalArchives->RemoveAt(0);
1132 additionalArchives->Compress();
1133 }
1134
21ca8e59 1135 // Copy files locally
1136 ofstream out;
1137 out.open(output, ios::out);
1138 TMap *map;
1139 TString turl, dirname, filename, temp;
1140 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1141 gSystem->MakeDirectory(outputdir);
1142 gSystem->ChangeDirectory(outputdir);
4a8b667c 1143 Int_t ncopied = 0;
21ca8e59 1144 for (Int_t i=0; i<nfound; i++) {
1145 map = (TMap*)res->At(i);
1146 turl = map->GetValue("turl")->GetName();
1147 filename = gSystem->BaseName(turl.Data());
1148 dirname = gSystem->DirName(turl.Data());
1149 dirname = gSystem->BaseName(dirname.Data());
1150 gSystem->MakeDirectory(dirname);
43b807f6 1151
1152 TString source(turl);
1153 TString targetFileName(filename);
1154
1155 if (strlen(archivefile) > 0) {
4a8b667c 1156// TODO here the archive in which the file resides should be determined
1157// however whereis returns only a guid, and guid2lfn does not work
1158// Therefore we use the one provided as argument for now
1159 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1160 targetFileName = archivefile;
43b807f6 1161 }
1162 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
4a8b667c 1163 Bool_t success = kTRUE;
1164 if (additionalArchives) {
1165 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1166 TString target;
1167 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1168 gSystem->MakeDirectory(gSystem->DirName(target));
1169 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1170 }
1171 }
2ba4c287 1172
4a8b667c 1173 if (success) {
1174 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1175 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1176 ncopied++;
1177 }
21ca8e59 1178 }
1179 }
1180 gSystem->ChangeDirectory(cdir);
1181 delete res;
4a8b667c 1182 delete additionalArchives;
1183 return ncopied;
21ca8e59 1184}
1185
c57f56b7 1186//______________________________________________________________________________
1187Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1188{
1189// Create dataset for the grid data directory + run number.
3af861ff 1190 const Int_t gMaxEntries = 15000;
d3b18c4c 1191 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1192 if (!Connect()) {
1193 Error("CreateDataset", "Cannot create dataset with no grid connection");
1194 return kFALSE;
1195 }
1196
1197 // Cd workspace
d3b18c4c 1198 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1199 TString workdir = gGrid->GetHomeDirectory();
1200 workdir += fGridWorkingDir;
1201
1202 // Compose the 'find' command arguments
74013535 1203 TString format;
c57f56b7 1204 TString command;
1205 TString options = "-x collection ";
bb885a9e 1206 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1207 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1208 TString conditions = "";
3af861ff 1209 Int_t nstart = 0;
1210 Int_t ncount = 0;
1211 Int_t stage = 0;
c57f56b7 1212 TString file;
1213 TString path;
319593fb 1214 Int_t nruns = 0;
904f9f5f 1215 TString schunk, schunk2;
ab254fd1 1216 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1217 if (!fRunNumbers.Length() && !fRunRange[0]) {
1218 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1219 // Make a single data collection from data directory.
1220 path = fGridDataDir;
923e2ca5 1221 if (!DirectoryExists(path)) {
d2a409b2 1222 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1223 return kFALSE;
1224 }
923e2ca5 1225// CdWork();
d2a409b2 1226 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1227 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1228 while (1) {
1229 ncount = 0;
1230 stage++;
1231 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1232 command = "find ";
1233 command += Form("%s -o %d ",options.Data(), nstart);
1234 command += path;
1235 command += " ";
1236 command += pattern;
1237 command += conditions;
1238 printf("command: %s\n", command.Data());
1239 TGridResult *res = gGrid->Command(command);
1240 if (res) delete res;
1241 // Write standard output to file
1242 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1243 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1244 Bool_t nullFile = kFALSE;
1245 if (!hasGrep) {
1246 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1247 } else {
1248 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1249 if (nullFile) {
1250 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1251 gSystem->Exec("rm -f __tmp*");
1252 return kFALSE;
1253 }
1254 TString line;
1255 ifstream in;
1256 in.open("__tmp__");
1257 in >> line;
1258 in.close();
1259 gSystem->Exec("rm -f __tmp__");
1260 ncount = line.Atoi();
1261 }
1262 }
1263 if (ncount == gMaxEntries) {
1264 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1265 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1266 if (!cbase) cbase = cadd;
1267 else {
1268 cbase->Add(cadd);
1269 delete cadd;
defd7a3a 1270 }
3af861ff 1271 nstart += ncount;
1272 } else {
1273 if (cbase) {
1274 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1275 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1276 cbase->Add(cadd);
1277 delete cadd;
1278 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1279 delete cbase; cbase = 0;
1280 } else {
1281 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1282 }
1283 gSystem->Exec("rm -f __tmp*");
1284 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1285 break;
1286 }
0f389141 1287 }
1288 Bool_t fileExists = FileExists(file);
1289 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1290 // Copy xml file to alien space
0f389141 1291 if (fileExists) gGrid->Rm(file);
d2a409b2 1292 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1293 if (!FileExists(file)) {
1294 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1295 return kFALSE;
1296 }
1297 // Update list of files to be processed.
1298 }
1299 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1300 return kTRUE;
1301 }
c57f56b7 1302 // Several runs
a2f5fc01 1303 Bool_t nullResult = kTRUE;
a8739e8a 1304 if (fRunNumbers.Length()) {
1305 TObjArray *arr = fRunNumbers.Tokenize(" ");
1306 TObjString *os;
1307 TIter next(arr);
1308 while ((os=(TObjString*)next())) {
3af861ff 1309 nstart = 0;
1310 stage = 0;
dc030a24 1311 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1312 if (!DirectoryExists(path)) continue;
1313// CdWork();
a8739e8a 1314 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1315 else file = Form("%s.xml", os->GetString().Data());
319593fb 1316 // If local collection file does not exist, create it via 'find' command.
3af861ff 1317 while (1) {
1318 ncount = 0;
1319 stage++;
1320 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1321 command = "find ";
1322 command += Form("%s -o %d ",options.Data(), nstart);
1323 command += path;
1324 command += pattern;
1325 command += conditions;
1326 TGridResult *res = gGrid->Command(command);
1327 if (res) delete res;
1328 // Write standard output to file
1329 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1330 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1331 Bool_t nullFile = kFALSE;
1332 if (!hasGrep) {
1333 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1334 } else {
1335 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1336 if (nullFile) {
1337 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1338 gSystem->Exec("rm -f __tmp*");
1339 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1340 break;
3af861ff 1341 }
1342 TString line;
1343 ifstream in;
1344 in.open("__tmp__");
1345 in >> line;
1346 in.close();
1347 gSystem->Exec("rm -f __tmp__");
1348 ncount = line.Atoi();
1349 }
1350 nullResult = kFALSE;
1351 }
1352 if (ncount == gMaxEntries) {
1353 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1354 if (fNrunsPerMaster > 1) {
1355 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1356 file.Data(),gMaxEntries);
1357 return kFALSE;
1358 }
1359 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1360 if (!cbase) cbase = cadd;
1361 else {
1362 cbase->Add(cadd);
1363 delete cadd;
defd7a3a 1364 }
3af861ff 1365 nstart += ncount;
1366 } else {
c89a84bc 1367 if (cbase && fNrunsPerMaster<2) {
3af861ff 1368 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1369 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1370 cbase->Add(cadd);
1371 delete cadd;
1372 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1373 delete cbase; cbase = 0;
1374 } else {
1375 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1376 }
1377 gSystem->Exec("rm -f __tmp*");
1378 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1379 break;
d3339be3 1380 }
3af861ff 1381 }
a8739e8a 1382 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1383 // Check if there is one run per master job.
1384 if (fNrunsPerMaster<2) {
1385 if (FileExists(file)) {
0f389141 1386 if (fOverwriteMode) gGrid->Rm(file);
1387 else {
1388 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1389 continue;
1390 }
319593fb 1391 }
1392 // Copy xml file to alien space
1393 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1394 if (!FileExists(file)) {
1395 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1396 delete arr;
1397 return kFALSE;
1398 }
1399 } else {
1400 nruns++;
1401 if (((nruns-1)%fNrunsPerMaster) == 0) {
1402 schunk = os->GetString();
1403 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1404 } else {
1405 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1406 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1407 cbase->Add(cadd);
1408 delete cadd;
1409 }
1410 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1411 continue;
1412 }
1413 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1414 if (FileExists(schunk)) {
1415 if (fOverwriteMode) gGrid->Rm(file);
1416 else {
1417 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1418 continue;
1419 }
319593fb 1420 }
84fcd93f 1421 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1422 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1423 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1424 if (!FileExists(schunk)) {
1425 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1426 delete arr;
1427 return kFALSE;
1428 }
d3339be3 1429 }
a8739e8a 1430 }
1431 delete arr;
a2f5fc01 1432 if (nullResult) {
d3339be3 1433 Error("CreateDataset", "No valid dataset corresponding to the query!");
1434 return kFALSE;
1435 }
a8739e8a 1436 } else {
1437 // Process a full run range.
1438 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1439 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1440 nstart = 0;
1441 stage = 0;
74013535 1442 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1443 if (!DirectoryExists(path)) continue;
1444// CdWork();
74013535 1445 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1446 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1447 else file = Form(format.Data(), irun);
0f389141 1448 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1449 if (fOverwriteMode) gGrid->Rm(file);
1450 else {
1451 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1452 continue;
1453 }
a8739e8a 1454 }
319593fb 1455 // If local collection file does not exist, create it via 'find' command.
3af861ff 1456 while (1) {
1457 ncount = 0;
1458 stage++;
1459 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1460 command = "find ";
1461 command += Form("%s -o %d ",options.Data(), nstart);
1462 command += path;
1463 command += pattern;
1464 command += conditions;
1465 TGridResult *res = gGrid->Command(command);
1466 if (res) delete res;
1467 // Write standard output to file
1468 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1469 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1470 Bool_t nullFile = kFALSE;
1471 if (!hasGrep) {
1472 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1473 } else {
1474 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1475 if (nullFile) {
1476 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1477 gSystem->Exec("rm -f __tmp*");
1604c9ed 1478 break;
3af861ff 1479 }
1480 TString line;
1481 ifstream in;
1482 in.open("__tmp__");
1483 in >> line;
1484 in.close();
1485 gSystem->Exec("rm -f __tmp__");
1486 ncount = line.Atoi();
1487 }
1488 nullResult = kFALSE;
1489 }
1490 if (ncount == gMaxEntries) {
1491 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1492 if (fNrunsPerMaster > 1) {
1493 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1494 file.Data(),gMaxEntries);
1495 return kFALSE;
1496 }
1497 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1498 if (!cbase) cbase = cadd;
1499 else {
1500 cbase->Add(cadd);
1501 delete cadd;
defd7a3a 1502 }
3af861ff 1503 nstart += ncount;
1504 } else {
c89a84bc 1505 if (cbase && fNrunsPerMaster<2) {
3af861ff 1506 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1507 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1508 cbase->Add(cadd);
1509 delete cadd;
1510 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1511 delete cbase; cbase = 0;
1512 } else {
1513 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1514 }
1515 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1516 break;
d3339be3 1517 }
319593fb 1518 }
a8739e8a 1519 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1520 // Check if there is one run per master job.
1521 if (fNrunsPerMaster<2) {
1522 if (FileExists(file)) {
0f389141 1523 if (fOverwriteMode) gGrid->Rm(file);
1524 else {
1525 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1526 continue;
1527 }
319593fb 1528 }
1529 // Copy xml file to alien space
1530 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1531 if (!FileExists(file)) {
1532 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1533 return kFALSE;
1534 }
1535 } else {
1536 nruns++;
95e5b448 1537 // Check if the collection for the chunk exist locally.
1538 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1539 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1540 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1541 else continue;
1542 }
84fcd93f 1543 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1544 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1545 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1546 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1547 } else {
1548 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1549 cbase->Add(cadd);
1550 delete cadd;
1551 }
74013535 1552 format = Form("%%s_%s.xml", fRunPrefix.Data());
1553 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1554 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1555 continue;
1556 }
904f9f5f 1557 schunk = schunk2;
319593fb 1558 if (FileExists(schunk)) {
0f389141 1559 if (fOverwriteMode) gGrid->Rm(schunk);
1560 else {
1561 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1562 continue;
1563 }
319593fb 1564 }
84fcd93f 1565 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1566 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1567 if (FileExists(schunk)) {
0f389141 1568 if (fOverwriteMode) gGrid->Rm(schunk);
1569 else {
1570 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1571 continue;
1572 }
95e5b448 1573 }
319593fb 1574 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1575 if (!FileExists(schunk)) {
1576 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1577 return kFALSE;
1578 }
1579 }
c57f56b7 1580 }
a2f5fc01 1581 if (nullResult) {
d3339be3 1582 Error("CreateDataset", "No valid dataset corresponding to the query!");
1583 return kFALSE;
1584 }
a8739e8a 1585 }
c57f56b7 1586 return kTRUE;
1587}
1588
1589//______________________________________________________________________________
1590Bool_t AliAnalysisAlien::CreateJDL()
1591{
1592// Generate a JDL file according to current settings. The name of the file is
1593// specified by fJDLName.
ec5acdce 1594 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
c57f56b7 1595 Bool_t error = kFALSE;
1596 TObjArray *arr = 0;
1597 Bool_t copy = kTRUE;
d3b18c4c 1598 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1599 Bool_t generate = kTRUE;
1600 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1601 if (!Connect()) {
1602 Error("CreateJDL", "Alien connection required");
1603 return kFALSE;
1604 }
1605 // Check validity of alien workspace
d3b18c4c 1606 TString workdir;
1607 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1608 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1609 workdir += fGridWorkingDir;
1610 if (generate) {
1611 TObjString *os;
1612 if (!fInputFiles) {
1613 Error("CreateJDL()", "Define some input files for your analysis.");
1614 error = kTRUE;
1615 }
1616 // Compose list of input files
1617 // Check if output files were defined
1618 if (!fOutputFiles.Length()) {
1619 Error("CreateJDL", "You must define at least one output file");
1620 error = kTRUE;
1621 }
1622 // Check if an output directory was defined and valid
1623 if (!fGridOutputDir.Length()) {
1624 Error("CreateJDL", "You must define AliEn output directory");
1625 error = kTRUE;
1626 } else {
d3b18c4c 1627 if (!fProductionMode) {
1628 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1629 if (!DirectoryExists(fGridOutputDir)) {
1630 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1631 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1632 } else {
1633 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1634 // error = kTRUE;
1635 }
67a3b254 1636 } else {
1637 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1638 }
1639 gGrid->Cd(workdir);
1640 }
c57f56b7 1641 }
1642 // Exit if any error up to now
1643 if (error) return kFALSE;
1644 // Set JDL fields
0f389141 1645 if (!fUser.IsNull()) {
1646 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1647 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1648 }
1649 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1650 TString mergeExec = fExecutable;
1651 mergeExec.ReplaceAll(".sh", "_merge.sh");
1652 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1653 mergeExec.ReplaceAll(".sh", ".C");
1654 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1655 if (!fArguments.IsNull())
1656 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1657 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1658 else {
1659 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1660 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1661 }
b3e07543 1662
1f0d1ca2 1663 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1664 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1665 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1666 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1667
0f389141 1668 if (fMaxInitFailed > 0) {
c57f56b7 1669 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1670 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1671 }
1672 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1673 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1674 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1675 }
1676 if (!IsOneStageMerging()) {
f47d5cb4 1677 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1678 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1679 }
1680 if (fSplitMode.Length()) {
c57f56b7 1681 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1682 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1683 }
1684 fMergingJDL->SetValue("Split", "\"se\"");
1685 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1686 if (!fAliROOTVersion.IsNull()) {
1687 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1688 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1689 }
1690 if (!fROOTVersion.IsNull()) {
c57f56b7 1691 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1692 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1693 }
1694 if (!fAPIVersion.IsNull()) {
c57f56b7 1695 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1696 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1697 }
648174cf 1698 if (!fExternalPackages.IsNull()) {
1699 arr = fExternalPackages.Tokenize(" ");
1700 TIter next(arr);
1701 while ((os=(TObjString*)next())) {
1702 TString pkgname = os->GetString();
1703 Int_t index = pkgname.Index("::");
1704 TString pkgversion = pkgname(index+2, pkgname.Length());
1705 pkgname.Remove(index);
1706 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1707 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1708 }
1709 delete arr;
1710 }
0f389141 1711 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1712 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1713 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1714 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1715 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1716 TString analysisFile = fExecutable;
1717 analysisFile.ReplaceAll(".sh", ".root");
1718 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1719 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1720 if (fAdditionalLibs.Length()) {
1721 arr = fAdditionalLibs.Tokenize(" ");
1722 TIter next(arr);
1723 while ((os=(TObjString*)next())) {
1724 if (os->GetString().Contains(".so")) continue;
1725 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1726 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1727 }
1728 delete arr;
1729 }
4e5c5506 1730 if (fPackages) {
1731 TIter next(fPackages);
1732 TObject *obj;
0f389141 1733 while ((obj=next())) {
4e5c5506 1734 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1735 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1736 }
4e5c5506 1737 }
95e16f01 1738 const char *comment = "List of output files and archives";
c57f56b7 1739 if (fOutputArchive.Length()) {
e4d9a069 1740 TString outputArchive = fOutputArchive;
1741 if (!fRegisterExcludes.IsNull()) {
1742 arr = fRegisterExcludes.Tokenize(" ");
1743 TIter next1(arr);
1744 while ((os=(TObjString*)next1())) {
1745 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1746 outputArchive.ReplaceAll(os->GetString(),"");
1747 }
1748 delete arr;
1749 }
1750 arr = outputArchive.Tokenize(" ");
c57f56b7 1751 TIter next(arr);
0f389141 1752 Bool_t first = kTRUE;
0f389141 1753 while ((os=(TObjString*)next())) {
0f389141 1754 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1755 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
0f389141 1756 else
95e16f01 1757 fGridJDL->AddToSet("Output", os->GetString());
1758 if (first) fGridJDL->AddToSetDescription("Output", comment);
0f389141 1759 first = kFALSE;
1760 }
c57f56b7 1761 delete arr;
f790bc1b 1762 // Output archive for the merging jdl
f790bc1b 1763 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1764 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1765 // Add normal output files, extra files + terminate files
1766 TString files = GetListOfFiles("outextter");
37d21c01 1767 // Do not register files in fRegisterExcludes
1768 if (!fRegisterExcludes.IsNull()) {
1769 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1770 TIter next1(arr);
1771 while ((os=(TObjString*)next1())) {
1772 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1773 files.ReplaceAll(os->GetString(),"");
1774 }
66d71516 1775 delete arr;
0f389141 1776 }
f790bc1b 1777 files.ReplaceAll(".root", "*.root");
ec5acdce 1778
1779 if (mgr->IsCollectThroughput())
95e16f01 1780 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",files.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 1781 else
1782 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1783 } else {
95f84779 1784 TString files = fOutputArchive;
1785 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1786 outputArchive = files;
f790bc1b 1787 }
0f389141 1788 arr = outputArchive.Tokenize(" ");
705adb3e 1789 TIter next2(arr);
0f389141 1790 first = kTRUE;
705adb3e 1791 while ((os=(TObjString*)next2())) {
0cdf65a8 1792 TString currentfile = os->GetString();
0cdf65a8 1793 if (!currentfile.Contains("@") && fCloseSE.Length())
95e16f01 1794 fMergingJDL->AddToSet("Output", Form("%s@%s",currentfile.Data(), fCloseSE.Data()));
0f389141 1795 else
95e16f01 1796 fMergingJDL->AddToSet("Output", currentfile);
1797 if (first) fMergingJDL->AddToSetDescription("Output", comment);
0f389141 1798 first = kFALSE;
1799 }
1800 delete arr;
c57f56b7 1801 }
149d288c 1802 arr = fOutputFiles.Tokenize(",");
c57f56b7 1803 TIter next(arr);
0f389141 1804 Bool_t first = kTRUE;
43da816a 1805 while ((os=(TObjString*)next())) {
e1eaf596 1806 // Ignore ouputs in jdl that are also in outputarchive
1807 TString sout = os->GetString();
f790bc1b 1808 sout.ReplaceAll("*", "");
1809 sout.ReplaceAll(".root", "");
e1eaf596 1810 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1811 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1812 // Ignore fRegisterExcludes
1813 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1814 if (!first) comment = NULL;
43da816a 1815 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1816 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
43da816a 1817 else
95e16f01 1818 fGridJDL->AddToSet("Output", os->GetString());
1819 if (first) fGridJDL->AddToSetDescription("Output", comment);
f790bc1b 1820 if (fMergeExcludes.Contains(sout)) continue;
1821 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1822 fMergingJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
f790bc1b 1823 else
95e16f01 1824 fMergingJDL->AddToSet("Output", os->GetString());
1825 if (first) fMergingJDL->AddToSetDescription("Output", comment);
1826 first = kFALSE;
43da816a 1827 }
c57f56b7 1828 delete arr;
0f389141 1829 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1830 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1831 TString validationScript = fValidationScript;
0f389141 1832 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1833 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1834 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1835 if (fMasterResubmitThreshold) {
1836 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1837 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1838 }
d2a409b2 1839 // Write a jdl with 2 input parameters: collection name and output dir name.
1840 WriteJDL(copy);
c57f56b7 1841 }
1842 // Copy jdl to grid workspace
a8739e8a 1843 if (copy) {
b5e4aaa7 1844 // Check if an output directory was defined and valid
1845 if (!fGridOutputDir.Length()) {
1846 Error("CreateJDL", "You must define AliEn output directory");
1847 return kFALSE;
1848 } else {
1849 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1850 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1851 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1852 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1853 } else {
1854 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1855 return kFALSE;
1856 }
1857 }
1858 gGrid->Cd(workdir);
1859 }
648174cf 1860 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1861 TString mergeJDLName = fExecutable;
1862 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1863 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1864 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1865 if (fProductionMode) {
648174cf 1866 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1867 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1868 }
648174cf 1869 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1870 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1871 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1872 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1873 Fatal("","Terminating");
1874// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1875 if (fMergeViaJDL) {
1876 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1877// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1878 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1879 Fatal("","Terminating");
0f389141 1880 }
648174cf 1881 }
c57f56b7 1882 if (fAdditionalLibs.Length()) {
e7c71df0 1883 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1884 TObjString *os;
1885 TIter next(arr);
1886 while ((os=(TObjString*)next())) {
c57f56b7 1887 if (os->GetString().Contains(".so")) continue;
4e5c5506 1888 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1889 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1890// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1891 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1892 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1893 Fatal("","Terminating");
c57f56b7 1894 }
1895 delete arr;
1896 }
4e5c5506 1897 if (fPackages) {
1898 TIter next(fPackages);
1899 TObject *obj;
1900 while ((obj=next())) {
fdbbc7be 1901 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1902 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1903// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1904 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1905 Form("%s/%s", workdir.Data(), obj->GetName())))
1906 Fatal("","Terminating");
4e5c5506 1907 }
1908 }
c57f56b7 1909 }
1910 return kTRUE;
1911}
1912
a8739e8a 1913//______________________________________________________________________________
d2a409b2 1914Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1915{
1916// Writes one or more JDL's corresponding to findex. If findex is negative,
1917// all run numbers are considered in one go (jdl). For non-negative indices
1918// they correspond to the indices in the array fInputFiles.
1919 if (!fInputFiles) return kFALSE;
d3b18c4c 1920 TObject *os;
1921 TString workdir;
1922 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1923 workdir += fGridWorkingDir;
f47d5cb4 1924 TString stageName = "$2";
1925 if (fProductionMode) stageName = "$4";
1926 if (!fMergeDirName.IsNull()) {
1927 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1928 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1929 } else {
1930 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1931 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1932 }
d3b18c4c 1933 if (fProductionMode) {
a8739e8a 1934 TIter next(fInputFiles);
b3e07543 1935 while ((os=next())) {
d3b18c4c 1936 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1937 }
56754bf6 1938 if (!fOutputToRunNo)
1939 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1940 else
1941 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1942 } else {
1943 if (!fRunNumbers.Length() && !fRunRange[0]) {
1944 // One jdl with no parameters in case input data is specified by name.
1945 TIter next(fInputFiles);
1946 while ((os=next()))
1947 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1948 if (!fOutputSingle.IsNull())
1949 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1950 else {
1951 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1952 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1953 }
1954 } else {
1955 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1956 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1957 if (!fOutputSingle.IsNull()) {
1958 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1959 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1960 } else {
1961 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1962 }
1963 }
a8739e8a 1964 }
1965
a8739e8a 1966 // Generate the JDL as a string
1967 TString sjdl = fGridJDL->Generate();
0f389141 1968 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1969 // Final merge jdl
1970 if (!fMergeDirName.IsNull()) {
1971 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1972 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1973 } else {
1974 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1975 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1976 }
b3e07543 1977 TString sjdl2 = fMergingJDL->Generate();
1978 Int_t index, index1;
95e16f01 1979 sjdl.ReplaceAll("\",\"", "\",\n \"");
a8739e8a 1980 sjdl.ReplaceAll("(member", "\n (member");
1981 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1982 sjdl.ReplaceAll("{", "{\n ");
1983 sjdl.ReplaceAll("};", "\n};");
1984 sjdl.ReplaceAll("{\n \n", "{\n");
1985 sjdl.ReplaceAll("\n\n", "\n");
1986 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 1987 sjdl1.ReplaceAll("\",\"", "\",\n \"");
0f389141 1988 sjdl1.ReplaceAll("(member", "\n (member");
1989 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1990 sjdl1.ReplaceAll("{", "{\n ");
1991 sjdl1.ReplaceAll("};", "\n};");
1992 sjdl1.ReplaceAll("{\n \n", "{\n");
1993 sjdl1.ReplaceAll("\n\n", "\n");
1994 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 1995 sjdl2.ReplaceAll("\",\"", "\",\n \"");
b3e07543 1996 sjdl2.ReplaceAll("(member", "\n (member");
1997 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1998 sjdl2.ReplaceAll("{", "{\n ");
1999 sjdl2.ReplaceAll("};", "\n};");
2000 sjdl2.ReplaceAll("{\n \n", "{\n");
2001 sjdl2.ReplaceAll("\n\n", "\n");
2002 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 2003 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 2004 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 2005 index = sjdl.Index("JDLVariables");
2006 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 2007 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2008 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 2009 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 2010 index = fJobTag.Index(":");
2011 if (index < 0) index = fJobTag.Length();
2012 TString jobTag = fJobTag;
f47d5cb4 2013 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 2014 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 2015 if (fProductionMode) {
2016 sjdl1.Prepend("# Generated merging jdl (production mode) \
2017 \n# $1 = full alien path to output directory to be merged \
2018 \n# $2 = train number \
2019 \n# $3 = production (like LHC10b) \
2020 \n# $4 = merging stage \
2021 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2022 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2023 sjdl2.Prepend("# Generated merging jdl \
2024 \n# $1 = full alien path to output directory to be merged \
2025 \n# $2 = train number \
2026 \n# $3 = production (like LHC10b) \
2027 \n# $4 = merging stage \
2028 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2029 } else {
2030 sjdl1.Prepend("# Generated merging jdl \
2031 \n# $1 = full alien path to output directory to be merged \
2032 \n# $2 = merging stage \
03335324 2033 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2034 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2035 sjdl2.Prepend("# Generated merging jdl \
2036 \n# $1 = full alien path to output directory to be merged \
2037 \n# $2 = merging stage \
03335324 2038 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2039 }
0f389141 2040 index = sjdl1.Index("JDLVariables");
2041 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 2042 index = sjdl2.Index("JDLVariables");
2043 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 2044 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2045 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
2046 index = sjdl2.Index("Split =");
2047 if (index>=0) {
2048 index1 = sjdl2.Index("\n", index);
2049 sjdl2.Remove(index, index1-index+1);
2050 }
2051 index = sjdl2.Index("SplitMaxInputFileNumber");
2052 if (index>=0) {
2053 index1 = sjdl2.Index("\n", index);
2054 sjdl2.Remove(index, index1-index+1);
2055 }
2056 index = sjdl2.Index("InputDataCollection");
2057 if (index>=0) {
2058 index1 = sjdl2.Index(";", index);
2059 sjdl2.Remove(index, index1-index+1);
2060 }
2061 index = sjdl2.Index("InputDataListFormat");
2062 if (index>=0) {
2063 index1 = sjdl2.Index("\n", index);
2064 sjdl2.Remove(index, index1-index+1);
2065 }
2066 index = sjdl2.Index("InputDataList");
2067 if (index>=0) {
2068 index1 = sjdl2.Index("\n", index);
2069 sjdl2.Remove(index, index1-index+1);
2070 }
f47d5cb4 2071 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2072 // Write jdl to file
a8739e8a 2073 ofstream out;
d2a409b2 2074 out.open(fJDLName.Data(), ios::out);
a8739e8a 2075 if (out.bad()) {
d3b18c4c 2076 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2077 return kFALSE;
2078 }
2079 out << sjdl << endl;
b3e07543 2080 out.close();
0f389141 2081 TString mergeJDLName = fExecutable;
2082 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2083 if (fMergeViaJDL) {
2084 ofstream out1;
2085 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2086 if (out1.bad()) {
d3b18c4c 2087 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2088 return kFALSE;
2089 }
2090 out1 << sjdl1 << endl;
b3e07543 2091 out1.close();
2092 ofstream out2;
2093 TString finalJDL = mergeJDLName;
2094 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2095 out2.open(finalJDL.Data(), ios::out);
2096 if (out2.bad()) {
2097 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2098 return kFALSE;
2099 }
2100 out2 << sjdl2 << endl;
2101 out2.close();
0f389141 2102 }
a8739e8a 2103
2104 // Copy jdl to grid workspace
2105 if (!copy) {
d3b18c4c 2106 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2107 } else {
d2a409b2 2108 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2109 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2110 TString finalJDL = mergeJDLName;
2111 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2112 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2113 if (fProductionMode) {
b5fe9cba 2114 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2115 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2116 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2117 }
d2a409b2 2118 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2119 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2120 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2121 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2122// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2123 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2124 Fatal("","Terminating");
0f389141 2125 if (fMergeViaJDL) {
b3e07543 2126 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2127// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2128// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2129 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2130 Fatal("","Terminating");
2131 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2132 Fatal("","Terminating");
0f389141 2133 }
a8739e8a 2134 }
2135 return kTRUE;
2136}
2137
c57f56b7 2138//______________________________________________________________________________
5513444a 2139Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2140{
2141// Returns true if file exists.
5513444a 2142 if (!gGrid) return kFALSE;
2ea55496 2143 TString slfn = lfn;
2144 slfn.ReplaceAll("alien://","");
2145 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2146 if (!res) return kFALSE;
2147 TMap *map = dynamic_cast<TMap*>(res->At(0));
2148 if (!map) {
2149 delete res;
2150 return kFALSE;
2151 }
2152 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2153 if (!objs || !objs->GetString().Length()) {
2154 delete res;
2155 return kFALSE;
2156 }
2157 delete res;
2158 return kTRUE;
2159}
2160
923e2ca5 2161//______________________________________________________________________________
2162Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2163{
2164// Returns true if directory exists. Can be also a path.
2165 if (!gGrid) return kFALSE;
2166 // Check if dirname is a path
2167 TString dirstripped = dirname;
2168 dirstripped = dirstripped.Strip();
2169 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2170 TString dir = gSystem->BaseName(dirstripped);
2171 dir += "/";
2172 TString path = gSystem->DirName(dirstripped);
2173 TGridResult *res = gGrid->Ls(path, "-F");
2174 if (!res) return kFALSE;
2175 TIter next(res);
2176 TMap *map;
2177 TObject *obj;
2178 while ((map=dynamic_cast<TMap*>(next()))) {
2179 obj = map->GetValue("name");
2180 if (!obj) break;
2181 if (dir == obj->GetName()) {
2182 delete res;
2183 return kTRUE;
2184 }
2185 }
2186 delete res;
2187 return kFALSE;
2188}
2189
c57f56b7 2190//______________________________________________________________________________
a2f5fc01 2191void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2192{
2193// Check input data type.
a2f5fc01 2194 isCollection = kFALSE;
2195 isXml = kFALSE;
2196 useTags = kFALSE;
c57f56b7 2197 if (!gGrid) {
2198 Error("CheckDataType", "No connection to grid");
2199 return;
2200 }
a2f5fc01 2201 isCollection = IsCollection(lfn);
c57f56b7 2202 TString msg = "\n##### file: ";
2203 msg += lfn;
a2f5fc01 2204 if (isCollection) {
c57f56b7 2205 msg += " type: raw_collection;";
2206 // special treatment for collections
a2f5fc01 2207 isXml = kFALSE;
c57f56b7 2208 // check for tag files in the collection
2209 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2210 if (!res) {
2211 msg += " using_tags: No (unknown)";
23329835 2212 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2213 return;
2214 }
2215 const char* typeStr = res->GetKey(0, "origLFN");
2216 if (!typeStr || !strlen(typeStr)) {
2217 msg += " using_tags: No (unknown)";
23329835 2218 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2219 return;
2220 }
2221 TString file = typeStr;
a2f5fc01 2222 useTags = file.Contains(".tag");
2223 if (useTags) msg += " using_tags: Yes";
c57f56b7 2224 else msg += " using_tags: No";
23329835 2225 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2226 return;
2227 }
2228 TString slfn(lfn);
2229 slfn.ToLower();
a2f5fc01 2230 isXml = slfn.Contains(".xml");
2231 if (isXml) {
c57f56b7 2232 // Open xml collection and check if there are tag files inside
2233 msg += " type: xml_collection;";
2234 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2235 if (!coll) {
2236 msg += " using_tags: No (unknown)";
23329835 2237 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2238 return;
2239 }
2240 TMap *map = coll->Next();
2241 if (!map) {
2242 msg += " using_tags: No (unknown)";
23329835 2243 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2244 return;
2245 }
2246 map = (TMap*)map->GetValue("");
2247 TString file;
2248 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2249 useTags = file.Contains(".tag");
c57f56b7 2250 delete coll;
a2f5fc01 2251 if (useTags) msg += " using_tags: Yes";
c57f56b7 2252 else msg += " using_tags: No";
23329835 2253 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2254 return;
2255 }
a2f5fc01 2256 useTags = slfn.Contains(".tag");
c57f56b7 2257 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2258 else msg += " type: unknown file;";
a2f5fc01 2259 if (useTags) msg += " using_tags: Yes";
c57f56b7 2260 else msg += " using_tags: No";
23329835 2261 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2262}
2263
4e5c5506 2264//______________________________________________________________________________
2265void AliAnalysisAlien::EnablePackage(const char *package)
2266{
2267// Enables a par file supposed to exist in the current directory.
2268 TString pkg(package);
2269 pkg.ReplaceAll(".par", "");
2270 pkg += ".par";
2271 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2272 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2273 return;
2274 }
2275 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2276 Info("EnablePackage", "AliEn plugin will use .par packages");
2277 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2278 if (!fPackages) {
2279 fPackages = new TObjArray();
2280 fPackages->SetOwner();
2281 }
2282 fPackages->Add(new TObjString(pkg));
2283}
2284
3bdcb562 2285//______________________________________________________________________________
2286TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2287{
2288// Make a tree from files having the location specified in fFileForTestMode.
2289// Inspired from JF's CreateESDChain.
2290 if (fFileForTestMode.IsNull()) {
2291 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2292 return NULL;
2293 }
2294 if (gSystem->AccessPathName(fFileForTestMode)) {
2295 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2296 return NULL;
2297 }
2298 // Open the file
2299 ifstream in;
2300 in.open(fFileForTestMode);
2301 Int_t count = 0;
2302 // Read the input list of files and add them to the chain
b505f1f8 2303 TString line;
d6c60197 2304 TString streeName(treeName);
2305 if (IsUseMCchain()) streeName = "TE";
2306 TChain *chain = new TChain(streeName);
b505f1f8 2307 TChain *chainFriend = 0;
d6c60197 2308 if (!fFriendChainName.IsNull()) chainFriend = new TChain(streeName);
b505f1f8 2309 while (in.good())
2310 {
3bdcb562 2311 in >> line;
ffbe06b9 2312 if (line.IsNull() || line.BeginsWith("#")) continue;
3bdcb562 2313 if (count++ == fNtestFiles) break;
2314 TString esdFile(line);
2315 TFile *file = TFile::Open(esdFile);
b505f1f8 2316 if (file && !file->IsZombie()) {
2317 chain->Add(esdFile);
3bdcb562 2318 file->Close();
b505f1f8 2319 if (!fFriendChainName.IsNull()) {
4b5d9e21 2320 if (esdFile.Index("#") > -1)
2321 esdFile.Remove(esdFile.Index("#"));
2322 esdFile = gSystem->DirName(esdFile);
2323 esdFile += "/" + fFriendChainName;
b505f1f8 2324 file = TFile::Open(esdFile);
2325 if (file && !file->IsZombie()) {
2326 file->Close();
2327 chainFriend->Add(esdFile);
2328 } else {
2329 Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data());
2330 return 0;
2331 }
4b5d9e21 2332 }
3bdcb562 2333 } else {
2334 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2335 }
b505f1f8 2336 }
2337 in.close();
2338 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2339 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2340 delete chain;
b505f1f8 2341 delete chainFriend;
3bdcb562 2342 return NULL;
b505f1f8 2343 }
3bdcb562 2344// chain->ls();
b505f1f8 2345 if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend);
2346 return chain;
3bdcb562 2347}
2348
d2a409b2 2349//______________________________________________________________________________
2350const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2351{
2352// Get job status for all jobs with jobid>jobidstart.
2353 static char mstatus[20];
2354 mstatus[0] = '\0';
2355 nrunning = 0;
2356 nwaiting = 0;
2357 nerror = 0;
2358 ndone = 0;
2359 TGridJobStatusList *list = gGrid->Ps("");
2360 if (!list) return mstatus;
2361 Int_t nentries = list->GetSize();
2362 TGridJobStatus *status;
2363 Int_t pid;
2364 for (Int_t ijob=0; ijob<nentries; ijob++) {
2365 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2366 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2367 if (pid<jobidstart) continue;
2368 if (pid == lastid) {
8ddad121 2369 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2370 }
2371 switch (status->GetStatus()) {
2372 case TGridJobStatus::kWAITING:
2373 nwaiting++; break;
2374 case TGridJobStatus::kRUNNING:
2375 nrunning++; break;
2376 case TGridJobStatus::kABORTED:
2377 case TGridJobStatus::kFAIL:
2378 case TGridJobStatus::kUNKNOWN:
2379 nerror++; break;
2380 case TGridJobStatus::kDONE:
2381 ndone++;
2382 }
2383 }
2384 list->Delete();
2385 delete list;
2386 return mstatus;
2387}
2388
c57f56b7 2389//______________________________________________________________________________
2390Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2391{
2392// Returns true if file is a collection. Functionality duplicated from
2393// TAlien::Type() because we don't want to directly depend on TAlien.
2394 if (!gGrid) {
2395 Error("IsCollection", "No connection to grid");
2396 return kFALSE;
2397 }
2398 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2399 if (!res) return kFALSE;
2400 const char* typeStr = res->GetKey(0, "type");
2401 if (!typeStr || !strlen(typeStr)) return kFALSE;
2402 if (!strcmp(typeStr, "collection")) return kTRUE;
2403 delete res;
2404 return kFALSE;
2405}
2406
fe2d7fc2 2407//______________________________________________________________________________
2408Bool_t AliAnalysisAlien::IsSingleOutput() const
2409{
2410// Check if single-ouput option is on.
2411 return (!fOutputSingle.IsNull());
2412}
2413
16a4353c 2414//______________________________________________________________________________
2415void AliAnalysisAlien::Print(Option_t *) const
2416{
2417// Print current plugin settings.
84fcd93f 2418 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2419 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2420 if (mgr && mgr->IsProofMode()) {
2421 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2422 if (TestBit(AliAnalysisGrid::kTest))
2423 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2424 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2425 if (!fProofDataSet.IsNull())
2426 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2427 if (fProofReset==1)
2428 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2429 if (fProofReset>1)
2430 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
874de7ea 2431 if (!fROOTVersion.IsNull())
2432 printf("= ROOT version requested________________________ %s\n", fROOTVersion.Data());
d3b18c4c 2433 else
2434 printf("= ROOT version requested________________________ default\n");
2435 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2436 if (!fAliRootMode.IsNull())
2437 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2438 if (fNproofWorkers)
2439 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2440 if (fNproofWorkersPerSlave)
2441 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2442 if (TestSpecialBit(kClearPackages))
2443 printf("= ClearPackages requested...\n");
2444 if (fIncludePath.Data())
2445 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2446 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2447 if (fPackages && fPackages->GetEntries()) {
2448 TIter next(fPackages);
2449 TObject *obj;
2450 TString list;
2451 while ((obj=next())) list += obj->GetName();
2452 printf("= Par files to be used: ________________________ %s\n", list.Data());
2453 }
2454 if (TestSpecialBit(kProofConnectGrid))
2455 printf("= Requested PROOF connection to grid\n");
2456 return;
2457 }
e1c22e21 2458 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2459 if (fOverwriteMode) {
2460 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2461 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2462 }
348be253 2463 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2464 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2465 printf("= Production mode:______________________________ %d\n", fProductionMode);
2466 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2467 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2468 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2469 if (fUser.Length())
84fcd93f 2470 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2471 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2472 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2473 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2474 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2475 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2476 if (fRunNumbers.Length())
84fcd93f 2477 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2478 if (fRunRange[0])
74013535 2479 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2480 if (!fRunRange[0] && !fRunNumbers.Length()) {
2481 TIter next(fInputFiles);
2482 TObject *obj;
2483 TString list;
2484 while ((obj=next())) list += obj->GetName();
84fcd93f 2485 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2486 }
2487 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2488 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2489 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2490 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2491 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2492 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2493 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2494 printf("=====================================================================\n");
2495 printf("= Job price: ___________________________________ %d\n", fPrice);
2496 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2497 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2498 if (fMaxInitFailed>0)
84fcd93f 2499 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2500 if (fMasterResubmitThreshold>0)
84fcd93f 2501 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2502 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2503 if (fNrunsPerMaster>0)
84fcd93f 2504 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2505 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2506 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2507 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2508 if (fArguments.Length())
84fcd93f 2509 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2510 if (fExecutableArgs.Length())
2511 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2512 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2513 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2514 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2515 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2516 if (fDatasetName)
84fcd93f 2517 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2518 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2519 if (fIncludePath.Data())
84fcd93f 2520 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2521 if (fCloseSE.Length())
84fcd93f 2522 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2523 if (fFriendChainName.Length())
84fcd93f 2524 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2525 if (fPackages && fPackages->GetEntries()) {
16a4353c 2526 TIter next(fPackages);
2527 TObject *obj;
2528 TString list;
2529 while ((obj=next())) list += obj->GetName();
84fcd93f 2530 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2531 }
2532}
2533
c57f56b7 2534//______________________________________________________________________________
2535void AliAnalysisAlien::SetDefaults()
2536{
2537// Set default values for everything. What cannot be filled will be left empty.
2538 if (fGridJDL) delete fGridJDL;
2539 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2540 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2541 fPrice = 1;
2542 fTTL = 30000;
2543 fSplitMaxInputFileNumber = 100;
2544 fMaxInitFailed = 0;
2545 fMasterResubmitThreshold = 0;
bb885a9e 2546 fNtestFiles = 10;
149d288c 2547 fNreplicas = 2;
a8739e8a 2548 fRunRange[0] = 0;
2549 fRunRange[1] = 0;
8e3bd06a 2550 fRunPrefix = "%d";
319593fb 2551 fNrunsPerMaster = 1;
16a4353c 2552 fMaxMergeFiles = 100;
c57f56b7 2553 fRunNumbers = "";
2554 fExecutable = "analysis.sh";
5aa94dc3 2555 fExecutableCommand = "root -b -q -x";
c57f56b7 2556 fArguments = "";
631c0b05 2557 fExecutableArgs = "";
c57f56b7 2558 fAnalysisMacro = "myAnalysis.C";
2559 fAnalysisSource = "";
2560 fAdditionalLibs = "";
2561 fSplitMode = "se";
2562 fAPIVersion = "";
2563 fROOTVersion = "";
2564 fAliROOTVersion = "";
2565 fUser = ""; // Your alien user name
2566 fGridWorkingDir = "";
2567 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2568 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2569 fFriendChainName = "";
c57f56b7 2570 fGridOutputDir = "output";
b83f84f0 2571 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2572 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2573 fInputFormat = "xml-single";
2574 fJDLName = "analysis.jdl";
c6cb3634 2575 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2576 fMergeExcludes = "";
0f389141 2577 fMergeViaJDL = 0;
348be253 2578 SetUseCopy(kTRUE);
2579 SetCheckCopy(kTRUE);
149d288c 2580 SetDefaultOutputs(kTRUE);
e1c22e21 2581 fOverwriteMode = 1;
c57f56b7 2582}
2583
a41b2882 2584//______________________________________________________________________________
2585void AliAnalysisAlien::SetFriendChainName(const char *name, const char *libnames)
2586{
2587 // Set file name for the chain of friends and optionally additional libs to be loaded.
2588 // Libs should be separated by blancs.
2589 fFriendChainName = name;
2590 fFriendLibs = libnames;
1a3f255c 2591 if (fFriendLibs.Length()) {
2592 if(!fFriendLibs.Contains(".so"))
2593 Fatal("SetFriendChainName()", "You should provide explicit library names (with extension)");
2594 fFriendLibs.ReplaceAll(",", " ");
2595 fFriendLibs.Strip();
2596 fFriendLibs.ReplaceAll(" ", " ");
a41b2882 2597 }
2598}
2599
874de7ea 2600//______________________________________________________________________________
2601void AliAnalysisAlien::SetRootVersionForProof(const char *version)
2602{
2603// Obsolete method. Use SetROOTVersion instead
2604 Warning("SetRootVersionForProof", "Obsolete. Use SetROOTVersion instead");
95e16f01 2605 if (fROOTVersion.IsNull()) SetROOTVersion(version);
2606 else Error("SetRootVersionForProof", "ROOT version already set to %s", fROOTVersion.Data());
874de7ea 2607}
2608
0f389141 2609//______________________________________________________________________________
b3e07543 2610Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2611{
b3e07543 2612// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2613 // First check if the result is already in the output directory.
2614 if (FileExists(Form("%s/%s",aliendir,filename))) {
2615 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2616 return kFALSE;
2617 }
b3e07543 2618 // Now check the last stage done.
2619 Int_t stage = 0;
2620 while (1) {
2621 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2622 stage++;
2623 }
2624 // Next stage of merging
2625 stage++;
2626 TString pattern = "*root_archive.zip";
2627 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2628 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2629 if (res) delete res;
2630 // Write standard output to file
2631 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2632 // Count the number of files inside
2633 ifstream ifile;
2634 ifile.open(Form("Stage_%d.xml",stage));
2635 if (!ifile.good()) {
2636 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2637 return kFALSE;
b3e07543 2638 }
2639 TString line;
2640 Int_t nfiles = 0;
2641 while (!ifile.eof()) {
2642 ifile >> line;
2643 if (line.Contains("/event")) nfiles++;
7c2cd90a 2644 }
b3e07543 2645 ifile.close();
2646 if (!nfiles) {
2647 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2648 return kFALSE;
2649 } else {
2650 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2651 }
b3e07543 2652 // Copy the file in the output directory
2653 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2654// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2655 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2656 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2657 // Check if this is the last stage to be done.
2658 Bool_t laststage = (nfiles<nperchunk);
2659 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
0ad3ea1f 2660 Int_t jobId = 0;
b3e07543 2661 if (laststage) {
2662 printf("### Submiting final merging stage %d\n", stage);
2663 TString finalJDL = jdl;
2664 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2665 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
0ad3ea1f 2666 jobId = SubmitSingleJob(query);
b3e07543 2667 } else {
2668 printf("### Submiting merging stage %d\n", stage);
03335324 2669 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
0ad3ea1f 2670 jobId = SubmitSingleJob(query);
7c2cd90a 2671 }
0ad3ea1f 2672 if (!jobId) return kFALSE;
2673
2674 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
2675 fGridJobIDs.Append(Form("%d", jobId));
2676 if (!fGridStages.IsNull()) fGridStages.Append(" ");
2677 fGridStages.Append(Form("%s_merge_stage%d",
2678 laststage ? "final" : "partial", stage));
2679
b3e07543 2680 return kTRUE;
2681}
7c2cd90a 2682
105631b5 2683//______________________________________________________________________________
2684AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2685{
2686// Loat the analysis manager from a file.
2687 TFile *file = TFile::Open(fname);
2688 if (!file) {
2689 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2690 return 0;
2691 }
2692 TIter nextkey(file->GetListOfKeys());
2693 AliAnalysisManager *mgr = 0;
2694 TKey *key;
2695 while ((key=(TKey*)nextkey())) {
2696 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2697 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2698 }
2699 if (!mgr)
2700 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2701 return mgr;
2702}
2703
7c2cd90a 2704//______________________________________________________________________________
2705Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2706{
2707// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2708 if (!gGrid) return 0;
2709 printf("=> %s ------> ",query);
2710 TGridResult *res = gGrid->Command(query);
2711 if (!res) return 0;
2712 TString jobId = res->GetKey(0,"jobId");
2713 delete res;
2714 if (jobId.IsNull()) {
2715 printf("submission failed. Reason:\n");
2716 gGrid->Stdout();
2717 gGrid->Stderr();
2718 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2719 return 0;
2720 }
0ad3ea1f 2721 Int_t ijobId = jobId.Atoi();
2722 printf(" Job id: '%s' (%d)\n", jobId.Data(), ijobId);
2723 return ijobId;
7c2cd90a 2724}
2725
ec5acdce 2726//______________________________________________________________________________
2727Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2728{
2729// Merges a collection of output files using concatenation.
d66aa2a9 2730 TString scoll(collection);
2731 if (!scoll.Contains(".xml")) return kFALSE;
ec5acdce 2732 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2733 if (!coll) {
2734 ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2735 return kFALSE;
2736 }
2737 // Iterate grid collection
2738 TString outtmp;
2739 Bool_t merged = kFALSE;
2740 Int_t ifile = 0;
2741 while (coll->Next()) {
2742 TString fname = gSystem->DirName(coll->GetTURL());
2743 fname += "/";
2744 fname += output;
2745 outtmp = Form("%d_%s", ifile, output);
2746 if (!TFile::Cp(fname, outtmp)) {
2747 ::Error("MergeInfo", "Could not copy %s", fname.Data());
2748 continue;
2749 }
2750 ifile++;
2751 if (ifile<2) {
2752 gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2753 continue;
2754 }
2755 gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2756 gSystem->Exec("cp tempmerged lastmerged");
2757 }
2758 if (ifile) {
2759 gSystem->Exec(Form("cp lastmerged %s", output));
2760 gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2761 merged = kTRUE;
2762 }
2763 return merged;
2764}
2765
7c2cd90a 2766//______________________________________________________________________________
b3e07543 2767Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2768{
b3e07543 2769// Merge given output files from basedir. Basedir can be an alien output directory
2770// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2771// files in a group (ignored for xml input). Merging can be done in stages:
2772// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2773// stage=1 : works with an xml of all root_archive.zip in the output directory
2774// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2775 TString outputFile = output;
0f389141 2776 TString command;
a2f5fc01 2777 TString outputChunk;
2778 TString previousChunk = "";
b3e07543 2779 TObjArray *listoffiles = new TObjArray();
2780// listoffiles->SetOwner();
a2f5fc01 2781 Int_t countChunk = 0;
2782 Int_t countZero = nmaxmerge;
0f389141 2783 Bool_t merged = kTRUE;
d66aa2a9 2784 Bool_t isGrid = kTRUE;
a2f5fc01 2785 Int_t index = outputFile.Index("@");
2786 if (index > 0) outputFile.Remove(index);
7c2cd90a 2787 TString inputFile = outputFile;
b3e07543 2788 TString sbasedir = basedir;
2789 if (sbasedir.Contains(".xml")) {
2790 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2791 nmaxmerge = 9999999;
2792 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2793 if (!coll) {
2794 ::Error("MergeOutput", "Input XML collection empty.");
2795 return kFALSE;
2796 }
2797 // Iterate grid collection
2798 while (coll->Next()) {
2799 TString fname = gSystem->DirName(coll->GetTURL());
2800 fname += "/";
2801 fname += inputFile;
2802 listoffiles->Add(new TNamed(fname.Data(),""));
2803 }
ffbe06b9 2804 } else if (sbasedir.Contains(".txt")) {
d66aa2a9 2805 // The file having the .txt extension is expected to contain a list of
2806 // folders where the output files will be looked. For alien folders,
2807 // the full folder LFN is expected (starting with alien://)
ffbe06b9 2808 // Assume lfn's on each line
2809 TString line;
2810 ifstream in;
2811 in.open(sbasedir);
d66aa2a9 2812 if (in.fail()) {
2813 ::Error("MergeOutput", "File %s cannot be opened. Merging stopped." ,sbasedir.Data());
2814 return kTRUE;
2815 }
ffbe06b9 2816 Int_t nfiles = 0;
2817 while (in.good()) {
2818 in >> line;
2819 if (line.IsNull() || line.BeginsWith("#")) continue;
d66aa2a9 2820 line.Strip();
2821 if (!line.Contains("alien:")) isGrid = kFALSE;
2822 line += "/";
2823 line += outputFile;
ffbe06b9 2824 nfiles++;
2825 listoffiles->Add(new TNamed(line.Data(),""));
2826 }
d66aa2a9 2827 in.close();
ffbe06b9 2828 if (!nfiles) {
2829 ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data());
2830 delete listoffiles;
2831 return kFALSE;
2832 }
b3e07543 2833 } else {
2834 command = Form("find %s/ *%s", basedir, inputFile.Data());
2835 printf("command: %s\n", command.Data());
2836 TGridResult *res = gGrid->Command(command);
2837 if (!res) {
2838 ::Error("MergeOutput","No result for the find command\n");
2839 delete listoffiles;
2840 return kFALSE;
2841 }
2842 TIter nextmap(res);
2843 TMap *map = 0;
2844 while ((map=(TMap*)nextmap())) {
2845 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2846 if (!objs || !objs->GetString().Length()) {
2847 // Nothing found - skip this output
2848 delete res;
2849 delete listoffiles;
2850 return kFALSE;
2851 }
2852 listoffiles->Add(new TNamed(objs->GetName(),""));
2853 }
2854 delete res;
2855 }
2856 if (!listoffiles->GetEntries()) {
7c2cd90a 2857 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2858 delete listoffiles;
0f389141 2859 return kFALSE;
2860 }
2861
2862 TFileMerger *fm = 0;
b3e07543 2863 TIter next0(listoffiles);
2864 TObjArray *listoffilestmp = new TObjArray();
2865 listoffilestmp->SetOwner();
2866 TObject *nextfile;
2867 TString snextfile;
2868 // Keep only the files at upper level
2869 Int_t countChar = 0;
2870 while ((nextfile=next0())) {
2871 snextfile = nextfile->GetName();
2872 Int_t crtCount = snextfile.CountChar('/');
2873 if (nextfile == listoffiles->First()) countChar = crtCount;
2874 if (crtCount < countChar) countChar = crtCount;
2875 }
2876 next0.Reset();
2877 while ((nextfile=next0())) {
2878 snextfile = nextfile->GetName();
2879 Int_t crtCount = snextfile.CountChar('/');
2880 if (crtCount > countChar) {
2881 delete nextfile;
2882 continue;
2883 }
2884 listoffilestmp->Add(nextfile);
2885 }
2886 delete listoffiles;
2887 listoffiles = listoffilestmp; // Now contains 'good' files
2888 listoffiles->Print();
2889 TIter next(listoffiles);
7c2cd90a 2890 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2891 outputChunk = outputFile;
2892 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2893 // Check for existent temporary merge files
e1c22e21 2894 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2895 // Preserve old merging functionality for stage 0.
2896 if (stage==0) {
2897 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2898 while (1) {
2899 // Skip as many input files as in a chunk
2ea55496 2900 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2901 nextfile = next();
2902 if (!nextfile) {
2ea55496 2903 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2904 delete listoffiles;
2ea55496 2905 return kFALSE;
2906 }
b3e07543 2907 snextfile = nextfile->GetName();
7c2cd90a 2908 }
2909 outputChunk = outputFile;
2910 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2911 countChunk++;
2912 if (gSystem->AccessPathName(outputChunk)) continue;
2913 // Merged file with chunks up to <countChunk> found
2914 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2915 previousChunk = outputChunk;
2916 break;
2917 }
2918 }
2919 countZero = nmaxmerge;
2920
b3e07543 2921 while ((nextfile=next())) {
2922 snextfile = nextfile->GetName();
2ea55496 2923 // Loop 'find' results and get next LFN
7c2cd90a 2924 if (countZero == nmaxmerge) {
2925 // First file in chunk - create file merger and add previous chunk if any.
d66aa2a9 2926 fm = new TFileMerger(isGrid);
7c2cd90a 2927 fm->SetFastMethod(kTRUE);
2928 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2929 outputChunk = outputFile;
2930 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2931 }
2932 // If last file found, put merged results in the output file
b3e07543 2933 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2934 // Add file to be merged and decrement chunk counter.
b3e07543 2935 fm->AddFile(snextfile);
7c2cd90a 2936 countZero--;
b3e07543 2937 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2938 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2939 // Nothing found - skip this output
2940 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2941 merged = kFALSE;
2942 break;
7c2cd90a 2943 }
2944 fm->OutputFile(outputChunk);
2945 // Merge the outputs, then go to next chunk
2946 if (!fm->Merge()) {
2947 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2948 merged = kFALSE;
2949 break;
7c2cd90a 2950 } else {
2951 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2952 gSystem->Unlink(previousChunk);
2953 }
b3e07543 2954 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2955 countChunk++;
2956 countZero = nmaxmerge;
2957 previousChunk = outputChunk;
0f389141 2958 }
0f389141 2959 }
b3e07543 2960 delete listoffiles;
2961 delete fm;
7c2cd90a 2962 return merged;
2963 }
2964 // Merging stage different than 0.
2965 // Move to the begining of the requested chunk.
d66aa2a9 2966 fm = new TFileMerger(isGrid);
7c2cd90a 2967 fm->SetFastMethod(kTRUE);
b3e07543 2968 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2969 delete listoffiles;
7c2cd90a 2970 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2971 // Nothing found - skip this output
2972 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2973 delete fm;
2974 return kFALSE;
2975 }
b3e07543 2976 fm->OutputFile(outputFile);
7c2cd90a 2977 // Merge the outputs
2978 if (!fm->Merge()) {
2979 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2980 delete fm;
2981 return kFALSE;
2982 } else {
b3e07543 2983 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2984 }
7c2cd90a 2985 delete fm;
2986 return kTRUE;
0f389141 2987}
2988
c57f56b7 2989//______________________________________________________________________________
2990Bool_t AliAnalysisAlien::MergeOutputs()
2991{
2992// Merge analysis outputs existing in the AliEn space.
2993 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2994 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2995 if (!Connect()) {
2996 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2997 return kFALSE;
0f389141 2998 }
c9e8f7fd 2999 if (fMergeViaJDL) {
3000 if (!TestBit(AliAnalysisGrid::kMerge)) {
3001 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
3002 return kFALSE;
3003 }
3004 if (fProductionMode) {
3005 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
3006 return kFALSE;
3007 }
0f389141 3008 Info("MergeOutputs", "Submitting merging JDL");
a03be957 3009 if (!SubmitMerging()) return kFALSE;
0f389141 3010 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 3011 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 3012 return kFALSE;
c57f56b7 3013 }
3014 // Get the output path
dc030a24 3015 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 3016 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 3017 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
3018 return kFALSE;
3019 }
3020 if (!fOutputFiles.Length()) {
3021 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3022 return kFALSE;
0f389141 3023 }
3024 // Check if fast read option was requested
e1c22e21 3025 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
3026 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 3027 if (fFastReadOption) {
d3339be3 3028 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
3029 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 3030 gEnv->SetValue("XNet.ConnectTimeout",50);
3031 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 3032 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 3033 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 3034 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 3035 }
e8b839ab 3036 // Make sure we change the temporary directory
3037 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 3038 // Set temporary compilation directory to current one
3039 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 3040 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3041 TIter next(list);
3042 TObjString *str;
a2f5fc01 3043 TString outputFile;
c57f56b7 3044 Bool_t merged = kTRUE;
3045 while((str=(TObjString*)next())) {
a2f5fc01 3046 outputFile = str->GetString();
3047 Int_t index = outputFile.Index("@");
3048 if (index > 0) outputFile.Remove(index);
3049 TString outputChunk = outputFile;
3050 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 3051 // Skip already merged outputs
a2f5fc01 3052 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 3053 if (fOverwriteMode) {
a2f5fc01 3054 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
3055 gSystem->Unlink(outputFile);
3056 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3057 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3058 outputChunk.Data());
3059 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3060 }
3061 } else {
a2f5fc01 3062 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 3063 continue;
3064 }
3065 } else {
a2f5fc01 3066 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3067 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3068 outputChunk.Data());
3069 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3070 }
3071 }
37d21c01 3072 if (fMergeExcludes.Contains(outputFile.Data()) ||
3073 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 3074 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 3075 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 3076 if (!merged) {
3077 Error("MergeOutputs", "Terminate() will NOT be executed");
09d5920f 3078 delete list;
3079 return kFALSE;
ff07ec61 3080 }
3081 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
3082 if (fileOpened) fileOpened->Close();
c57f56b7 3083 }
09d5920f 3084 delete list;
0f389141 3085 return kTRUE;
c57f56b7 3086}
3087
bb885a9e 3088//______________________________________________________________________________
3089void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
3090{
3091// Use the output files connected to output containers from the analysis manager
3092// rather than the files defined by SetOutputFiles
3093 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 3094 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 3095 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
3096}
3097
149d288c 3098//______________________________________________________________________________
3099void AliAnalysisAlien::SetOutputFiles(const char *list)
3100{
3101// Manually set the output files list.
3102// Removes duplicates. Not allowed if default outputs are not disabled.
3103 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3104 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
3105 return;
3106 }
3107 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
3108 fOutputFiles = "";
3109 TString slist = list;
3110 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
3111 TObjArray *arr = slist.Tokenize(" ");
3112 TObjString *os;
3113 TIter next(arr);
3114 TString sout;
3115 while ((os=(TObjString*)next())) {
3116 sout = os->GetString();
3117 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
3118 if (fOutputFiles.Contains(sout)) continue;
3119 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
3120 fOutputFiles += sout;
3121 }
3122 delete arr;
f790bc1b 3123}
149d288c 3124
3125//______________________________________________________________________________
3126void AliAnalysisAlien::SetOutputArchive(const char *list)
3127{
3128// Manually set the output archive list. Free text - you are on your own...
3129// Not allowed if default outputs are not disabled.
3130 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3131 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
3132 return;
3133 }
3134 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
3135 fOutputArchive = list;
3136}
3137
3138//______________________________________________________________________________
3139void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
3140{
3141// Setting a prefered output SE is not allowed anymore.
3142 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
3143}
3144
28ce6c9c 3145//______________________________________________________________________________
3146void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
3147{
3148// Set some PROOF special parameter.
3149 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3150 if (pair) {
3151 TObject *old = pair->Key();
3152 TObject *val = pair->Value();
3153 fProofParam.Remove(old);
3154 delete old;
3155 delete val;
3156 }
3157 fProofParam.Add(new TObjString(pname), new TObjString(value));
3158}
3159
3160//______________________________________________________________________________
3161const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
3162{
3163// Returns a special PROOF parameter.
3164 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3165 if (!pair) return 0;
3166 return pair->Value()->GetName();
3167}
3168
5513444a 3169//______________________________________________________________________________
3170Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 3171{
3172// Start remote grid analysis.
3bdcb562 3173 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3174 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
3175 if (!mgr || !mgr->IsInitialized()) {
3176 Error("StartAnalysis", "You need an initialized analysis manager for this");
3177 return kFALSE;
3178 }
3179 // Are we in PROOF mode ?
3180 if (mgr->IsProofMode()) {
baa34407 3181 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3182 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3183 if (fProofCluster.IsNull()) {
3184 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3185 return kFALSE;
3186 }
3187 if (fProofDataSet.IsNull() && !testMode) {
3188 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3189 return kFALSE;
3190 }
3191 // Set the needed environment
3192 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3193 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3194 if (fProofReset && !testMode) {
3195 if (fProofReset==1) {
3196 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3197 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3198 } else {
3199 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3200 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3201 }
3202 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3203 return kFALSE;
3204 }
baa34407 3205
3206 if (!testMode) {
3207 // Check if there is an old active session
3208 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3209 if (nsessions) {
3210 Error("StartAnalysis","You have to reset your old session first\n");
3211 return kFALSE;
3212 }
53911ca6 3213 }
3bdcb562 3214 // Do we need to change the ROOT version ? The success of this cannot be checked.
874de7ea 3215 if (!fROOTVersion.IsNull() && !testMode) {
cbccef47 3216 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"VO_ALICE@ROOT::%s\");",
874de7ea 3217 fProofCluster.Data(), fROOTVersion.Data()));
3bdcb562 3218 }
3219 // Connect to PROOF and check the status
3220 Long_t proof = 0;
27734f0e 3221 TString sworkers;
3222 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3223 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3224 if (!testMode) {
27734f0e 3225 if (!sworkers.IsNull())
3226 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3227 else
3228 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3229 } else {
3230 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3231 if (!proof) {
3232 Error("StartAnalysis", "Could not start PROOF in test mode");
3233 return kFALSE;
3234 }
3235 }
3236 if (!proof) {
3237 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3238 return kFALSE;
3239 }
27734f0e 3240 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3241 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3242 // Set proof special parameters if any
3243 TIter nextpp(&fProofParam);
3244 TObject *proofparam;
3245 while ((proofparam=nextpp())) {
3246 TString svalue = GetProofParameter(proofparam->GetName());
3247 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3248 }
3bdcb562 3249 // Is dataset existing ?
3250 if (!testMode) {
3251 TString dataset = fProofDataSet;
3252 Int_t index = dataset.Index("#");
3253 if (index>=0) dataset.Remove(index);
830acc4c 3254// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3255// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3256// return kFALSE;
3257// }
3258// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3259 }
3260 // Is ClearPackages() needed ?
3261 if (TestSpecialBit(kClearPackages)) {
3262 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3263 gROOT->ProcessLine("gProof->ClearPackages();");
3264 }
3265 // Is a given aliroot mode requested ?
3266 TList optionsList;
1ac846e9 3267 TString parLibs;
830acc4c 3268 if (!fAliRootMode.IsNull()) {
3bdcb562 3269 TString alirootMode = fAliRootMode;
3270 if (alirootMode == "default") alirootMode = "";
3271 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3272 optionsList.SetOwner();
3273 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3274 // Check the additional libs to be loaded
3275 TString extraLibs;
1ac846e9 3276 Bool_t parMode = kFALSE;
dc1ffbcb 3277 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3278 // Parse the extra libs for .so
3279 if (fAdditionalLibs.Length()) {
a41b2882 3280 TString additionalLibs = fAdditionalLibs;
3281 additionalLibs.Strip();
3282 if (additionalLibs.Length() && fFriendLibs.Length())
3283 additionalLibs += " ";
3284 additionalLibs += fFriendLibs;
3285 TObjArray *list = additionalLibs.Tokenize(" ");
3bdcb562 3286 TIter next(list);
3287 TObjString *str;
1ac846e9 3288 while((str=(TObjString*)next())) {
3289 if (str->GetString().Contains(".so")) {
3290 if (parMode) {
3291 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3292 break;
3293 }
3294 TString stmp = str->GetName();
3295 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3296 stmp.ReplaceAll(".so","");
3297 if (!extraLibs.IsNull()) extraLibs += ":";
3298 extraLibs += stmp;
3299 continue;
3300 }
3301 if (str->GetString().Contains(".par")) {
3302 // The first par file found in the list will not allow any further .so
3303 parMode = kTRUE;
3304 if (!parLibs.IsNull()) parLibs += ":";
3305 parLibs += str->GetName();
3306 continue;
3307 }
3bdcb562 3308 }
3309 if (list) delete list;
3310 }
a41b2882 3311 if (!extraLibs.IsNull()) {
3312 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3313 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3314 }
830acc4c 3315 // Check extra includes
3316 if (!fIncludePath.IsNull()) {
3317 TString includePath = fIncludePath;
3318 includePath.ReplaceAll(" ",":");
dc030a24 3319 includePath.ReplaceAll("$ALICE_ROOT/","");
3320 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3321 includePath.ReplaceAll("-I","");
dc030a24 3322 includePath.Remove(TString::kTrailing, ':');
830acc4c 3323 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3324 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3325 }
27734f0e 3326 // Check if connection to grid is requested
3327 if (TestSpecialBit(kProofConnectGrid))
3328 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3329 // Enable AliRoot par
3330 if (testMode) {
3331 // Enable proof lite package
3332 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3333 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3334 TNamed *obj = (TNamed*)optionsList.At(i);
3335 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3336 }
3337 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3338 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3339 Info("StartAnalysis", "AliRootProofLite enabled");
3340 } else {
3341 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3342 return kFALSE;
3343 }
3344 } else {
dc030a24 3345 if ( ! fAliROOTVersion.IsNull() ) {
3346 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3347 fAliROOTVersion.Data(), &optionsList))) {
3348 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3349 return kFALSE;
3350 }
3351 }
27734f0e 3352 }
1ac846e9 3353 // Enable first par files from fAdditionalLibs
3354 if (!parLibs.IsNull()) {
3355 TObjArray *list = parLibs.Tokenize(":");
3356 TIter next(list);
3357 TObjString *package;
3358 while((package=(TObjString*)next())) {
40501bb7 3359 TString spkg = package->GetName();
3360 spkg.ReplaceAll(".par", "");
3361 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3362 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3363 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3364 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3365 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3366 return kFALSE;
3367 }
3368 } else {
3369 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3370 return kFALSE;
3371 }
3372 }
3373 if (list) delete list;
3374 }
3bdcb562 3375 } else {
3376 if (fAdditionalLibs.Contains(".so") && !testMode) {
3377 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3378 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3379 return kFALSE;
3380 }
3381 }
3382 // Enable par files if requested
3383 if (fPackages && fPackages->GetEntries()) {
3384 TIter next(fPackages);
3385 TObject *package;
3386 while ((package=next())) {
1ac846e9 3387 // Skip packages already enabled
3388 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3389 TString spkg = package->GetName();
3390 spkg.ReplaceAll(".par", "");
3391 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3392 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3393 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3394 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3395 return kFALSE;
3396 }
3397 } else {
3398 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3399 return kFALSE;
3400 }
3401 }
3402 }
3403 // Do we need to load analysis source files ?
3404 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3405 if (fAnalysisSource.Length()) {
3406 TObjArray *list = fAnalysisSource.Tokenize(" ");
3407 TIter next(list);
3408 TObjString *str;
3409 while((str=(TObjString*)next())) {
3410 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3411 }
3bdcb562 3412 if (list) delete list;
3413 }
830acc4c 3414 if (testMode) {
3415 // Register dataset to proof lite.
3416 if (fFileForTestMode.IsNull()) {
3417 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3418 return kFALSE;
3419 }
3420 if (gSystem->AccessPathName(fFileForTestMode)) {
3421 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3422 return kFALSE;
3423 }
3424 TFileCollection *coll = new TFileCollection();
3425 coll->AddFromFile(fFileForTestMode);
8ddad121 3426 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3427 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3428 }
3bdcb562 3429 return kTRUE;
3430 }
c57f56b7 3431
43da816a 3432 // Check if output files have to be taken from the analysis manager
3433 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3434 // Add output files and AOD files
3435 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3436 // Add extra files registered to the analysis manager
f790bc1b 3437 TString extra = GetListOfFiles("ext");
3438 if (!extra.IsNull()) {
149d288c 3439 extra.ReplaceAll(".root", "*.root");
f790bc1b 3440 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3441 fOutputFiles += extra;
84fcd93f 3442 }
149d288c 3443 // Compose the output archive.
b83f84f0 3444 fOutputArchive = "log_archive.zip:std*@disk=1 ";
ec5acdce 3445 if (mgr->IsCollectThroughput())
95e16f01 3446 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",fOutputFiles.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 3447 else
3448 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3449 }
f7b1cbc2 3450// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3451 if (TestBit(AliAnalysisGrid::kOffline)) {
3452 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3453 \n there nor any job run. You can revise the JDL and analysis \
3454 \n macro then run the same in \"submit\" mode.");
3455 } else if (TestBit(AliAnalysisGrid::kTest)) {
3456 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3457 \n dataset.");
3458 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3459 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3460 \n space and job submitted.");
3461 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3462 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3463 if (fMergeViaJDL) CheckInputData();
5513444a 3464 return kTRUE;
c57f56b7 3465 } else {
3466 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3467 }
3468
348be253 3469 Print();
c57f56b7 3470 if (!Connect()) {
3471 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3472 return kFALSE;
16a4353c 3473 }
d3b18c4c 3474 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3475 if (!CheckInputData()) {
3476 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3477 return kFALSE;
c57f56b7 3478 }
d3339be3 3479 if (!CreateDataset(fDataPattern)) {
3480 TString serror;
3481 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3482 if (fRunNumbers.Length()) serror = "run numbers";
3483 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3484 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3485 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3486 return kFALSE;
3487 }
21ca8e59 3488 WriteAnalysisFile();
c57f56b7 3489 WriteAnalysisMacro();
3490 WriteExecutable();
3491 WriteValidationScript();
0f389141 3492 if (fMergeViaJDL) {
3493 WriteMergingMacro();
3494 WriteMergeExecutable();
3495 WriteValidationScript(kTRUE);
3496 }
5513444a 3497 if (!CreateJDL()) return kFALSE;
3498 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3499 if (testMode) {
c57f56b7 3500 // Locally testing the analysis
3501 Info("StartAnalysis", "\n_______________________________________________________________________ \
3502 \n Running analysis script in a daughter shell as on a worker node \
3503 \n_______________________________________________________________________");
149d288c 3504 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3505 TIter next(list);
3506 TObjString *str;
a2f5fc01 3507 TString outputFile;
c57f56b7 3508 while((str=(TObjString*)next())) {
a2f5fc01 3509 outputFile = str->GetString();
3510 Int_t index = outputFile.Index("@");
3511 if (index > 0) outputFile.Remove(index);
3512 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3513 }
3514 delete list;
3515 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3516 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3517// gSystem->Exec("cat stdout");
5513444a 3518 return kFALSE;
c57f56b7 3519 }
5513444a 3520 // Check if submitting is managed by LPM manager
a3e84053 3521 if (fProductionMode) {
7ac61826 3522 //TString prodfile = fJDLName;
3523 //prodfile.ReplaceAll(".jdl", ".prod");
3524 //WriteProductionFile(prodfile);
5513444a 3525 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3526 return kFALSE;
3527 }
a8739e8a 3528 // Submit AliEn job(s)
d2a409b2 3529 gGrid->Cd(fGridOutputDir);
a8739e8a 3530 TGridResult *res;
c57f56b7 3531 TString jobID = "";
0ad3ea1f 3532 fGridJobIDs = "";
3533 fGridStages = "";
d2a409b2 3534 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3535 // Submit a given xml or a set of runs
a8739e8a 3536 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3537 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3538 if (res) {
3539 const char *cjobId = res->GetKey(0,"jobId");
3540 if (!cjobId) {
a03be957 3541 gGrid->Stdout();
3542 gGrid->Stderr();
a8739e8a 3543 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3544 return kFALSE;
a8739e8a 3545 } else {
3546 Info("StartAnalysis", "\n_______________________________________________________________________ \
3547 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3548 \n_______________________________________________________________________",
3549 fJDLName.Data(), cjobId);
3550 jobID = cjobId;
0ad3ea1f 3551 if (jobID.Atoi()) {
3552 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3553 fGridJobIDs.Append(jobID);
3554 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3555 fGridStages.Append("full");
3556 }
a8739e8a 3557 }
3558 delete res;
a03be957 3559 } else {
3560 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3561 return kFALSE;
a8739e8a 3562 }
3563 } else {
d2a409b2 3564 // Submit for a range of enumeration of runs.
a03be957 3565 if (!Submit()) return kFALSE;
0ad3ea1f 3566 jobID = fGridJobIDs;
c57f56b7 3567 }
a8739e8a 3568
d88e4159 3569 if (fDropToShell) {
3570 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3571 \n You may exit at any time and terminate the job later using the option <terminate> \
3572 \n ##################################################################################", jobID.Data());
3573 gSystem->Exec("aliensh");
3574 } else {
3575 Info("StartAnalysis", "\n#### SUBMITTED JOB %s TO ALIEN QUEUE #### \
3576 \n Remember to terminate the job later using the option <terminate> \
3577 \n ##################################################################################", jobID.Data());
3578 }
5513444a 3579 return kTRUE;
c57f56b7 3580}
3581
f790bc1b 3582//______________________________________________________________________________
3583const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3584{
3585// Get a comma-separated list of output files of the requested type.
3586// Type can be (case unsensitive):
3587// aod - list of aod files (std, extensions and filters)
3588// out - list of output files connected to containers (but not aod's or extras)
3589// ext - list of extra files registered to the manager
3590// ter - list of files produced in terminate
3591 static TString files;
3592 files = "";
3593 TString stype = type;
3594 stype.ToLower();
3595 TString aodfiles, extra;
3596 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3597 if (!mgr) {
3598 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3599 return files.Data();
3600 }
3601 if (mgr->GetOutputEventHandler()) {
3602 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3603 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3604 if (!extraaod.IsNull()) {
3605 aodfiles += ",";
3606 aodfiles += extraaod;
3607 }
3608 }
3609 if (stype.Contains("aod")) {
3610 files = aodfiles;
3611 if (stype == "aod") return files.Data();
3612 }
3613 // Add output files that are not in the list of AOD files
3614 TString outputfiles = "";
3615 TIter next(mgr->GetOutputs());
3616 AliAnalysisDataContainer *output;
3617 const char *filename = 0;
3618 while ((output=(AliAnalysisDataContainer*)next())) {
3619 filename = output->GetFileName();
3620 if (!(strcmp(filename, "default"))) continue;
3621 if (outputfiles.Contains(filename)) continue;
3622 if (aodfiles.Contains(filename)) continue;
3623 if (!outputfiles.IsNull()) outputfiles += ",";
3624 outputfiles += filename;
3625 }
3626 if (stype.Contains("out")) {
3627 if (!files.IsNull()) files += ",";
3628 files += outputfiles;
3629 if (stype == "out") return files.Data();
3630 }
3631 // Add extra files registered to the analysis manager
3632 TString sextra;
3633 extra = mgr->GetExtraFiles();
3634 if (!extra.IsNull()) {
3635 extra.Strip();
3636 extra.ReplaceAll(" ", ",");
3637 TObjArray *fextra = extra.Tokenize(",");
3638 TIter nextx(fextra);
3639 TObject *obj;
3640 while ((obj=nextx())) {
3641 if (aodfiles.Contains(obj->GetName())) continue;
3642 if (outputfiles.Contains(obj->GetName())) continue;
3643 if (sextra.Contains(obj->GetName())) continue;
3644 if (!sextra.IsNull()) sextra += ",";
3645 sextra += obj->GetName();
3646 }
3647 delete fextra;
3648 if (stype.Contains("ext")) {
3649 if (!files.IsNull()) files += ",";
3650 files += sextra;
3651 }
3652 }
3653 if (stype == "ext") return files.Data();
3654 TString termfiles;
3655 if (!fTerminateFiles.IsNull()) {
3656 fTerminateFiles.Strip();
3657 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3658 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3659 TIter nextx(fextra);
3660 TObject *obj;
3661 while ((obj=nextx())) {
3662 if (aodfiles.Contains(obj->GetName())) continue;
3663 if (outputfiles.Contains(obj->GetName())) continue;
3664 if (termfiles.Contains(obj->GetName())) continue;
3665 if (sextra.Contains(obj->GetName())) continue;
3666 if (!termfiles.IsNull()) termfiles += ",";
3667 termfiles += obj->GetName();
3668 }
3669 delete fextra;
3670 }
3671 if (stype.Contains("ter")) {
3672 if (!files.IsNull() && !termfiles.IsNull()) {
3673 files += ",";
3674 files += termfiles;
3675 }
3676 }
3677 return files.Data();
3678}
3679
d2a409b2 3680//______________________________________________________________________________
a03be957 3681Bool_t AliAnalysisAlien::Submit()
d2a409b2 3682{
3683// Submit all master jobs.
3684 Int_t nmasterjobs = fInputFiles->GetEntries();
3685 Long_t tshoot = gSystem->Now();
a03be957 3686 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3687 while (fNsubmitted < nmasterjobs) {
3688 Long_t now = gSystem->Now();
3689 if ((now-tshoot)>30000) {
3690 tshoot = now;
a03be957 3691 if (!SubmitNext()) return kFALSE;
d2a409b2 3692 }
3693 }
a03be957 3694 return kTRUE;
d2a409b2 3695}
3696
0f389141 3697//______________________________________________________________________________
a03be957 3698Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3699{
3700// Submit all merging jobs.
dc030a24 3701 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3702 gGrid->Cd(fGridOutputDir);
3703 TString mergeJDLName = fExecutable;
3704 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3705 if (!fInputFiles) {
3706 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3707 return kFALSE;
3708 }
0f389141 3709 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3710 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3711 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3712 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3713 if (fOutputToRunNo) {
3714 // The output directory is the run number
3715 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3716 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3717 } else {
17b4e9a4 3718 if (!fRunNumbers.Length() && !fRunRange[0]) {
3719 // The output directory is the grid outdir
3720 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3721 runOutDir = fGridOutputDir;
3722 } else {
3723 // The output directory is the master number in 3 digits format
3724 printf("### Submitting merging job for master <%03d>\n", i);
3725 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3726 }
7c2cd90a 3727 }
3728 // Check now the number of merging stages.
2b222f7c 3729 TObjArray *list = fOutputFiles.Tokenize(",");
3730 TIter next(list);
3731 TObjString *str;
3732 TString outputFile;
3733 while((str=(TObjString*)next())) {
3734 outputFile = str->GetString();
3735 Int_t index = outputFile.Index("@");
3736 if (index > 0) outputFile.Remove(index);
37d21c01 3737 if (!fMergeExcludes.Contains(outputFile) &&
3738 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3739 }
3740 delete list;
b3e07543 3741 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3742 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3743 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3744 }
a03be957 3745 if (!ntosubmit) return kTRUE;
d88e4159 3746 if (fDropToShell) {
3747 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3748 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3749 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3750 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3751 \n ################################################################################################################");
3752 gSystem->Exec("aliensh");
3753 } else {
3754 Info("StartAnalysis", "\n #### STARTED MERGING JOBS FOR YOU #### \
3755 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL') \
3756 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3757 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3758 \n ################################################################################################################");
3759 }
a03be957 3760 return kTRUE;
0f389141 3761}
3762
d2a409b2 3763//______________________________________________________________________________
a03be957 3764Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3765{
08d5b699 3766// Submit next bunch of master jobs if the queue is free. The first master job is
3767// submitted right away, while the next will not be unless the previous was split.
3768// The plugin will not submit new master jobs if there are more that 500 jobs in
3769// waiting phase.
d2a409b2 3770 static Bool_t iscalled = kFALSE;
3771 static Int_t firstmaster = 0;
3772 static Int_t lastmaster = 0;
3773 static Int_t npermaster = 0;
a03be957 3774 if (iscalled) return kTRUE;
d2a409b2 3775 iscalled = kTRUE;
3776 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3777 Int_t ntosubmit = 0;
3778 TGridResult *res;
3779 TString jobID = "";
3bdcb562 3780 Int_t nmasterjobs = fInputFiles->GetEntries();
3781 if (!fNsubmitted) {
3782 ntosubmit = 1;
3783 if (!IsUseSubmitPolicy()) {
90d50a8c 3784 if (nmasterjobs>5)
3bdcb562 3785 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3786 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3787 ntosubmit = nmasterjobs;
3788 }
3789 } else {
d2a409b2 3790 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3791 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3792 // If last master not split, just return
a03be957 3793 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3794 // No more than 100 waiting jobs
08d5b699 3795 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3796 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3797 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3798 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3799 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3800 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3801 }
d2a409b2 3802 for (Int_t i=0; i<ntosubmit; i++) {
3803 // Submit for a range of enumeration of runs.
a03be957 3804 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3805 TString query;
cd11251e 3806 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3807 runOutDir.ReplaceAll(".xml", "");
3808 if (fOutputToRunNo)
3809 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3810 else
3811 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3812 printf("********* %s\n",query.Data());
d2a409b2 3813 res = gGrid->Command(query);
3814 if (res) {
98ca124f 3815 TString cjobId1 = res->GetKey(0,"jobId");
3816 if (!cjobId1.Length()) {
d2a409b2 3817 iscalled = kFALSE;
a03be957 3818 gGrid->Stdout();
3819 gGrid->Stderr();
3820 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3821 return kFALSE;
d2a409b2 3822 } else {
3823 Info("StartAnalysis", "\n_______________________________________________________________________ \
3824 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3825 \n_______________________________________________________________________",
98ca124f 3826 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
0ad3ea1f 3827 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3828 fGridJobIDs.Append(cjobId1);
3829 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3830 fGridStages.Append("full");
d2a409b2 3831 jobID += cjobId1;
3832 jobID += " ";
98ca124f 3833 lastmaster = cjobId1.Atoi();
d2a409b2 3834 if (!firstmaster) firstmaster = lastmaster;
3835 fNsubmitted++;
3836 }
3837 delete res;
a03be957 3838 } else {
3839 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3840 return kFALSE;
d2a409b2 3841 }
3842 }
3843 iscalled = kFALSE;
a03be957 3844 return kTRUE;
d2a409b2 3845}
3846
c57f56b7 3847//______________________________________________________________________________
3848void AliAnalysisAlien::WriteAnalysisFile()
3849{
f10e8481 3850// Write current analysis manager into the file <analysisFile>
3851 TString analysisFile = fExecutable;
3852 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3853 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3854 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3855 if (!mgr || !mgr->IsInitialized()) {
3856 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3857 return;
3858 }
3859 // Check analysis type
3860 TObject *handler;
3861 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3862 handler = (TObject*)mgr->GetInputEventHandler();
3863 if (handler) {
65eb22f8 3864 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3865 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3866 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3867 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3868 } else {
3869 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3870 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3871 }
c57f56b7 3872 }
3873 TDirectory *cdir = gDirectory;
f10e8481 3874 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3875 if (file) {
e16a394c 3876 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3877 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3878 // Unless merging makes no sense
3879 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3880 mgr->Write();
3881 delete file;
fe2d7fc2 3882 // Enable termination for local jobs
3883 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3884 }
3885 if (cdir) cdir->cd();
f10e8481 3886 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3887 }
3888 Bool_t copy = kTRUE;
d3b18c4c 3889 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3890 if (copy) {
3891 CdWork();
3892 TString workdir = gGrid->GetHomeDirectory();
3893 workdir += fGridWorkingDir;
d3b18c4c 3894 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3895 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3896 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3897 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3898 }
3899}
3900
3901//______________________________________________________________________________
3902void AliAnalysisAlien::WriteAnalysisMacro()
3903{
3904// Write the analysis macro that will steer the analysis in grid mode.
3905 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3906 ofstream out;
3907 out.open(fAnalysisMacro.Data(), ios::out);
3908 if (!out.good()) {
3909 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3910 return;
3911 }
5b273635 3912 Bool_t hasSTEERBase = kFALSE;
3913 Bool_t hasESD = kFALSE;
3914 Bool_t hasAOD = kFALSE;
3915 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3916 Bool_t hasOADB = kFALSE;
5b273635 3917 Bool_t hasANALYSISalice = kFALSE;
3918 Bool_t hasCORRFW = kFALSE;
c57f56b7 3919 TString func = fAnalysisMacro;
3920 TString type = "ESD";
3921 TString comment = "// Analysis using ";
c85cfc0f 3922 if (IsUseMCchain()) {
3923 type = "MC";
3924 comment += "MC";
3925 } else {
3926 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3927 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3928 type = "AOD";
3929 comment += "AOD";
3930 }
3931 }
0df6ccf2 3932 if (type!="AOD" && fFriendChainName!="") {
3933 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3934 return;
3935 }
c57f56b7 3936 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3937 else comment += " data";
3938 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3939 func.ReplaceAll(".C", "");
3940 out << "void " << func.Data() << "()" << endl;
3941 out << "{" << endl;
3942 out << comment.Data() << endl;
3943 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3944 out << " TStopwatch timer;" << endl;
3945 out << " timer.Start();" << endl << endl;
13f0c00b 3946 // Change temp directory to current one
fdee57f1 3947 if (!IsLocalTest()) {
3948 out << "// connect to AliEn and make the chain" << endl;
3949 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
3950 }
13f0c00b 3951 out << "// Set temporary merging directory to current one" << endl;
3952 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3953 out << "// Set temporary compilation directory to current one" << endl;
3954 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3955 // Reset existing include path
3956 out << "// Reset existing include path and add current directory first in the search" << endl;
3957 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3958 if (!fExecutableCommand.Contains("aliroot")) {
3959 out << "// load base root libraries" << endl;
3960 out << " gSystem->Load(\"libTree\");" << endl;
3961 out << " gSystem->Load(\"libGeom\");" << endl;
3962 out << " gSystem->Load(\"libVMC\");" << endl;
3963 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3964 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3965 }
d5c6455a 3966 if (fAdditionalRootLibs.Length()) {
47a4137d 3967 // in principle libtree /lib geom libvmc etc. can go into this list, too
3968 out << "// Add aditional libraries" << endl;
3969 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3970 TIter next(list);
3971 TObjString *str;
3972 while((str=(TObjString*)next())) {
3973 if (str->GetString().Contains(".so"))
3974 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3975 }
47a4137d 3976 if (list) delete list;
d5c6455a 3977 }
57377eb5 3978 out << "// Load analysis framework libraries" << endl;
652561d5 3979 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3980 if (!fPackages) {
f790bc1b 3981 if (!fExecutableCommand.Contains("aliroot")) {
3982 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3983 out << " gSystem->Load(\"libESD\");" << endl;
3984 out << " gSystem->Load(\"libAOD\");" << endl;
3985 }
4e5c5506 3986 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3987 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3988 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3989 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3990 } else {
4e5c5506 3991 TIter next(fPackages);
3992 TObject *obj;
57377eb5 3993 TString pkgname;
57377eb5 3994 while ((obj=next())) {
3995 pkgname = obj->GetName();
4478e6f1 3996 if (pkgname == "STEERBase" ||
3997 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3998 if (pkgname == "ESD" ||
3999 pkgname == "ESD.par") hasESD = kTRUE;
4000 if (pkgname == "AOD" ||
4001 pkgname == "AOD.par") hasAOD = kTRUE;
4002 if (pkgname == "ANALYSIS" ||
4003 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4004 if (pkgname == "OADB" ||
4005 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 4006 if (pkgname == "ANALYSISalice" ||
4007 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4008 if (pkgname == "CORRFW" ||
4009 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 4010 }
4011 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 4012 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4013 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 4014 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4015 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 4016 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4017 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 4018 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4019 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4020 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4021 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 4022 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4023 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 4024 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4025 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 4026 out << "// Compile other par packages" << endl;
4027 next.Reset();
fcc9bb6f 4028 while ((obj=next())) {
4029 pkgname = obj->GetName();
4478e6f1 4030 if (pkgname == "STEERBase" ||
4031 pkgname == "STEERBase.par" ||
4032 pkgname == "ESD" ||
4033 pkgname == "ESD.par" ||
4034 pkgname == "AOD" ||
4035 pkgname == "AOD.par" ||
4036 pkgname == "ANALYSIS" ||
4037 pkgname == "ANALYSIS.par" ||
7ea40ac8 4038 pkgname == "OADB" ||
4039 pkgname == "OADB.par" ||
4478e6f1 4040 pkgname == "ANALYSISalice" ||
4041 pkgname == "ANALYSISalice.par" ||
4042 pkgname == "CORRFW" ||
4043 pkgname == "CORRFW.par") continue;
5b273635 4044 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 4045 }
4e5c5506 4046 }
b3e07543 4047 out << "// include path" << endl;
4048 // Get the include path from the interpreter and remove entries pointing to AliRoot
4049 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4050 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4051 out << " TIter nextpath(listpaths);" << endl;
4052 out << " TObjString *pname;" << endl;
4053 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4054 out << " TString current = pname->GetName();" << endl;
4055 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4056 out << " gSystem->AddIncludePath(current);" << endl;
4057 out << " }" << endl;
4058 out << " if (listpaths) delete listpaths;" << endl;
4059 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4060 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4061 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 4062 if (fAdditionalLibs.Length()) {
4063 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4064 TString additionalLibs = fAdditionalLibs;
4065 additionalLibs.Strip();
4066 if (additionalLibs.Length() && fFriendLibs.Length())
4067 additionalLibs += " ";
4068 additionalLibs += fFriendLibs;
4069 TObjArray *list = additionalLibs.Tokenize(" ");
6da75e0b 4070 TIter next(list);
4071 TObjString *str;
4072 while((str=(TObjString*)next())) {
4073 if (str->GetString().Contains(".so"))
4074 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 4075 if (str->GetString().Contains(".par"))
4076 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 4077 }
4078 if (list) delete list;
4079 }
4080 out << endl;
c57f56b7 4081 out << "// analysis source to be compiled at runtime (if any)" << endl;
4082 if (fAnalysisSource.Length()) {
4083 TObjArray *list = fAnalysisSource.Tokenize(" ");
4084 TIter next(list);
4085 TObjString *str;
4086 while((str=(TObjString*)next())) {
4087 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4088 }
4089 if (list) delete list;
4090 }
4091 out << endl;
b3e07543 4092// out << " printf(\"Currently load libraries:\\n\");" << endl;
4093// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 4094 if (fFastReadOption) {
d3339be3 4095 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
4096 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 4097 out << "// fast xrootd reading enabled" << endl;
4098 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4099 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4100 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4101 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4102 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4103 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 4104 }
c57f56b7 4105 out << "// read the analysis manager from file" << endl;
f10e8481 4106 TString analysisFile = fExecutable;
4107 analysisFile.ReplaceAll(".sh", ".root");
105631b5 4108 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4109 << analysisFile << "\");" << endl;
4110 out << " if (!mgr) return;" << endl;
21ca8e59 4111 if (IsLocalTest()) {
4112 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
4113 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 4114 if (fFileForTestMode.IsNull())
4115 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
4116 else
4117 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 4118 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4b5d9e21 4119 if (!fFriendChainName.IsNull())
b9f1efa5 4120 out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\",\"" << fFriendLibs << "\");" << endl;
024dfa52 4121 if (IsUseMCchain())
4122 out << " plugin->SetUseMCchain();" << endl;
21ca8e59 4123 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 4124 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 4125 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 4126 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
4127 } else {
7a366e0b 4128 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 4129 out << " mgr->SetNSysInfo(100);" << endl;
4130 }
21ca8e59 4131 }
c57f56b7 4132 out << " mgr->PrintStatus();" << endl;
52b6a92b 4133 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 4134 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 4135 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 4136 } else {
9a8288ff 4137 if (TestBit(AliAnalysisGrid::kTest))
4138 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4139 else
4140 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 4141 }
4142 }
21ca8e59 4143 if (!IsLocalTest()) {
242accb2 4144 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 4145 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
4146 } else {
4147 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 4148 }
f7498086 4149 out << " timer.Stop();" << endl;
4150 out << " timer.Print();" << endl;
c57f56b7 4151 out << "}" << endl << endl;
21ca8e59 4152 if (!IsLocalTest()) {
fcc9bb6f 4153 out <<"//________________________________________________________________________________" << endl;
c57f56b7 4154 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
4155 out << "{" << endl;
4156 out << "// Create a chain using url's from xml file" << endl;
242accb2 4157 out << " TString filename;" << endl;
4158 out << " Int_t run = 0;" << endl;
c85cfc0f 4159 if (IsUseMCchain()) {
4160 out << " TString treename = \"TE\";" << endl;
4161 } else {
4162 out << " TString treename = type;" << endl;
4163 out << " treename.ToLower();" << endl;
4164 out << " treename += \"Tree\";" << endl;
4165 }
e02fee64 4166 out << " printf(\"***************************************\\n\");" << endl;
4167 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
4168 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 4169 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
4170 out << " if (!coll) {" << endl;
4171 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
4172 out << " return NULL;" << endl;
4173 out << " }" << endl;
242accb2 4174 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 4175 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 4176 if(fFriendChainName!="") {
4177 out << " TChain *chainFriend = new TChain(treename);" << endl;
4178 }
c57f56b7 4179 out << " coll->Reset();" << endl;
0df6ccf2 4180 out << " while (coll->Next()) {" << endl;
242accb2 4181 out << " filename = coll->GetTURL("");" << endl;
4182 out << " if (mgr) {" << endl;
4183 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
4184 out << " if (nrun && nrun != run) {" << endl;
4185 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
4186 out << " mgr->SetRunFromPath(nrun);" << endl;
4187 out << " run = nrun;" << endl;
4188 out << " }" << endl;
4189 out << " }" << endl;
4190 out << " chain->Add(filename);" << endl;
0df6ccf2 4191 if(fFriendChainName!="") {
4192 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
4b5d9e21 4193 out << " if (fileFriend.Index(\"#\") > -1) fileFriend.Remove(fileFriend.Index(\"#\"));" << endl;
4194 out << " fileFriend = gSystem->DirName(fileFriend);" << endl;
4195 out << " fileFriend += \"/\";" << endl;
4196 out << " fileFriend += \"" << fFriendChainName << "\";";
4197 out << " TFile *file = TFile::Open(fileFriend);" << endl;
4198 out << " if (file) {" << endl;
4199 out << " file->Close();" << endl;
4200 out << " chainFriend->Add(fileFriend.Data());" << endl;
4201 out << " } else {" << endl;
4202 out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
4203 out << " return 0;" << endl;
4204 out << " }" << endl;
0df6ccf2 4205 }
4206 out << " }" << endl;
c57f56b7 4207 out << " if (!chain->GetNtrees()) {" << endl;
4208 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
4209 out << " return NULL;" << endl;
4210 out << " }" << endl;
0df6ccf2 4211 if(fFriendChainName!="") {
4212 out << " chain->AddFriend(chainFriend);" << endl;
4213 }
c57f56b7 4214 out << " return chain;" << endl;
fcc9bb6f 4215 out << "}" << endl << endl;
c57f56b7 4216 }
5b273635 4217 if (hasANALYSISalice) {
4218 out <<"//________________________________________________________________________________" << endl;
4219 out << "Bool_t SetupPar(const char *package) {" << endl;
4220 out << "// Compile the package and set it up." << endl;
4221 out << " TString pkgdir = package;" << endl;
4222 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4223 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4224 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4225 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4226 out << " // Check for BUILD.sh and execute" << endl;
4227 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4228 out << " printf(\"*******************************\\n\");" << endl;
4229 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4230 out << " printf(\"*******************************\\n\");" << endl;
4231 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4232 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4233 out << " gSystem->ChangeDirectory(cdir);" << endl;
4234 out << " return kFALSE;" << endl;
4235 out << " }" << endl;
4236 out << " } else {" << endl;
4237 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4238 out << " gSystem->ChangeDirectory(cdir);" << endl;
4239 out << " return kFALSE;" << endl;
4240 out << " }" << endl;
4241 out << " // Check for SETUP.C and execute" << endl;
4242 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4243 out << " printf(\"*******************************\\n\");" << endl;
4244 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4245 out << " printf(\"*******************************\\n\");" << endl;
4246 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4247 out << " } else {" << endl;
4248 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4249 out << " gSystem->ChangeDirectory(cdir);" << endl;
4250 out << " return kFALSE;" << endl;
4251 out << " }" << endl;
4252 out << " // Restore original workdir" << endl;
4253 out << " gSystem->ChangeDirectory(cdir);" << endl;
4254 out << " return kTRUE;" << endl;
4255 out << "}" << endl;
4256 }
c57f56b7 4257 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4258 }
4259 Bool_t copy = kTRUE;
d3b18c4c 4260 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4261 if (copy) {
4262 CdWork();
4263 TString workdir = gGrid->GetHomeDirectory();
4264 workdir += fGridWorkingDir;
4265 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4266 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4267// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4268 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4269 Form("alien://%s/%s", workdir.Data(),
4270 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4271 }
4272}
4273
0f389141 4274//______________________________________________________________________________
4275void AliAnalysisAlien::WriteMergingMacro()
4276{
4277// Write a macro to merge the outputs per master job.
4278 if (!fMergeViaJDL) return;
4279 if (!fOutputFiles.Length()) {
4280 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4281 return;
4282 }
ec5acdce 4283 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
0f389141 4284 TString mergingMacro = fExecutable;
4285 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4286 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4287 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4288 ofstream out;
4289 out.open(mergingMacro.Data(), ios::out);
4290 if (!out.good()) {
4291 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4292 return;
4293 }
5b273635 4294 Bool_t hasSTEERBase = kFALSE;
4295 Bool_t hasESD = kFALSE;
4296 Bool_t hasAOD = kFALSE;
4297 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4298 Bool_t hasOADB = kFALSE;
5b273635 4299 Bool_t hasANALYSISalice = kFALSE;
4300 Bool_t hasCORRFW = kFALSE;
0f389141 4301 TString func = mergingMacro;
4302 TString comment;
4303 func.ReplaceAll(".C", "");
f47d5cb4 4304 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4305 out << "{" << endl;
4306 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4307 out << " TStopwatch timer;" << endl;
4308 out << " timer.Start();" << endl << endl;
b3e07543 4309 // Reset existing include path
4310 out << "// Reset existing include path and add current directory first in the search" << endl;
4311 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4312 if (!fExecutableCommand.Contains("aliroot")) {
4313 out << "// load base root libraries" << endl;
4314 out << " gSystem->Load(\"libTree\");" << endl;
4315 out << " gSystem->Load(\"libGeom\");" << endl;
4316 out << " gSystem->Load(\"libVMC\");" << endl;
4317 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4318 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4319 }
0f389141 4320 if (fAdditionalRootLibs.Length()) {
4321 // in principle libtree /lib geom libvmc etc. can go into this list, too
4322 out << "// Add aditional libraries" << endl;
4323 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4324 TIter next(list);
4325 TObjString *str;
4326 while((str=(TObjString*)next())) {
4327 if (str->GetString().Contains(".so"))
4328 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4329 }
4330 if (list) delete list;
4331 }
0f389141 4332 out << "// Load analysis framework libraries" << endl;
4333 if (!fPackages) {
7c2cd90a 4334 if (!fExecutableCommand.Contains("aliroot")) {
4335 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4336 out << " gSystem->Load(\"libESD\");" << endl;
4337 out << " gSystem->Load(\"libAOD\");" << endl;
4338 }
0f389141 4339 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4340 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4341 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4342 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4343 } else {
4344 TIter next(fPackages);
4345 TObject *obj;
4346 TString pkgname;
5b273635 4347 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4348 while ((obj=next())) {
4349 pkgname = obj->GetName();
4350 if (pkgname == "STEERBase" ||
4351 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4352 if (pkgname == "ESD" ||
4353 pkgname == "ESD.par") hasESD = kTRUE;
4354 if (pkgname == "AOD" ||
4355 pkgname == "AOD.par") hasAOD = kTRUE;
4356 if (pkgname == "ANALYSIS" ||
4357 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4358 if (pkgname == "OADB" ||
4359 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4360 if (pkgname == "ANALYSISalice" ||
4361 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4362 if (pkgname == "CORRFW" ||
4363 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4364 }
5b273635 4365 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4366 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4367 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4368 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4369 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4370 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4371 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4372 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4373 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4374 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4375 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4376 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4377 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4378 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4379 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4380 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4381 out << "// Compile other par packages" << endl;
4382 next.Reset();
4383 while ((obj=next())) {
4384 pkgname = obj->GetName();
4385 if (pkgname == "STEERBase" ||
4386 pkgname == "STEERBase.par" ||
4387 pkgname == "ESD" ||
4388 pkgname == "ESD.par" ||
4389 pkgname == "AOD" ||
4390 pkgname == "AOD.par" ||
4391 pkgname == "ANALYSIS" ||
4392 pkgname == "ANALYSIS.par" ||
7ea40ac8 4393 pkgname == "OADB" ||
4394 pkgname == "OADB.par" ||
0f389141 4395 pkgname == "ANALYSISalice" ||
4396 pkgname == "ANALYSISalice.par" ||
4397 pkgname == "CORRFW" ||
4398 pkgname == "CORRFW.par") continue;
5b273635 4399 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4400 }
4401 }
b3e07543 4402 out << "// include path" << endl;
4403 // Get the include path from the interpreter and remove entries pointing to AliRoot
4404 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4405 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4406 out << " TIter nextpath(listpaths);" << endl;
4407 out << " TObjString *pname;" << endl;
4408 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4409 out << " TString current = pname->GetName();" << endl;
4410 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4411 out << " gSystem->AddIncludePath(current);" << endl;
4412 out << " }" << endl;
4413 out << " if (listpaths) delete listpaths;" << endl;
4414 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4415 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4416 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4417 if (fAdditionalLibs.Length()) {
4418 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4419 TString additionalLibs = fAdditionalLibs;
4420 additionalLibs.Strip();
4421 if (additionalLibs.Length() && fFriendLibs.Length())
4422 additionalLibs += " ";
4423 additionalLibs += fFriendLibs;
4424 TObjArray *list = additionalLibs.Tokenize(" ");
0f389141 4425 TIter next(list);
4426 TObjString *str;
4427 while((str=(TObjString*)next())) {
4428 if (str->GetString().Contains(".so"))
4429 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4430 }
4431 if (list) delete list;
4432 }
4433 out << endl;
4434 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4435 if (fAnalysisSource.Length()) {
4436 TObjArray *list = fAnalysisSource.Tokenize(" ");
4437 TIter next(list);
4438 TObjString *str;
4439 while((str=(TObjString*)next())) {
4440 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4441 }
4442 if (list) delete list;
4443 }
149d288c 4444 out << endl;
4445
0f389141 4446 if (fFastReadOption) {
4447 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4448 out << "// fast xrootd reading enabled" << endl;
4449 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4450 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4451 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4452 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4453 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4454 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4455 }
4456 // Change temp directory to current one
fdee57f1 4457 out << "// Connect to AliEn" << endl;
4458 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
e8b839ab 4459 out << "// Set temporary merging directory to current one" << endl;
4460 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4461 out << "// Set temporary compilation directory to current one" << endl;
4462 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
7c2cd90a 4463 out << " TString outputDir = dir;" << endl;
f790bc1b 4464 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4465 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4466 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4467 out << " TIter *iter = new TIter(list);" << endl;
4468 out << " TObjString *str;" << endl;
a2f5fc01 4469 out << " TString outputFile;" << endl;
0f389141 4470 out << " Bool_t merged = kTRUE;" << endl;
ec5acdce 4471 TString analysisFile = fExecutable;
4472 analysisFile.ReplaceAll(".sh", ".root");
4473 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4474 << analysisFile << "\");" << endl;
4475 out << " if (!mgr) {" << endl;
4476 out << " printf(\"ERROR: Analysis manager could not be extracted from file \");" << endl;
4477 out << " return;" << endl;
4478 out << " }" << endl;
d66aa2a9 4479 if (IsLocalTest()) {
4480 out << " printf(\"===================================\n\");" << endl;
4481 out << " printf(\"Testing merging...\\n\");" << endl;
4482 out << " printf(\"===================================\n\");" << endl;
4483 }
0f389141 4484 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4485 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4486 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4487 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4488 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4489 out << " // Skip already merged outputs" << endl;
a2f5fc01 4490 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
d66aa2a9 4491 out << " printf(\"Output file <%s> found. Not merging again.\\n\",outputFile.Data());" << endl;
0f389141 4492 out << " continue;" << endl;
4493 out << " }" << endl;
a2f5fc01 4494 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4495 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4496 out << " if (!merged) {" << endl;
a2f5fc01 4497 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4498 out << " return;" << endl;
0f389141 4499 out << " }" << endl;
4500 out << " }" << endl;
d66aa2a9 4501 if (mgr && mgr->IsCollectThroughput() && !IsLocalTest()) {
ec5acdce 4502 out << " TString infolog = \"" << mgr->GetFileInfoLog() << "\";" << endl;
4503 out << " AliAnalysisAlien::MergeInfo(infolog, outputDir);" << endl;
4504 }
7c2cd90a 4505 out << " // all outputs merged, validate" << endl;
4506 out << " ofstream out;" << endl;
4507 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4508 out << " out.close();" << endl;
4509 out << " // read the analysis manager from file" << endl;
d66aa2a9 4510 if (IsLocalTest()) {
4511 out << " printf(\"===================================\n\");" << endl;
4512 out << " printf(\"Testing Terminate()...\\n\");" << endl;
4513 out << " printf(\"===================================\n\");" << endl;
4514 } else {
4515 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
4516 }
5c4250fc 4517 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4518 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4519 out << " mgr->PrintStatus();" << endl;
ec5acdce 4520 if (mgr) {
4521 if (mgr->GetDebugLevel()>3) {
f866cba5 4522 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4523 } else {
9a8288ff 4524 if (TestBit(AliAnalysisGrid::kTest))
4525 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4526 else
4527 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4528 }
4529 }
9a8288ff 4530 out << " TTree *tree = NULL;" << endl;
4531 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4532 out << "}" << endl << endl;
5b273635 4533 if (hasANALYSISalice) {
4534 out <<"//________________________________________________________________________________" << endl;
4535 out << "Bool_t SetupPar(const char *package) {" << endl;
4536 out << "// Compile the package and set it up." << endl;
4537 out << " TString pkgdir = package;" << endl;
4538 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4539 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4540 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4541 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4542 out << " // Check for BUILD.sh and execute" << endl;
4543 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4544 out << " printf(\"*******************************\\n\");" << endl;
4545 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4546 out << " printf(\"*******************************\\n\");" << endl;
4547 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4548 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4549 out << " gSystem->ChangeDirectory(cdir);" << endl;
4550 out << " return kFALSE;" << endl;
4551 out << " }" << endl;
4552 out << " } else {" << endl;
4553 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4554 out << " gSystem->ChangeDirectory(cdir);" << endl;
4555 out << " return kFALSE;" << endl;
4556 out << " }" << endl;
4557 out << " // Check for SETUP.C and execute" << endl;
4558 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4559 out << " printf(\"*******************************\\n\");" << endl;
4560 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4561 out << " printf(\"*******************************\\n\");" << endl;
4562 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4563 out << " } else {" << endl;
4564 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4565 out << " gSystem->ChangeDirectory(cdir);" << endl;
4566 out << " return kFALSE;" << endl;
4567 out << " }" << endl;
4568 out << " // Restore original workdir" << endl;
4569 out << " gSystem->ChangeDirectory(cdir);" << endl;
4570 out << " return kTRUE;" << endl;
4571 out << "}" << endl;
4572 }
0f389141 4573 }
4574 Bool_t copy = kTRUE;
d3b18c4c 4575 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4576 if (copy) {
4577 CdWork();
4578 TString workdir = gGrid->GetHomeDirectory();
4579 workdir += fGridWorkingDir;
4580 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4581 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4582// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4583 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4584 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4585 }
4586}
4587
4588//______________________________________________________________________________
4589Bool_t AliAnalysisAlien::SetupPar(const char *package)
4590{
205b201f 4591// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4592// Note that for loading the compiled library. The current directory should have precedence in
4593// LD_LIBRARY_PATH
4594 TString pkgdir = package;
4595 pkgdir.ReplaceAll(".par","");
b50a41c3 4596 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4597 TString cdir = gSystem->WorkingDirectory();
4598 gSystem->ChangeDirectory(pkgdir);
4599 // Check for BUILD.sh and execute
4600 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4601 printf("**************************************************\n");
4602 printf("*** Building PAR archive %s\n", package);
4603 printf("**************************************************\n");
4604 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4605 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4606 gSystem->ChangeDirectory(cdir);
4607 return kFALSE;
4608 }
4609 } else {
4610 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4611 gSystem->ChangeDirectory(cdir);
4612 return kFALSE;
4613 }
4614 // Check for SETUP.C and execute
4615 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4616 printf("**************************************************\n");
4617 printf("*** Setup PAR archive %s\n", package);
4618 printf("**************************************************\n");
4619 gROOT->Macro("PROOF-INF/SETUP.C");
4620 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4621 } else {
4622 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4623 gSystem->ChangeDirectory(cdir);
4624 return kFALSE;
4625 }
4626 // Restore original workdir
4627 gSystem->ChangeDirectory(cdir);
4628 return kTRUE;
4629}
4630
c57f56b7 4631//______________________________________________________________________________
4632void AliAnalysisAlien::WriteExecutable()
4633{
4634// Generate the alien executable script.
5aa94dc3 4635 // Patch executable with -x to catch error code
4636 if (fExecutableCommand.Contains("root") &&
4637 fExecutableCommand.Contains("-q") &&
4638 !fExecutableCommand.Contains("-x")) fExecutableCommand += " -x";
c57f56b7 4639 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4640 ofstream out;
4641 out.open(fExecutable.Data(), ios::out);
4642 if (out.bad()) {
5513444a 4643 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4644 return;
4645 }
4646 out << "#!/bin/bash" << endl;
2a4d5166 4647 // Make sure we can properly compile par files
4648 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4649 out << "echo \"=========================================\"" << endl;
4650 out << "echo \"############## PATH : ##############\"" << endl;
4651 out << "echo $PATH" << endl;
4652 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4653 out << "echo $LD_LIBRARY_PATH" << endl;
4654 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4655 out << "echo $ROOTSYS" << endl;
4656 out << "echo \"############## which root : ##############\"" << endl;
4657 out << "which root" << endl;
4658 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4659 out << "echo $ALICE_ROOT" << endl;
4660 out << "echo \"############## which aliroot : ##############\"" << endl;
4661 out << "which aliroot" << endl;
9c5ddadc 4662 out << "echo \"############## system limits : ##############\"" << endl;
4663 out << "ulimit -a" << endl;
4664 out << "echo \"############## memory : ##############\"" << endl;
4665 out << "free -m" << endl;
c57f56b7 4666 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4667 out << fExecutableCommand << " ";
370c9880 4668 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4669 out << "RET=$?" << endl;
4670 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4671 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4672 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4673 out << " let sig=\"$RET - 128\""<<endl;
4674 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4675 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4676 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4677 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4678 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4679 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4680 out << " fi"<<endl;
4681 out << " exit $RET"<< endl;
4682 out << "fi" << endl << endl ;
4683 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4684 out << "echo \"############## memory after: ##############\"" << endl;
4685 out << "free -m" << endl;
c57f56b7 4686 }
4687 Bool_t copy = kTRUE;
d3b18c4c 4688 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4689 if (copy) {
4690 CdWork();
4691 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4692 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4693 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4694 workdir += fGridWorkingDir;
4695 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4696 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4697 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4698// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4699 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4700 executable.Data())) Fatal("","Terminating");
c57f56b7 4701 }
4702}
4703
0f389141 4704//______________________________________________________________________________
4705void AliAnalysisAlien::WriteMergeExecutable()
4706{
4707// Generate the alien executable script for the merging job.
4708 if (!fMergeViaJDL) return;
4709 TString mergeExec = fExecutable;
4710 mergeExec.ReplaceAll(".sh", "_merge.sh");
4711 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4712 ofstream out;
4713 out.open(mergeExec.Data(), ios::out);
4714 if (out.bad()) {
4715 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4716 return;
4717 }
4718 out << "#!/bin/bash" << endl;
2a4d5166 4719 // Make sure we can properly compile par files
4720 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4721 out << "echo \"=========================================\"" << endl;
4722 out << "echo \"############## PATH : ##############\"" << endl;
4723 out << "echo $PATH" << endl;
4724 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4725 out << "echo $LD_LIBRARY_PATH" << endl;
4726 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4727 out << "echo $ROOTSYS" << endl;
4728 out << "echo \"############## which root : ##############\"" << endl;
4729 out << "which root" << endl;
4730 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4731 out << "echo $ALICE_ROOT" << endl;
4732 out << "echo \"############## which aliroot : ##############\"" << endl;
4733 out << "which aliroot" << endl;
4734 out << "echo \"############## system limits : ##############\"" << endl;
4735 out << "ulimit -a" << endl;
4736 out << "echo \"############## memory : ##############\"" << endl;
4737 out << "free -m" << endl;
4738 out << "echo \"=========================================\"" << endl << endl;
0f389141 4739 TString mergeMacro = fExecutable;
4740 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4741 if (IsOneStageMerging())
4742 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4743 else
f47d5cb4 4744 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4745 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4746 out << "RET=$?" << endl;
4747 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4748 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4749 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4750 out << " let sig=\"$RET - 128\""<<endl;
4751 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4752 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4753 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4754 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4755 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4756 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4757 out << " fi"<<endl;
4758 out << " exit $RET"<< endl;
4759 out << "fi" << endl << endl ;
0f389141 4760 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4761 out << "echo \"############## memory after: ##############\"" << endl;
4762 out << "free -m" << endl;
0f389141 4763 }
4764 Bool_t copy = kTRUE;
d3b18c4c 4765 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4766 if (copy) {
4767 CdWork();
4768 TString workdir = gGrid->GetHomeDirectory();
4769 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4770 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4771 workdir += fGridWorkingDir;
4772 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4773 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4774 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4775// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4776 if (!copyLocal2Alien("WriteMergeExecutable",
4777 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4778 }
4779}
4780
c57f56b7 4781//______________________________________________________________________________
5513444a 4782void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4783{
4784// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4785// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4786// Next lines: full_path_to_dataset XXX (XXX is a string)
4787// To submit, one has to: submit jdl XXX for all lines
4788 ofstream out;
4789 out.open(filename, ios::out);
4790 if (out.bad()) {
4791 Error("WriteProductionFile", "Bad file name: %s", filename);
4792 return;
4793 }
d3b18c4c 4794 TString workdir;
4795 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4796 workdir = gGrid->GetHomeDirectory();
5513444a 4797 workdir += fGridWorkingDir;
f5e8c702 4798 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4799 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4800 out << locjdl << " " << njobspermaster << endl;
5513444a 4801 Int_t nmasterjobs = fInputFiles->GetEntries();
4802 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4803 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4804 runOutDir.ReplaceAll(".xml", "");
4805 if (fOutputToRunNo)
4806 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4807 else
4808 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4809 }
d3b18c4c 4810 if (gGrid) {
4811 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4812 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4813// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4814 if (!copyLocal2Alien("WriteProductionFile", filename,
4815 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4816 }
5513444a 4817}
4818
4819//______________________________________________________________________________
0f389141 4820void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4821{
4822// Generate the alien validation script.
4823 // Generate the validation script
4824 TObjString *os;
d3b18c4c 4825 if (fValidationScript.IsNull()) {
4826 fValidationScript = fExecutable;
4827 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4828 }
4829 TString validationScript = fValidationScript;
4830 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4831 if (!Connect()) {
4832 Error("WriteValidationScript", "Alien connection required");
4833 return;
4834 }
d3b18c4c 4835 if (!fTerminateFiles.IsNull()) {
4836 fTerminateFiles.Strip();
4837 fTerminateFiles.ReplaceAll(" ",",");
4838 }
a2f5fc01 4839 TString outStream = "";
4840 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4841 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4842 ofstream out;
0d5d317c 4843 out.open(validationScript, ios::out);
c57f56b7 4844 out << "#!/bin/bash" << endl;
4845 out << "##################################################" << endl;
4846 out << "validateout=`dirname $0`" << endl;
4847 out << "validatetime=`date`" << endl;
4848 out << "validated=\"0\";" << endl;
4849 out << "error=0" << endl;
4850 out << "if [ -z $validateout ]" << endl;
4851 out << "then" << endl;
4852 out << " validateout=\".\"" << endl;
4853 out << "fi" << endl << endl;
4854 out << "cd $validateout;" << endl;
4855 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4856 out << "echo \"*******************************************************\"" << outStream << endl;
4857 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4858 out << "" << endl;
a2f5fc01 4859 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4860 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4861 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4862 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4863 out << "ls -la ./" << outStream << endl;
4864 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4865 out << "##################################################" << endl;
ebec370a 4866 out << "" << endl;
4867
4868 out << "if [ ! -f stderr ] ; then" << endl;
4869 out << " error=1" << endl;
a2f5fc01 4870 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4871 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4872 out << "fi" << endl;
4873
b34c9f51 4874 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4875 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4876 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4877 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4878 out << "" << endl;
4879
ebec370a 4880 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4881 out << " error=1" << endl;
a2f5fc01 4882 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4883 out << " echo \"$parArch\" " << outStream << endl;
4884 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4885 out << "fi" << endl;
4886
4887 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4888 out << " error=1" << endl;
a2f5fc01 4889 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4890 out << " echo \"$segViol\" " << outStream << endl;
4891 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4892 out << "fi" << endl;
4893
4894 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4895 out << " error=1" << endl;
a2f5fc01 4896 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4897 out << " echo \"$segFault\" " << outStream << endl;
4898 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4899 out << "fi" << endl;
4900
b34c9f51 4901 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4902 out << " error=1" << endl;
a2f5fc01 4903 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4904 out << " echo \"$glibcErr\" " << outStream << endl;
4905 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4906 out << "fi" << endl;
4907
ebec370a 4908 // Part dedicated to the specific analyses running into the train
4909
d3b18c4c 4910 TString outputFiles = fOutputFiles;
4911 if (merge && !fTerminateFiles.IsNull()) {
4912 outputFiles += ",";
4913 outputFiles += fTerminateFiles;
4914 }
4915 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4916 TIter next1(arr);
a2f5fc01 4917 TString outputFile;
bbd88db8 4918 while (!merge && (os=(TObjString*)next1())) {
4919 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4920 outputFile = os->GetString();
4921 Int_t index = outputFile.Index("@");
4922 if (index > 0) outputFile.Remove(index);
bbd88db8 4923 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4924 if (outputFile.Contains("*")) continue;
4925 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4926 out << " error=1" << endl;
7c2cd90a 4927 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4928 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4929 out << "fi" << endl;
4930 }
4931 delete arr;
7c2cd90a 4932 out << "if ! [ -f outputs_valid ] ; then" << endl;
4933 out << " error=1" << endl;
4934 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4935 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4936 out << "fi" << endl;
923e2ca5 4937
c57f56b7 4938 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4939 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4940 if (!IsKeepLogs()) {
4941 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4942 outStream = "";
149d288c 4943 out << " rm -f std*" << endl;
4944 }
c57f56b7 4945 out << "fi" << endl;
4946
a2f5fc01 4947 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4948 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4949 out << "cd -" << endl;
4950 out << "exit $error" << endl;
4951 }
4952 Bool_t copy = kTRUE;
d3b18c4c 4953 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4954 if (copy) {
4955 CdWork();
4956 TString workdir = gGrid->GetHomeDirectory();
4957 workdir += fGridWorkingDir;
d3b18c4c 4958 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4959 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 4960// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4961 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
4962 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 4963 }
4964}