]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
set titles of variables in histograms to italic
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
715da79c 32#include "TInterpreter.h"
c57f56b7 33#include "TFile.h"
830acc4c 34#include "TFileCollection.h"
3bdcb562 35#include "TChain.h"
c57f56b7 36#include "TObjString.h"
37#include "TObjArray.h"
77f34eae 38#include "TMacro.h"
c57f56b7 39#include "TGrid.h"
40#include "TGridResult.h"
41#include "TGridCollection.h"
42#include "TGridJDL.h"
d2a409b2 43#include "TGridJobStatusList.h"
44#include "TGridJobStatus.h"
c57f56b7 45#include "TFileMerger.h"
46#include "AliAnalysisManager.h"
77f34eae 47#include "AliAnalysisTaskCfg.h"
bb885a9e 48#include "AliVEventHandler.h"
49#include "AliAnalysisDataContainer.h"
65eb22f8 50#include "AliMultiInputEventHandler.h"
c57f56b7 51
c82bb898 52using std::ofstream;
53using std::ifstream;
54using std::ios;
55using std::endl;
c57f56b7 56ClassImp(AliAnalysisAlien)
6c395669 57#if 0
58;
59#endif
c57f56b7 60
6c395669 61namespace {
62 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
63 {
64 TString sl(Form("file:%s", loc));
65 TString sr(Form("alien://%s", rem));
66 Bool_t ret = TFile::Cp(sl, sr);
67 if (!ret) {
68 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
69 }
70 return ret;
71 }
72}
73
c57f56b7 74//______________________________________________________________________________
75AliAnalysisAlien::AliAnalysisAlien()
76 :AliAnalysisGrid(),
77 fGridJDL(NULL),
0f389141 78 fMergingJDL(NULL),
c57f56b7 79 fPrice(0),
80 fTTL(0),
81 fSplitMaxInputFileNumber(0),
82 fMaxInitFailed(0),
83 fMasterResubmitThreshold(0),
bb885a9e 84 fNtestFiles(0),
319593fb 85 fNrunsPerMaster(0),
16a4353c 86 fMaxMergeFiles(0),
b3e07543 87 fMaxMergeStages(0),
d2a409b2 88 fNsubmitted(0),
a3e84053 89 fProductionMode(0),
cd11251e 90 fOutputToRunNo(0),
0f389141 91 fMergeViaJDL(0),
92 fFastReadOption(0),
e1c22e21 93 fOverwriteMode(1),
149d288c 94 fNreplicas(2),
3bdcb562 95 fNproofWorkers(0),
27734f0e 96 fNproofWorkersPerSlave(0),
3bdcb562 97 fProofReset(0),
715da79c 98 fNMCevents(0),
99 fNMCjobs(0),
c57f56b7 100 fRunNumbers(),
101 fExecutable(),
0a1c1f7f 102 fExecutableCommand(),
c57f56b7 103 fArguments(),
631c0b05 104 fExecutableArgs(),
c57f56b7 105 fAnalysisMacro(),
106 fAnalysisSource(),
d3b18c4c 107 fValidationScript(),
d5c6455a 108 fAdditionalRootLibs(),
c57f56b7 109 fAdditionalLibs(),
715da79c 110 fGeneratorLibs(),
c57f56b7 111 fSplitMode(),
112 fAPIVersion(),
113 fROOTVersion(),
114 fAliROOTVersion(),
648174cf 115 fExternalPackages(),
c57f56b7 116 fUser(),
117 fGridWorkingDir(),
118 fGridDataDir(),
119 fDataPattern(),
120 fGridOutputDir(),
121 fOutputArchive(),
122 fOutputFiles(),
123 fInputFormat(),
e7c71df0 124 fDatasetName(),
c57f56b7 125 fJDLName(),
d3b18c4c 126 fTerminateFiles(),
bb885a9e 127 fMergeExcludes(),
37d21c01 128 fRegisterExcludes(),
f965131e 129 fIncludePath(),
bb885a9e 130 fCloseSE(),
0df6ccf2 131 fFriendChainName(),
c6cb3634 132 fJobTag(),
648174cf 133 fOutputSingle(),
5fce53f4 134 fRunPrefix(),
3bdcb562 135 fProofCluster(),
136 fProofDataSet(),
137 fFileForTestMode(),
3bdcb562 138 fAliRootMode(),
d88e4159 139 fProofProcessOpt(),
f47d5cb4 140 fMergeDirName(),
4e5c5506 141 fInputFiles(0),
28ce6c9c 142 fPackages(0),
77f34eae 143 fModules(0),
d88e4159 144 fProofParam(),
a41b2882 145 fDropToShell(true),
715da79c 146 fMCLoop(false),
a41b2882 147 fGridJobIDs(""),
148 fGridStages(""),
780ec36d 149 fFriendLibs(""),
150 fTreeName()
c57f56b7 151{
152// Dummy ctor.
153 SetDefaults();
154}
155
156//______________________________________________________________________________
157AliAnalysisAlien::AliAnalysisAlien(const char *name)
158 :AliAnalysisGrid(name),
159 fGridJDL(NULL),
0f389141 160 fMergingJDL(NULL),
c57f56b7 161 fPrice(0),
162 fTTL(0),
163 fSplitMaxInputFileNumber(0),
164 fMaxInitFailed(0),
165 fMasterResubmitThreshold(0),
bb885a9e 166 fNtestFiles(0),
319593fb 167 fNrunsPerMaster(0),
16a4353c 168 fMaxMergeFiles(0),
b3e07543 169 fMaxMergeStages(0),
d2a409b2 170 fNsubmitted(0),
a3e84053 171 fProductionMode(0),
cd11251e 172 fOutputToRunNo(0),
0f389141 173 fMergeViaJDL(0),
174 fFastReadOption(0),
e1c22e21 175 fOverwriteMode(1),
149d288c 176 fNreplicas(2),
3bdcb562 177 fNproofWorkers(0),
27734f0e 178 fNproofWorkersPerSlave(0),
3bdcb562 179 fProofReset(0),
715da79c 180 fNMCevents(0),
181 fNMCjobs(0),
c57f56b7 182 fRunNumbers(),
183 fExecutable(),
0a1c1f7f 184 fExecutableCommand(),
c57f56b7 185 fArguments(),
631c0b05 186 fExecutableArgs(),
c57f56b7 187 fAnalysisMacro(),
188 fAnalysisSource(),
d3b18c4c 189 fValidationScript(),
d5c6455a 190 fAdditionalRootLibs(),
c57f56b7 191 fAdditionalLibs(),
715da79c 192 fGeneratorLibs(),
c57f56b7 193 fSplitMode(),
194 fAPIVersion(),
195 fROOTVersion(),
196 fAliROOTVersion(),
648174cf 197 fExternalPackages(),
c57f56b7 198 fUser(),
199 fGridWorkingDir(),
200 fGridDataDir(),
201 fDataPattern(),
202 fGridOutputDir(),
203 fOutputArchive(),
204 fOutputFiles(),
205 fInputFormat(),
e7c71df0 206 fDatasetName(),
c57f56b7 207 fJDLName(),
d3b18c4c 208 fTerminateFiles(),
bb885a9e 209 fMergeExcludes(),
37d21c01 210 fRegisterExcludes(),
f965131e 211 fIncludePath(),
bb885a9e 212 fCloseSE(),
0df6ccf2 213 fFriendChainName(),
c6cb3634 214 fJobTag(),
648174cf 215 fOutputSingle(),
5fce53f4 216 fRunPrefix(),
3bdcb562 217 fProofCluster(),
218 fProofDataSet(),
219 fFileForTestMode(),
3bdcb562 220 fAliRootMode(),
d88e4159 221 fProofProcessOpt(),
f47d5cb4 222 fMergeDirName(),
4e5c5506 223 fInputFiles(0),
28ce6c9c 224 fPackages(0),
77f34eae 225 fModules(0),
d88e4159 226 fProofParam(),
0ad3ea1f 227 fDropToShell(true),
715da79c 228 fMCLoop(false),
0ad3ea1f 229 fGridJobIDs(""),
a41b2882 230 fGridStages(""),
780ec36d 231 fFriendLibs(""),
232 fTreeName()
c57f56b7 233{
234// Default ctor.
235 SetDefaults();
236}
237
238//______________________________________________________________________________
239AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
240 :AliAnalysisGrid(other),
241 fGridJDL(NULL),
0f389141 242 fMergingJDL(NULL),
c57f56b7 243 fPrice(other.fPrice),
244 fTTL(other.fTTL),
245 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
246 fMaxInitFailed(other.fMaxInitFailed),
247 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 248 fNtestFiles(other.fNtestFiles),
319593fb 249 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 250 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 251 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 252 fNsubmitted(other.fNsubmitted),
a3e84053 253 fProductionMode(other.fProductionMode),
cd11251e 254 fOutputToRunNo(other.fOutputToRunNo),
0f389141 255 fMergeViaJDL(other.fMergeViaJDL),
256 fFastReadOption(other.fFastReadOption),
257 fOverwriteMode(other.fOverwriteMode),
149d288c 258 fNreplicas(other.fNreplicas),
3bdcb562 259 fNproofWorkers(other.fNproofWorkers),
27734f0e 260 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 261 fProofReset(other.fProofReset),
715da79c 262 fNMCevents(other.fNMCevents),
263 fNMCjobs(other.fNMCjobs),
c57f56b7 264 fRunNumbers(other.fRunNumbers),
265 fExecutable(other.fExecutable),
0a1c1f7f 266 fExecutableCommand(other.fExecutableCommand),
c57f56b7 267 fArguments(other.fArguments),
631c0b05 268 fExecutableArgs(other.fExecutableArgs),
c57f56b7 269 fAnalysisMacro(other.fAnalysisMacro),
270 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 271 fValidationScript(other.fValidationScript),
d5c6455a 272 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 273 fAdditionalLibs(other.fAdditionalLibs),
715da79c 274 fGeneratorLibs(other.fGeneratorLibs),
c57f56b7 275 fSplitMode(other.fSplitMode),
276 fAPIVersion(other.fAPIVersion),
277 fROOTVersion(other.fROOTVersion),
278 fAliROOTVersion(other.fAliROOTVersion),
648174cf 279 fExternalPackages(other.fExternalPackages),
c57f56b7 280 fUser(other.fUser),
281 fGridWorkingDir(other.fGridWorkingDir),
282 fGridDataDir(other.fGridDataDir),
283 fDataPattern(other.fDataPattern),
284 fGridOutputDir(other.fGridOutputDir),
285 fOutputArchive(other.fOutputArchive),
286 fOutputFiles(other.fOutputFiles),
287 fInputFormat(other.fInputFormat),
e7c71df0 288 fDatasetName(other.fDatasetName),
c57f56b7 289 fJDLName(other.fJDLName),
d3b18c4c 290 fTerminateFiles(other.fTerminateFiles),
bb885a9e 291 fMergeExcludes(other.fMergeExcludes),
37d21c01 292 fRegisterExcludes(other.fRegisterExcludes),
f965131e 293 fIncludePath(other.fIncludePath),
bb885a9e 294 fCloseSE(other.fCloseSE),
0df6ccf2 295 fFriendChainName(other.fFriendChainName),
c6cb3634 296 fJobTag(other.fJobTag),
648174cf 297 fOutputSingle(other.fOutputSingle),
5fce53f4 298 fRunPrefix(other.fRunPrefix),
3bdcb562 299 fProofCluster(other.fProofCluster),
300 fProofDataSet(other.fProofDataSet),
301 fFileForTestMode(other.fFileForTestMode),
3bdcb562 302 fAliRootMode(other.fAliRootMode),
d88e4159 303 fProofProcessOpt(other.fProofProcessOpt),
f47d5cb4 304 fMergeDirName(other.fMergeDirName),
4e5c5506 305 fInputFiles(0),
28ce6c9c 306 fPackages(0),
77f34eae 307 fModules(0),
d88e4159 308 fProofParam(),
0ad3ea1f 309 fDropToShell(other.fDropToShell),
715da79c 310 fMCLoop(other.fMCLoop),
0ad3ea1f 311 fGridJobIDs(other.fGridJobIDs),
a41b2882 312 fGridStages(other.fGridStages),
780ec36d 313 fFriendLibs(other.fFriendLibs),
314 fTreeName(other.fTreeName)
c57f56b7 315{
316// Copy ctor.
317 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 318 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 319 fRunRange[0] = other.fRunRange[0];
320 fRunRange[1] = other.fRunRange[1];
c57f56b7 321 if (other.fInputFiles) {
322 fInputFiles = new TObjArray();
323 TIter next(other.fInputFiles);
324 TObject *obj;
325 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
326 fInputFiles->SetOwner();
327 }
4e5c5506 328 if (other.fPackages) {
329 fPackages = new TObjArray();
330 TIter next(other.fPackages);
331 TObject *obj;
332 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
333 fPackages->SetOwner();
334 }
77f34eae 335 if (other.fModules) {
336 fModules = new TObjArray();
337 fModules->SetOwner();
338 TIter next(other.fModules);
339 AliAnalysisTaskCfg *mod, *crt;
340 while ((crt=(AliAnalysisTaskCfg*)next())) {
341 mod = new AliAnalysisTaskCfg(*crt);
342 fModules->Add(mod);
343 }
344 }
c57f56b7 345}
346
347//______________________________________________________________________________
348AliAnalysisAlien::~AliAnalysisAlien()
349{
350// Destructor.
77f34eae 351 delete fGridJDL;
352 delete fMergingJDL;
353 delete fInputFiles;
354 delete fPackages;
355 delete fModules;
28ce6c9c 356 fProofParam.DeleteAll();
c57f56b7 357}
358
359//______________________________________________________________________________
360AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
361{
362// Assignment.
363 if (this != &other) {
364 AliAnalysisGrid::operator=(other);
365 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 366 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 367 fPrice = other.fPrice;
368 fTTL = other.fTTL;
369 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
370 fMaxInitFailed = other.fMaxInitFailed;
371 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 372 fNtestFiles = other.fNtestFiles;
a3e84053 373 fNrunsPerMaster = other.fNrunsPerMaster;
374 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 375 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 376 fNsubmitted = other.fNsubmitted;
377 fProductionMode = other.fProductionMode;
cd11251e 378 fOutputToRunNo = other.fOutputToRunNo;
0f389141 379 fMergeViaJDL = other.fMergeViaJDL;
380 fFastReadOption = other.fFastReadOption;
381 fOverwriteMode = other.fOverwriteMode;
149d288c 382 fNreplicas = other.fNreplicas;
3bdcb562 383 fNproofWorkers = other.fNproofWorkers;
27734f0e 384 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 385 fProofReset = other.fProofReset;
715da79c 386 fNMCevents = other.fNMCevents;
387 fNMCjobs = other.fNMCjobs;
c57f56b7 388 fRunNumbers = other.fRunNumbers;
389 fExecutable = other.fExecutable;
0a1c1f7f 390 fExecutableCommand = other.fExecutableCommand;
c57f56b7 391 fArguments = other.fArguments;
631c0b05 392 fExecutableArgs = other.fExecutableArgs;
c57f56b7 393 fAnalysisMacro = other.fAnalysisMacro;
394 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 395 fValidationScript = other.fValidationScript;
d5c6455a 396 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 397 fAdditionalLibs = other.fAdditionalLibs;
715da79c 398 fGeneratorLibs = other.fGeneratorLibs;
c57f56b7 399 fSplitMode = other.fSplitMode;
400 fAPIVersion = other.fAPIVersion;
401 fROOTVersion = other.fROOTVersion;
402 fAliROOTVersion = other.fAliROOTVersion;
648174cf 403 fExternalPackages = other.fExternalPackages;
c57f56b7 404 fUser = other.fUser;
405 fGridWorkingDir = other.fGridWorkingDir;
406 fGridDataDir = other.fGridDataDir;
407 fDataPattern = other.fDataPattern;
408 fGridOutputDir = other.fGridOutputDir;
409 fOutputArchive = other.fOutputArchive;
410 fOutputFiles = other.fOutputFiles;
411 fInputFormat = other.fInputFormat;
e7c71df0 412 fDatasetName = other.fDatasetName;
c57f56b7 413 fJDLName = other.fJDLName;
d3b18c4c 414 fTerminateFiles = other.fTerminateFiles;
bb885a9e 415 fMergeExcludes = other.fMergeExcludes;
37d21c01 416 fRegisterExcludes = other.fRegisterExcludes;
f965131e 417 fIncludePath = other.fIncludePath;
bb885a9e 418 fCloseSE = other.fCloseSE;
0df6ccf2 419 fFriendChainName = other.fFriendChainName;
c6cb3634 420 fJobTag = other.fJobTag;
648174cf 421 fOutputSingle = other.fOutputSingle;
5fce53f4 422 fRunPrefix = other.fRunPrefix;
3bdcb562 423 fProofCluster = other.fProofCluster;
424 fProofDataSet = other.fProofDataSet;
425 fFileForTestMode = other.fFileForTestMode;
3bdcb562 426 fAliRootMode = other.fAliRootMode;
d88e4159 427 fProofProcessOpt = other.fProofProcessOpt;
f47d5cb4 428 fMergeDirName = other.fMergeDirName;
d88e4159 429 fDropToShell = other.fDropToShell;
715da79c 430 fMCLoop = other.fMCLoop;
0ad3ea1f 431 fGridJobIDs = other.fGridJobIDs;
432 fGridStages = other.fGridStages;
a41b2882 433 fFriendLibs = other.fFriendLibs;
780ec36d 434 fTreeName = other.fTreeName;
c57f56b7 435 if (other.fInputFiles) {
436 fInputFiles = new TObjArray();
437 TIter next(other.fInputFiles);
438 TObject *obj;
439 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
440 fInputFiles->SetOwner();
441 }
4e5c5506 442 if (other.fPackages) {
443 fPackages = new TObjArray();
444 TIter next(other.fPackages);
445 TObject *obj;
446 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
447 fPackages->SetOwner();
448 }
77f34eae 449 if (other.fModules) {
450 fModules = new TObjArray();
451 fModules->SetOwner();
452 TIter next(other.fModules);
453 AliAnalysisTaskCfg *mod, *crt;
454 while ((crt=(AliAnalysisTaskCfg*)next())) {
455 mod = new AliAnalysisTaskCfg(*crt);
456 fModules->Add(mod);
457 }
458 }
c57f56b7 459 }
460 return *this;
461}
462
37d21c01 463//______________________________________________________________________________
464void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
465{
466// Add a single additional library to be loaded. Extension must be present.
467 TString lib(name);
468 if (!lib.Contains(".")) {
469 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
470 return;
471 }
472 if (fAdditionalLibs.Contains(name)) {
473 Warning("AddAdditionalLibrary", "Library %s already added.", name);
474 return;
475 }
476 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
477 fAdditionalLibs += lib;
478}
479
77f34eae 480//______________________________________________________________________________
481void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
482{
483// Adding a module. Checks if already existing. Becomes owned by this.
484 if (!module) return;
485 if (GetModule(module->GetName())) {
486 Error("AddModule", "A module having the same name %s already added", module->GetName());
487 return;
488 }
489 if (!fModules) {
490 fModules = new TObjArray();
491 fModules->SetOwner();
492 }
493 fModules->Add(module);
494}
495
496//______________________________________________________________________________
497void AliAnalysisAlien::AddModules(TObjArray *list)
498{
499// Adding a list of modules. Checks if already existing. Becomes owned by this.
500 TIter next(list);
501 AliAnalysisTaskCfg *module;
502 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
503}
504
505//______________________________________________________________________________
506Bool_t AliAnalysisAlien::CheckDependencies()
507{
508// Check if all dependencies are satisfied. Reorder modules if needed.
509 Int_t nmodules = GetNmodules();
510 if (!nmodules) {
511 Warning("CheckDependencies", "No modules added yet to check their dependencies");
512 return kTRUE;
513 }
514 AliAnalysisTaskCfg *mod = 0;
515 AliAnalysisTaskCfg *dep = 0;
516 TString depname;
517 Int_t i, j, k;
518 for (i=0; i<nmodules; i++) {
519 mod = (AliAnalysisTaskCfg*) fModules->At(i);
520 Int_t ndeps = mod->GetNdeps();
521 Int_t istart = i;
522 for (j=0; j<ndeps; j++) {
523 depname = mod->GetDependency(j);
524 dep = GetModule(depname);
525 if (!dep) {
526 Error("CheckDependencies","Dependency %s not added for module %s",
527 depname.Data(), mod->GetName());
528 return kFALSE;
529 }
530 if (dep->NeedsDependency(mod->GetName())) {
531 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
532 mod->GetName(), dep->GetName());
533 return kFALSE;
534 }
535 Int_t idep = fModules->IndexOf(dep);
536 // The dependency task must come first
537 if (idep>i) {
538 // Remove at idep and move all objects below up one slot
539 // down to index i included.
540 fModules->RemoveAt(idep);
3e40fd4c 541 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 542 fModules->AddAt(dep, i++);
543 }
544 //Redo from istart if dependencies were inserted
545 if (i>istart) i=istart-1;
546 }
547 }
548 return kTRUE;
549}
550
3e40fd4c 551//______________________________________________________________________________
552AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
553{
554// Create the analysis manager and optionally execute the macro in filename.
555 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
715da79c 556 if (mgr) {
557 mgr->SetMCLoop(fMCLoop);
558 return mgr;
559 }
3e40fd4c 560 mgr = new AliAnalysisManager(name);
561 mgr->SetGridHandler((AliAnalysisGrid*)this);
715da79c 562 mgr->SetMCLoop(fMCLoop);
3e40fd4c 563 if (strlen(filename)) {
564 TString line = gSystem->ExpandPathName(filename);
565 line.Prepend(".x ");
566 gROOT->ProcessLine(line.Data());
567 }
568 return mgr;
569}
570
77f34eae 571//______________________________________________________________________________
572Int_t AliAnalysisAlien::GetNmodules() const
573{
574// Get number of modules.
575 if (!fModules) return 0;
576 return fModules->GetEntries();
577}
578
579//______________________________________________________________________________
580AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
581{
582// Get a module by name.
583 if (!fModules) return 0;
584 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
585}
586
587//______________________________________________________________________________
588Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
589{
590// Load a given module.
591 if (mod->IsLoaded()) return kTRUE;
192ddca2 592 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
593 if (!mgr) {
594 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
595 return kFALSE;
596 }
77f34eae 597 Int_t ndeps = mod->GetNdeps();
598 TString depname;
599 for (Int_t j=0; j<ndeps; j++) {
600 depname = mod->GetDependency(j);
601 AliAnalysisTaskCfg *dep = GetModule(depname);
602 if (!dep) {
603 Error("LoadModule","Dependency %s not existing for module %s",
604 depname.Data(), mod->GetName());
605 return kFALSE;
606 }
607 if (!LoadModule(dep)) {
608 Error("LoadModule","Dependency %s for module %s could not be loaded",
609 depname.Data(), mod->GetName());
610 return kFALSE;
611 }
612 }
613 // Load libraries for the module
614 if (!mod->CheckLoadLibraries()) {
615 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
616 return kFALSE;
617 }
192ddca2 618 // Check if a custom file name was requested
619 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
620
621 // Check if a custom terminate file name was requested
622 if (strlen(mod->GetTerminateFileName())) {
623 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
624 fTerminateFiles += mod->GetTerminateFileName();
625 }
626
77f34eae 627 // Execute the macro
628 if (mod->ExecuteMacro()<0) {
629 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
630 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
631 return kFALSE;
632 }
633 // Configure dependencies
634 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
635 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
636 mod->GetConfigMacro()->GetTitle(), mod->GetName());
637 return kFALSE;
638 }
5e8c7c22 639 // Adjust extra libraries
640 Int_t nlibs = mod->GetNlibs();
641 TString lib;
642 for (Int_t i=0; i<nlibs; i++) {
643 lib = mod->GetLibrary(i);
5e8c7c22 644 lib = Form("lib%s.so", lib.Data());
c9d6bd12 645 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 646 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
647 fAdditionalLibs += lib;
648 }
77f34eae 649 return kTRUE;
650}
651
139fe6ba 652//______________________________________________________________________________
653Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
654{
655// Generate the full train.
656 fAdditionalLibs = "";
657 if (!LoadModules()) return kFALSE;
658 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
659 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 660 mgr->RunLocalInit();
139fe6ba 661 mgr->PrintStatus();
662 Int_t productionMode = fProductionMode;
663 SetProductionMode();
664 TString macro = fAnalysisMacro;
665 TString executable = fExecutable;
666 TString validation = fValidationScript;
667 TString execCommand = fExecutableCommand;
668 SetAnalysisMacro(Form("%s.C", name));
669 SetExecutable(Form("%s.sh", name));
a8e31fb0 670// SetExecutableCommand("aliroot -b -q ");
139fe6ba 671 SetValidationScript(Form("%s_validation.sh", name));
672 StartAnalysis();
673 SetProductionMode(productionMode);
674 fAnalysisMacro = macro;
675 fExecutable = executable;
676 fExecutableCommand = execCommand;
677 fValidationScript = validation;
678 return kTRUE;
679}
680
77f34eae 681//______________________________________________________________________________
21ca8e59 682Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 683{
684// Generate test macros for a single module or for the full train.
5e8c7c22 685 fAdditionalLibs = "";
77f34eae 686 if (strlen(modname)) {
21ca8e59 687 if (!CheckDependencies()) return kFALSE;
688 AliAnalysisTaskCfg *mod = GetModule(modname);
689 if (!mod) {
690 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
691 return kFALSE;
692 }
693 if (!LoadModule(mod)) return kFALSE;
a41b2882 694 if (!LoadFriendLibs()) return kFALSE;
21ca8e59 695 } else if (!LoadModules()) return kFALSE;
696 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
697 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 698 mgr->RunLocalInit();
21ca8e59 699 mgr->PrintStatus();
700 SetLocalTest(kTRUE);
701 Int_t productionMode = fProductionMode;
702 SetProductionMode();
703 TString macro = fAnalysisMacro;
704 TString executable = fExecutable;
705 TString validation = fValidationScript;
706 TString execCommand = fExecutableCommand;
707 SetAnalysisMacro(Form("%s.C", name));
708 SetExecutable(Form("%s.sh", name));
d66aa2a9 709 fOutputFiles = GetListOfFiles("outaod");
710 // Add extra files registered to the analysis manager
711 TString extra = GetListOfFiles("ext");
712 if (!extra.IsNull()) {
713 extra.ReplaceAll(".root", "*.root");
714 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
715 fOutputFiles += extra;
716 }
a8e31fb0 717// SetExecutableCommand("aliroot -b -q ");
21ca8e59 718 SetValidationScript(Form("%s_validation.sh", name));
719 WriteAnalysisFile();
720 WriteAnalysisMacro();
721 WriteExecutable();
722 WriteValidationScript();
ffbe06b9 723 WriteMergingMacro();
724 WriteMergeExecutable();
725 WriteValidationScript(kTRUE);
21ca8e59 726 SetLocalTest(kFALSE);
727 SetProductionMode(productionMode);
728 fAnalysisMacro = macro;
729 fExecutable = executable;
730 fExecutableCommand = execCommand;
731 fValidationScript = validation;
77f34eae 732 return kTRUE;
733}
734
735//______________________________________________________________________________
736Bool_t AliAnalysisAlien::LoadModules()
737{
738// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 739 fAdditionalLibs = "";
21ca8e59 740 Int_t nmodules = GetNmodules();
741 if (!nmodules) {
742 Warning("LoadModules", "No module to be loaded");
743 return kTRUE;
744 }
3e40fd4c 745 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
746 if (!mgr) {
747 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
748 return kFALSE;
749 }
77f34eae 750 if (!CheckDependencies()) return kFALSE;
21ca8e59 751 nmodules = GetNmodules();
77f34eae 752 AliAnalysisTaskCfg *mod;
21ca8e59 753 for (Int_t imod=0; imod<nmodules; imod++) {
754 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 755 if (!LoadModule(mod)) return kFALSE;
756 }
a41b2882 757 // Load additional friend libraries
758 return LoadFriendLibs();
77f34eae 759}
760
a41b2882 761//______________________________________________________________________________
762Bool_t AliAnalysisAlien::LoadFriendLibs() const
763{
764// Load libraries required for reading friends.
765 if (fFriendLibs.Length()) {
766 TObjArray *list = 0;
767 TString lib;
768 if (fFriendLibs.Contains(",")) list = fFriendLibs.Tokenize(",");
769 else list = fFriendLibs.Tokenize(" ");
770 for (Int_t ilib=0; ilib<list->GetEntriesFast(); ilib++) {
771 lib = list->At(ilib)->GetName();
772 lib.ReplaceAll(".so","");
773 lib.ReplaceAll(" ","");
774 if (lib.BeginsWith("lib")) lib.Remove(0, 3);
775 lib.Prepend("lib");
776 Int_t loaded = strlen(gSystem->GetLibraries(lib,"",kFALSE));
777 if (!loaded) loaded = gSystem->Load(lib);
778 if (loaded < 0) {
779 Error("LoadModules", "Cannot load library for friends %s", lib.Data());
780 return kFALSE;
781 }
782 }
783 delete list;
784 }
785 return kTRUE;
786}
787
74013535 788//______________________________________________________________________________
789void AliAnalysisAlien::SetRunPrefix(const char *prefix)
790{
791// Set the run number format. Can be a prefix or a format like "%09d"
792 fRunPrefix = prefix;
793 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
794}
795
f965131e 796//______________________________________________________________________________
797void AliAnalysisAlien::AddIncludePath(const char *path)
798{
799// Add include path in the remote analysis macro.
800 TString p(path);
801 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
802 else fIncludePath += Form("-I%s ", path);
803}
804
c57f56b7 805//______________________________________________________________________________
806void AliAnalysisAlien::AddRunNumber(Int_t run)
807{
808// Add a run number to the list of runs to be processed.
809 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 810 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 811}
812
98690895 813//______________________________________________________________________________
814void AliAnalysisAlien::AddRunList(const char* runList)
815{
816// Add several runs into the list of runs; they are expected to be separated by a blank character.
817 TString sList = runList;
818 TObjArray *list = sList.Tokenize(" ");
819 Int_t n = list->GetEntries();
820 for (Int_t i = 0; i < n; i++) {
821 TObjString *os = (TObjString*)list->At(i);
822 AddRunNumber(os->GetString().Atoi());
823 }
824 delete list;
825}
826
ee75cfc3 827//______________________________________________________________________________
828void AliAnalysisAlien::AddRunNumber(const char* run)
829{
830// Add a run number to the list of runs to be processed.
7aa5d539 831 TString runs = run;
832 TObjString *os;
833 TObjArray *arr = runs.Tokenize(" ");
834 TIter next(arr);
835 TString prefix;
836 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
837 while ((os=(TObjString*)next())){
838 if (fRunNumbers.Length()) fRunNumbers += " ";
839 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
840 }
841 delete arr;
ee75cfc3 842}
843
c57f56b7 844//______________________________________________________________________________
845void AliAnalysisAlien::AddDataFile(const char *lfn)
846{
847// Adds a data file to the input to be analysed. The file should be a valid LFN
848// or point to an existing file in the alien workdir.
849 if (!fInputFiles) fInputFiles = new TObjArray();
850 fInputFiles->Add(new TObjString(lfn));
851}
648174cf 852
853//______________________________________________________________________________
854void AliAnalysisAlien::AddExternalPackage(const char *package)
855{
856// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
857 if (fExternalPackages) fExternalPackages += " ";
858 fExternalPackages += package;
859}
860
c57f56b7 861//______________________________________________________________________________
862Bool_t AliAnalysisAlien::Connect()
863{
864// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
865 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 866 if (fProductionMode) return kTRUE;
c57f56b7 867 if (!gGrid) {
868 Info("Connect", "Trying to connect to AliEn ...");
869 TGrid::Connect("alien://");
870 }
871 if (!gGrid || !gGrid->IsConnected()) {
872 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
873 return kFALSE;
874 }
875 fUser = gGrid->GetUser();
876 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
877 return kTRUE;
878}
879
880//______________________________________________________________________________
881void AliAnalysisAlien::CdWork()
882{
883// Check validity of alien workspace. Create directory if possible.
884 if (!Connect()) {
885 Error("CdWork", "Alien connection required");
886 return;
887 }
888 TString homedir = gGrid->GetHomeDirectory();
889 TString workdir = homedir + fGridWorkingDir;
923e2ca5 890 if (DirectoryExists(workdir)) {
891 gGrid->Cd(workdir);
892 return;
893 }
894 // Work directory not existing - create it
895 gGrid->Cd(homedir);
b93f8109 896 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 897 gGrid->Cd(fGridWorkingDir);
d3b18c4c 898 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 899 } else {
d3b18c4c 900 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 901 workdir.Data(), homedir.Data());
902 fGridWorkingDir = "";
903 }
c57f56b7 904}
905
348be253 906//______________________________________________________________________________
907Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
908{
909// Check if file copying is possible.
d3b18c4c 910 if (fProductionMode) return kTRUE;
b5ae4fe0 911 TString salienpath(alienpath);
912 if (salienpath.Contains(" ")) {
913 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
914 return kFALSE;
915 }
348be253 916 if (!Connect()) {
917 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
918 return kFALSE;
919 }
d3339be3 920 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
921 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 922 // Check if alien_CLOSE_SE is defined
923 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
924 if (!closeSE.IsNull()) {
925 Info("CheckFileCopy", "Your current close storage is pointing to: \
926 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
927 } else {
928 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
929 }
930 // Check if grid directory exists.
931 if (!DirectoryExists(alienpath)) {
932 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
933 return kFALSE;
934 }
b5ae4fe0 935 TString stest = "plugin_test_copy";
936 TFile f(stest, "RECREATE");
348be253 937 // User may not have write permissions to current directory
938 if (f.IsZombie()) {
939 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
940 gSystem->WorkingDirectory());
941 return kFALSE;
942 }
943 f.Close();
b5ae4fe0 944 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
945 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 946 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 947 \n# 1. Make sure you have write permissions there. If this is the case: \
948 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
949 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
950 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
951 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 952 gSystem->Unlink(stest.Data());
348be253 953 return kFALSE;
954 }
b5ae4fe0 955 gSystem->Unlink(stest.Data());
956 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 957 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 958 return kTRUE;
959}
960
c57f56b7 961//______________________________________________________________________________
962Bool_t AliAnalysisAlien::CheckInputData()
963{
964// Check validity of input data. If necessary, create xml files.
d3b18c4c 965 if (fProductionMode) return kTRUE;
a8739e8a 966 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 967 if (!fGridDataDir.Length()) {
968 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
969 return kFALSE;
970 }
9f020952 971 if (fMergeViaJDL) {
972 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
973 return kFALSE;
974 }
d2a409b2 975 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 976 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
977 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 978 return kTRUE;
c57f56b7 979 }
980 // Process declared files
a2f5fc01 981 Bool_t isCollection = kFALSE;
982 Bool_t isXml = kFALSE;
983 Bool_t useTags = kFALSE;
c57f56b7 984 Bool_t checked = kFALSE;
d3b18c4c 985 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 986 TString file;
987 TString workdir = gGrid->GetHomeDirectory();
988 workdir += fGridWorkingDir;
989 if (fInputFiles) {
990 TObjString *objstr;
991 TIter next(fInputFiles);
992 while ((objstr=(TObjString*)next())) {
993 file = workdir;
994 file += "/";
995 file += objstr->GetString();
996 // Store full lfn path
997 if (FileExists(file)) objstr->SetString(file);
998 else {
999 file = objstr->GetName();
1000 if (!FileExists(objstr->GetName())) {
1001 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
1002 objstr->GetName(), workdir.Data());
1003 return kFALSE;
1004 }
1005 }
1006 Bool_t iscoll, isxml, usetags;
1007 CheckDataType(file, iscoll, isxml, usetags);
1008 if (!checked) {
1009 checked = kTRUE;
a2f5fc01 1010 isCollection = iscoll;
1011 isXml = isxml;
1012 useTags = usetags;
1013 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1014 } else {
a2f5fc01 1015 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 1016 Error("CheckInputData", "Some conflict was found in the types of inputs");
1017 return kFALSE;
1018 }
1019 }
1020 }
1021 }
1022 // Process requested run numbers
a8739e8a 1023 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 1024 // Check validity of alien data directory
1025 if (!fGridDataDir.Length()) {
1026 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
1027 return kFALSE;
1028 }
923e2ca5 1029 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 1030 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
1031 return kFALSE;
1032 }
a2f5fc01 1033 if (isCollection) {
c57f56b7 1034 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
1035 return kFALSE;
1036 }
1037
a2f5fc01 1038 if (checked && !isXml) {
c57f56b7 1039 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
1040 return kFALSE;
1041 }
1042 // Check validity of run number(s)
1043 TObjArray *arr;
1044 TObjString *os;
74013535 1045 TString format;
319593fb 1046 Int_t nruns = 0;
904f9f5f 1047 TString schunk, schunk2;
c57f56b7 1048 TString path;
1049 if (!checked) {
1050 checked = kTRUE;
a2f5fc01 1051 useTags = fDataPattern.Contains("tag");
1052 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1053 }
a2f5fc01 1054 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 1055 Error("CheckInputData", "Cannot mix input files using/not using tags");
1056 return kFALSE;
1057 }
1058 if (fRunNumbers.Length()) {
a8739e8a 1059 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 1060 arr = fRunNumbers.Tokenize(" ");
1061 TIter next(arr);
1062 while ((os=(TObjString*)next())) {
1063 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1064 if (!DirectoryExists(path)) {
1065 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 1066 continue;
c57f56b7 1067 }
1068 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
1069 TString msg = "\n##### file: ";
1070 msg += path;
1071 msg += " type: xml_collection;";
a2f5fc01 1072 if (useTags) msg += " using_tags: Yes";
c57f56b7 1073 else msg += " using_tags: No";
23329835 1074 Info("CheckDataType", "%s", msg.Data());
319593fb 1075 if (fNrunsPerMaster<2) {
d2a409b2 1076 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 1077 } else {
1078 nruns++;
1079 if (((nruns-1)%fNrunsPerMaster) == 0) {
1080 schunk = os->GetString();
1081 }
1082 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1083 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1084 AddDataFile(schunk);
319593fb 1085 }
c57f56b7 1086 }
1087 delete arr;
a8739e8a 1088 } else {
1089 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1090 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1091 format = Form("%%s/%s ", fRunPrefix.Data());
1092 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1093 if (!DirectoryExists(path)) {
a8739e8a 1094 continue;
1095 }
74013535 1096 format = Form("%%s/%s.xml", fRunPrefix.Data());
1097 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1098 TString msg = "\n##### file: ";
1099 msg += path;
1100 msg += " type: xml_collection;";
a2f5fc01 1101 if (useTags) msg += " using_tags: Yes";
a8739e8a 1102 else msg += " using_tags: No";
23329835 1103 Info("CheckDataType", "%s", msg.Data());
319593fb 1104 if (fNrunsPerMaster<2) {
74013535 1105 format = Form("%s.xml", fRunPrefix.Data());
1106 AddDataFile(Form(format.Data(),irun));
319593fb 1107 } else {
1108 nruns++;
1109 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1110 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1111 }
74013535 1112 format = Form("_%s.xml", fRunPrefix.Data());
1113 schunk2 = Form(format.Data(), irun);
319593fb 1114 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1115 schunk += schunk2;
d2a409b2 1116 AddDataFile(schunk);
319593fb 1117 }
a8739e8a 1118 }
904f9f5f 1119 if (!fInputFiles) {
1120 schunk += schunk2;
1121 AddDataFile(schunk);
1122 }
c57f56b7 1123 }
1124 return kTRUE;
1125}
1126
21ca8e59 1127//______________________________________________________________________________
4a8b667c 1128Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1129{
1130// Copy data from the given grid directory according a pattern and make a local
1131// dataset.
2ba4c287 1132// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1133 if (!Connect()) {
1134 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
4a8b667c 1135 return 0;
21ca8e59 1136 }
1137 if (!DirectoryExists(griddir)) {
1138 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
4a8b667c 1139 return 0;
21ca8e59 1140 }
1141 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1142 printf("Running command: %s\n", command.Data());
1143 TGridResult *res = gGrid->Command(command);
1144 Int_t nfound = res->GetEntries();
1145 if (!nfound) {
1146 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
4a8b667c 1147 return 0;
21ca8e59 1148 }
1149 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1150
1151 // archives
1152 TObjArray* additionalArchives = 0;
1153 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1154 additionalArchives = TString(archivefile).Tokenize(";");
1155 archivefile = additionalArchives->At(0)->GetName();
1156 additionalArchives->RemoveAt(0);
1157 additionalArchives->Compress();
1158 }
1159
21ca8e59 1160 // Copy files locally
1161 ofstream out;
1162 out.open(output, ios::out);
1163 TMap *map;
1164 TString turl, dirname, filename, temp;
1165 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1166 gSystem->MakeDirectory(outputdir);
1167 gSystem->ChangeDirectory(outputdir);
4a8b667c 1168 Int_t ncopied = 0;
21ca8e59 1169 for (Int_t i=0; i<nfound; i++) {
1170 map = (TMap*)res->At(i);
1171 turl = map->GetValue("turl")->GetName();
1172 filename = gSystem->BaseName(turl.Data());
1173 dirname = gSystem->DirName(turl.Data());
1174 dirname = gSystem->BaseName(dirname.Data());
1175 gSystem->MakeDirectory(dirname);
43b807f6 1176
1177 TString source(turl);
1178 TString targetFileName(filename);
1179
1180 if (strlen(archivefile) > 0) {
4a8b667c 1181// TODO here the archive in which the file resides should be determined
1182// however whereis returns only a guid, and guid2lfn does not work
1183// Therefore we use the one provided as argument for now
1184 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1185 targetFileName = archivefile;
43b807f6 1186 }
1187 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
4a8b667c 1188 Bool_t success = kTRUE;
1189 if (additionalArchives) {
1190 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1191 TString target;
1192 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1193 gSystem->MakeDirectory(gSystem->DirName(target));
1194 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1195 }
1196 }
2ba4c287 1197
4a8b667c 1198 if (success) {
1199 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1200 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1201 ncopied++;
1202 }
21ca8e59 1203 }
1204 }
1205 gSystem->ChangeDirectory(cdir);
1206 delete res;
4a8b667c 1207 delete additionalArchives;
1208 return ncopied;
21ca8e59 1209}
1210
c57f56b7 1211//______________________________________________________________________________
1212Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1213{
1214// Create dataset for the grid data directory + run number.
3af861ff 1215 const Int_t gMaxEntries = 15000;
d3b18c4c 1216 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1217 if (!Connect()) {
1218 Error("CreateDataset", "Cannot create dataset with no grid connection");
1219 return kFALSE;
1220 }
1221
1222 // Cd workspace
d3b18c4c 1223 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1224 TString workdir = gGrid->GetHomeDirectory();
1225 workdir += fGridWorkingDir;
1226
1227 // Compose the 'find' command arguments
74013535 1228 TString format;
c57f56b7 1229 TString command;
780ec36d 1230 TString delimiter = pattern;
1231 delimiter.Strip();
1232 if (delimiter.Contains(" ")) delimiter = "";
1233 else delimiter = " ";
c57f56b7 1234 TString options = "-x collection ";
bb885a9e 1235 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1236 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1237 TString conditions = "";
3af861ff 1238 Int_t nstart = 0;
1239 Int_t ncount = 0;
1240 Int_t stage = 0;
c57f56b7 1241 TString file;
1242 TString path;
319593fb 1243 Int_t nruns = 0;
904f9f5f 1244 TString schunk, schunk2;
ab254fd1 1245 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1246 if (!fRunNumbers.Length() && !fRunRange[0]) {
1247 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1248 // Make a single data collection from data directory.
1249 path = fGridDataDir;
923e2ca5 1250 if (!DirectoryExists(path)) {
d2a409b2 1251 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1252 return kFALSE;
780ec36d 1253 }
923e2ca5 1254// CdWork();
d2a409b2 1255 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1256 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1257 while (1) {
1258 ncount = 0;
1259 stage++;
1260 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1261 command = "find ";
1262 command += Form("%s -o %d ",options.Data(), nstart);
1263 command += path;
780ec36d 1264 command += delimiter;
3af861ff 1265 command += pattern;
1266 command += conditions;
1267 printf("command: %s\n", command.Data());
1268 TGridResult *res = gGrid->Command(command);
1269 if (res) delete res;
1270 // Write standard output to file
1271 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1272 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1273 Bool_t nullFile = kFALSE;
1274 if (!hasGrep) {
1275 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1276 } else {
1277 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1278 if (nullFile) {
1279 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1280 gSystem->Exec("rm -f __tmp*");
1281 return kFALSE;
1282 }
1283 TString line;
1284 ifstream in;
1285 in.open("__tmp__");
1286 in >> line;
1287 in.close();
1288 gSystem->Exec("rm -f __tmp__");
1289 ncount = line.Atoi();
1290 }
1291 }
1292 if (ncount == gMaxEntries) {
1293 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1294 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1295 if (!cbase) cbase = cadd;
1296 else {
1297 cbase->Add(cadd);
1298 delete cadd;
defd7a3a 1299 }
3af861ff 1300 nstart += ncount;
1301 } else {
1302 if (cbase) {
1303 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1304 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1305 cbase->Add(cadd);
1306 delete cadd;
1307 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1308 delete cbase; cbase = 0;
1309 } else {
1310 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1311 }
1312 gSystem->Exec("rm -f __tmp*");
1313 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1314 break;
1315 }
0f389141 1316 }
1317 Bool_t fileExists = FileExists(file);
1318 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1319 // Copy xml file to alien space
0f389141 1320 if (fileExists) gGrid->Rm(file);
d2a409b2 1321 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1322 if (!FileExists(file)) {
1323 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1324 return kFALSE;
1325 }
1326 // Update list of files to be processed.
1327 }
1328 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1329 return kTRUE;
1330 }
c57f56b7 1331 // Several runs
a2f5fc01 1332 Bool_t nullResult = kTRUE;
a8739e8a 1333 if (fRunNumbers.Length()) {
1334 TObjArray *arr = fRunNumbers.Tokenize(" ");
1335 TObjString *os;
1336 TIter next(arr);
1337 while ((os=(TObjString*)next())) {
3af861ff 1338 nstart = 0;
1339 stage = 0;
0dab9d6d 1340 path = Form("%s/%s", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1341 if (!DirectoryExists(path)) continue;
1342// CdWork();
a8739e8a 1343 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1344 else file = Form("%s.xml", os->GetString().Data());
319593fb 1345 // If local collection file does not exist, create it via 'find' command.
3af861ff 1346 while (1) {
1347 ncount = 0;
1348 stage++;
1349 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1350 command = "find ";
1351 command += Form("%s -o %d ",options.Data(), nstart);
1352 command += path;
780ec36d 1353 command += delimiter;
3af861ff 1354 command += pattern;
1355 command += conditions;
1356 TGridResult *res = gGrid->Command(command);
1357 if (res) delete res;
1358 // Write standard output to file
1359 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1360 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1361 Bool_t nullFile = kFALSE;
1362 if (!hasGrep) {
1363 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1364 } else {
1365 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1366 if (nullFile) {
1367 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1368 gSystem->Exec("rm -f __tmp*");
1369 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1370 break;
3af861ff 1371 }
1372 TString line;
1373 ifstream in;
1374 in.open("__tmp__");
1375 in >> line;
1376 in.close();
1377 gSystem->Exec("rm -f __tmp__");
1378 ncount = line.Atoi();
1379 }
1380 nullResult = kFALSE;
1381 }
1382 if (ncount == gMaxEntries) {
1383 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1384 if (fNrunsPerMaster > 1) {
1385 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1386 file.Data(),gMaxEntries);
1387 return kFALSE;
1388 }
1389 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1390 if (!cbase) cbase = cadd;
1391 else {
1392 cbase->Add(cadd);
1393 delete cadd;
defd7a3a 1394 }
3af861ff 1395 nstart += ncount;
1396 } else {
c89a84bc 1397 if (cbase && fNrunsPerMaster<2) {
3af861ff 1398 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1399 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1400 cbase->Add(cadd);
1401 delete cadd;
1402 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1403 delete cbase; cbase = 0;
1404 } else {
1405 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1406 }
1407 gSystem->Exec("rm -f __tmp*");
1408 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1409 break;
d3339be3 1410 }
3af861ff 1411 }
a8739e8a 1412 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1413 // Check if there is one run per master job.
1414 if (fNrunsPerMaster<2) {
1415 if (FileExists(file)) {
0f389141 1416 if (fOverwriteMode) gGrid->Rm(file);
1417 else {
1418 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1419 continue;
1420 }
319593fb 1421 }
1422 // Copy xml file to alien space
1423 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1424 if (!FileExists(file)) {
1425 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1426 delete arr;
1427 return kFALSE;
1428 }
1429 } else {
1430 nruns++;
1431 if (((nruns-1)%fNrunsPerMaster) == 0) {
1432 schunk = os->GetString();
1433 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1434 } else {
1435 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1436 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1437 cbase->Add(cadd);
1438 delete cadd;
1439 }
1440 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1441 continue;
1442 }
1443 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1444 if (FileExists(schunk)) {
1445 if (fOverwriteMode) gGrid->Rm(file);
1446 else {
1447 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1448 continue;
1449 }
319593fb 1450 }
84fcd93f 1451 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1452 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1453 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1454 if (!FileExists(schunk)) {
1455 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1456 delete arr;
1457 return kFALSE;
1458 }
d3339be3 1459 }
a8739e8a 1460 }
1461 delete arr;
a2f5fc01 1462 if (nullResult) {
d3339be3 1463 Error("CreateDataset", "No valid dataset corresponding to the query!");
1464 return kFALSE;
1465 }
a8739e8a 1466 } else {
1467 // Process a full run range.
1468 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
0dab9d6d 1469 format = Form("%%s/%s", fRunPrefix.Data());
3af861ff 1470 nstart = 0;
1471 stage = 0;
74013535 1472 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1473 if (!DirectoryExists(path)) continue;
1474// CdWork();
74013535 1475 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1476 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1477 else file = Form(format.Data(), irun);
0f389141 1478 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1479 if (fOverwriteMode) gGrid->Rm(file);
1480 else {
1481 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1482 continue;
1483 }
a8739e8a 1484 }
319593fb 1485 // If local collection file does not exist, create it via 'find' command.
3af861ff 1486 while (1) {
1487 ncount = 0;
1488 stage++;
1489 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1490 command = "find ";
1491 command += Form("%s -o %d ",options.Data(), nstart);
1492 command += path;
780ec36d 1493 command += delimiter;
3af861ff 1494 command += pattern;
1495 command += conditions;
1496 TGridResult *res = gGrid->Command(command);
1497 if (res) delete res;
1498 // Write standard output to file
1499 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1500 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1501 Bool_t nullFile = kFALSE;
1502 if (!hasGrep) {
1503 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1504 } else {
1505 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1506 if (nullFile) {
1507 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1508 gSystem->Exec("rm -f __tmp*");
1604c9ed 1509 break;
3af861ff 1510 }
1511 TString line;
1512 ifstream in;
1513 in.open("__tmp__");
1514 in >> line;
1515 in.close();
1516 gSystem->Exec("rm -f __tmp__");
1517 ncount = line.Atoi();
1518 }
1519 nullResult = kFALSE;
1520 }
1521 if (ncount == gMaxEntries) {
1522 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1523 if (fNrunsPerMaster > 1) {
1524 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1525 file.Data(),gMaxEntries);
1526 return kFALSE;
1527 }
1528 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1529 if (!cbase) cbase = cadd;
1530 else {
1531 cbase->Add(cadd);
1532 delete cadd;
defd7a3a 1533 }
3af861ff 1534 nstart += ncount;
1535 } else {
c89a84bc 1536 if (cbase && fNrunsPerMaster<2) {
3af861ff 1537 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1538 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1539 cbase->Add(cadd);
1540 delete cadd;
1541 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1542 delete cbase; cbase = 0;
1543 } else {
1544 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1545 }
1546 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1547 break;
d3339be3 1548 }
319593fb 1549 }
a8739e8a 1550 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1551 // Check if there is one run per master job.
1552 if (fNrunsPerMaster<2) {
1553 if (FileExists(file)) {
0f389141 1554 if (fOverwriteMode) gGrid->Rm(file);
1555 else {
1556 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1557 continue;
1558 }
319593fb 1559 }
1560 // Copy xml file to alien space
1561 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1562 if (!FileExists(file)) {
1563 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1564 return kFALSE;
1565 }
1566 } else {
1567 nruns++;
95e5b448 1568 // Check if the collection for the chunk exist locally.
1569 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1570 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1571 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1572 else continue;
1573 }
84fcd93f 1574 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1575 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1576 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1577 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1578 } else {
1579 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1580 cbase->Add(cadd);
1581 delete cadd;
1582 }
74013535 1583 format = Form("%%s_%s.xml", fRunPrefix.Data());
1584 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1585 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1586 continue;
1587 }
904f9f5f 1588 schunk = schunk2;
319593fb 1589 if (FileExists(schunk)) {
0f389141 1590 if (fOverwriteMode) gGrid->Rm(schunk);
1591 else {
1592 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1593 continue;
1594 }
319593fb 1595 }
84fcd93f 1596 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1597 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1598 if (FileExists(schunk)) {
0f389141 1599 if (fOverwriteMode) gGrid->Rm(schunk);
1600 else {
1601 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1602 continue;
1603 }
95e5b448 1604 }
319593fb 1605 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1606 if (!FileExists(schunk)) {
1607 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1608 return kFALSE;
1609 }
1610 }
c57f56b7 1611 }
a2f5fc01 1612 if (nullResult) {
d3339be3 1613 Error("CreateDataset", "No valid dataset corresponding to the query!");
1614 return kFALSE;
1615 }
a8739e8a 1616 }
c57f56b7 1617 return kTRUE;
1618}
1619
1620//______________________________________________________________________________
1621Bool_t AliAnalysisAlien::CreateJDL()
1622{
1623// Generate a JDL file according to current settings. The name of the file is
1624// specified by fJDLName.
ec5acdce 1625 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
c57f56b7 1626 Bool_t error = kFALSE;
1627 TObjArray *arr = 0;
1628 Bool_t copy = kTRUE;
d3b18c4c 1629 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1630 Bool_t generate = kTRUE;
1631 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1632 if (!Connect()) {
1633 Error("CreateJDL", "Alien connection required");
1634 return kFALSE;
1635 }
1636 // Check validity of alien workspace
d3b18c4c 1637 TString workdir;
1638 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1639 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1640 workdir += fGridWorkingDir;
1641 if (generate) {
1642 TObjString *os;
1e466628 1643 if (!fInputFiles && !fMCLoop) {
c57f56b7 1644 Error("CreateJDL()", "Define some input files for your analysis.");
1645 error = kTRUE;
1646 }
1647 // Compose list of input files
1648 // Check if output files were defined
1649 if (!fOutputFiles.Length()) {
1650 Error("CreateJDL", "You must define at least one output file");
1651 error = kTRUE;
1652 }
1653 // Check if an output directory was defined and valid
1654 if (!fGridOutputDir.Length()) {
1655 Error("CreateJDL", "You must define AliEn output directory");
1656 error = kTRUE;
1657 } else {
d3b18c4c 1658 if (!fProductionMode) {
1659 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1660 if (!DirectoryExists(fGridOutputDir)) {
1661 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1662 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1663 } else {
1664 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1665 // error = kTRUE;
1666 }
67a3b254 1667 } else {
1668 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1669 }
1670 gGrid->Cd(workdir);
1671 }
c57f56b7 1672 }
1673 // Exit if any error up to now
1674 if (error) return kFALSE;
1675 // Set JDL fields
0f389141 1676 if (!fUser.IsNull()) {
1677 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1678 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
6da4839e
AG		 1679 }
1680 TString executable = fExecutable;
1681 if (!executable.BeginsWith("/"))
1682 executable.Prepend(Form("%s/", workdir.Data()));
1683 fGridJDL->SetExecutable(executable, "This is the startup script");
1684 TString mergeExec = executable;
0f389141 1685 mergeExec.ReplaceAll(".sh", "_merge.sh");
1686 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1687 mergeExec.ReplaceAll(".sh", ".C");
9c5cde18 1688 fMergingJDL->AddToInputSandbox(Form("LF:%s", mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1689 if (!fArguments.IsNull())
1690 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1691 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1692 else {
1693 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1694 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1695 }
b3e07543 1696
1f0d1ca2 1697 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1698 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1699 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1700 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1701
0f389141 1702 if (fMaxInitFailed > 0) {
c57f56b7 1703 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1704 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1705 }
6da4839e 1706 if (fSplitMaxInputFileNumber > 0 && !fMCLoop) {
c57f56b7 1707 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1708 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1709 }
d7eb1f39 1710 if (!IsOneStageMerging()) {
f47d5cb4 1711 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1712 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1713 }
1714 if (fSplitMode.Length()) {
c57f56b7 1715 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1716 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1717 }
1718 fMergingJDL->SetValue("Split", "\"se\"");
1719 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1720 if (!fAliROOTVersion.IsNull()) {
1721 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1722 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1723 }
1724 if (!fROOTVersion.IsNull()) {
c57f56b7 1725 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1726 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1727 }
1728 if (!fAPIVersion.IsNull()) {
c57f56b7 1729 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1730 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1731 }
648174cf 1732 if (!fExternalPackages.IsNull()) {
1733 arr = fExternalPackages.Tokenize(" ");
1734 TIter next(arr);
1735 while ((os=(TObjString*)next())) {
1736 TString pkgname = os->GetString();
1737 Int_t index = pkgname.Index("::");
1738 TString pkgversion = pkgname(index+2, pkgname.Length());
1739 pkgname.Remove(index);
1740 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1741 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1742 }
1743 delete arr;
1744 }
6da4839e
AG		 1745 if (!fMCLoop) {
1746 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1747 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
6da4839e 1748 }
d7eb1f39
AG		 1749 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1750 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1751 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1752 TString analysisFile = fExecutable;
1753 analysisFile.ReplaceAll(".sh", ".root");
1754 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1755 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1756 if (fAdditionalLibs.Length()) {
1757 arr = fAdditionalLibs.Tokenize(" ");
1758 TIter next(arr);
1759 while ((os=(TObjString*)next())) {
1760 if (os->GetString().Contains(".so")) continue;
1761 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1762 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1763 }
1764 delete arr;
1765 }
4e5c5506 1766 if (fPackages) {
1767 TIter next(fPackages);
1768 TObject *obj;
0f389141 1769 while ((obj=next())) {
4e5c5506 1770 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1771 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1772 }
4e5c5506 1773 }
95e16f01 1774 const char *comment = "List of output files and archives";
c57f56b7 1775 if (fOutputArchive.Length()) {
e4d9a069 1776 TString outputArchive = fOutputArchive;
1777 if (!fRegisterExcludes.IsNull()) {
1778 arr = fRegisterExcludes.Tokenize(" ");
1779 TIter next1(arr);
1780 while ((os=(TObjString*)next1())) {
1781 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1782 outputArchive.ReplaceAll(os->GetString(),"");
1783 }
1784 delete arr;
1785 }
1786 arr = outputArchive.Tokenize(" ");
c57f56b7 1787 TIter next(arr);
0f389141 1788 Bool_t first = kTRUE;
0f389141 1789 while ((os=(TObjString*)next())) {
0f389141 1790 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1791 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
0f389141 1792 else
95e16f01 1793 fGridJDL->AddToSet("Output", os->GetString());
1794 if (first) fGridJDL->AddToSetDescription("Output", comment);
0f389141 1795 first = kFALSE;
1796 }
c57f56b7 1797 delete arr;
f790bc1b 1798 // Output archive for the merging jdl
f790bc1b 1799 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1800 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1801 // Add normal output files, extra files + terminate files
0c0825e4 1802 TString files;
1803 if (IsMergeAOD()) files = GetListOfFiles("outaodextter");
1804 else files = GetListOfFiles("outextter");
37d21c01 1805 // Do not register files in fRegisterExcludes
1806 if (!fRegisterExcludes.IsNull()) {
1807 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1808 TIter next1(arr);
1809 while ((os=(TObjString*)next1())) {
1810 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1811 files.ReplaceAll(os->GetString(),"");
1812 }
66d71516 1813 delete arr;
0f389141 1814 }
f790bc1b 1815 files.ReplaceAll(".root", "*.root");
ec5acdce 1816
1817 if (mgr->IsCollectThroughput())
95e16f01 1818 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",files.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 1819 else
1820 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1821 } else {
95f84779 1822 TString files = fOutputArchive;
1823 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1824 outputArchive = files;
f790bc1b 1825 }
0f389141 1826 arr = outputArchive.Tokenize(" ");
705adb3e 1827 TIter next2(arr);
0f389141 1828 first = kTRUE;
705adb3e 1829 while ((os=(TObjString*)next2())) {
0cdf65a8 1830 TString currentfile = os->GetString();
0cdf65a8 1831 if (!currentfile.Contains("@") && fCloseSE.Length())
95e16f01 1832 fMergingJDL->AddToSet("Output", Form("%s@%s",currentfile.Data(), fCloseSE.Data()));
0f389141 1833 else
95e16f01 1834 fMergingJDL->AddToSet("Output", currentfile);
1835 if (first) fMergingJDL->AddToSetDescription("Output", comment);
0f389141 1836 first = kFALSE;
1837 }
1838 delete arr;
c57f56b7 1839 }
149d288c 1840 arr = fOutputFiles.Tokenize(",");
c57f56b7 1841 TIter next(arr);
0f389141 1842 Bool_t first = kTRUE;
43da816a 1843 while ((os=(TObjString*)next())) {
e1eaf596 1844 // Ignore ouputs in jdl that are also in outputarchive
1845 TString sout = os->GetString();
f790bc1b 1846 sout.ReplaceAll("*", "");
1847 sout.ReplaceAll(".root", "");
e1eaf596 1848 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1849 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1850 // Ignore fRegisterExcludes
1851 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1852 if (!first) comment = NULL;
43da816a 1853 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1854 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
43da816a 1855 else
95e16f01 1856 fGridJDL->AddToSet("Output", os->GetString());
1857 if (first) fGridJDL->AddToSetDescription("Output", comment);
f790bc1b 1858 if (fMergeExcludes.Contains(sout)) continue;
1859 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1860 fMergingJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
f790bc1b 1861 else
95e16f01 1862 fMergingJDL->AddToSet("Output", os->GetString());
1863 if (first) fMergingJDL->AddToSetDescription("Output", comment);
1864 first = kFALSE;
43da816a 1865 }
c57f56b7 1866 delete arr;
0f389141 1867 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1868 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1869 TString validationScript = fValidationScript;
0f389141 1870 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1871 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1872 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1873 if (fMasterResubmitThreshold) {
1874 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1875 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1876 }
d2a409b2 1877 // Write a jdl with 2 input parameters: collection name and output dir name.
1878 WriteJDL(copy);
c57f56b7 1879 }
1880 // Copy jdl to grid workspace
a8739e8a 1881 if (copy) {
b5e4aaa7 1882 // Check if an output directory was defined and valid
1883 if (!fGridOutputDir.Length()) {
1884 Error("CreateJDL", "You must define AliEn output directory");
1885 return kFALSE;
1886 } else {
1887 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1888 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1889 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1890 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1891 } else {
1892 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1893 return kFALSE;
1894 }
1895 }
1896 gGrid->Cd(workdir);
1897 }
648174cf 1898 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1899 TString mergeJDLName = fExecutable;
1900 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1901 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1902 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1903 if (fProductionMode) {
648174cf 1904 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1905 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1906 }
648174cf 1907 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1908 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1909 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1910 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1911 Fatal("","Terminating");
1912// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1913 if (fMergeViaJDL) {
1914 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1915// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1916 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1917 Fatal("","Terminating");
0f389141 1918 }
648174cf 1919 }
c57f56b7 1920 if (fAdditionalLibs.Length()) {
e7c71df0 1921 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1922 TObjString *os;
1923 TIter next(arr);
1924 while ((os=(TObjString*)next())) {
c57f56b7 1925 if (os->GetString().Contains(".so")) continue;
4e5c5506 1926 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1927 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1928// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1929 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1930 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1931 Fatal("","Terminating");
c57f56b7 1932 }
1933 delete arr;
1934 }
4e5c5506 1935 if (fPackages) {
1936 TIter next(fPackages);
1937 TObject *obj;
1938 while ((obj=next())) {
fdbbc7be 1939 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1940 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1941// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1942 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1943 Form("%s/%s", workdir.Data(), obj->GetName())))
1944 Fatal("","Terminating");
4e5c5506 1945 }
1946 }
c57f56b7 1947 }
1948 return kTRUE;
1949}
1950
a8739e8a 1951//______________________________________________________________________________
d2a409b2 1952Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1953{
1954// Writes one or more JDL's corresponding to findex. If findex is negative,
1955// all run numbers are considered in one go (jdl). For non-negative indices
1956// they correspond to the indices in the array fInputFiles.
dec33edc 1957 if (!fInputFiles && !fMCLoop) return kFALSE;
d3b18c4c 1958 TObject *os;
1959 TString workdir;
1960 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1961 workdir += fGridWorkingDir;
f47d5cb4 1962 TString stageName = "$2";
1963 if (fProductionMode) stageName = "$4";
1964 if (!fMergeDirName.IsNull()) {
1965 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1966 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1967 } else {
1968 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1969 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1970 }
d3b18c4c 1971 if (fProductionMode) {
a8739e8a 1972 TIter next(fInputFiles);
b3e07543 1973 while ((os=next())) {
d3b18c4c 1974 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1975 }
56754bf6 1976 if (!fOutputToRunNo)
1977 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1978 else
1979 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1980 } else {
1981 if (!fRunNumbers.Length() && !fRunRange[0]) {
1982 // One jdl with no parameters in case input data is specified by name.
1983 TIter next(fInputFiles);
1984 while ((os=next()))
1985 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1986 if (!fOutputSingle.IsNull())
1987 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1988 else {
1989 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
b1dade98 1990// fMergingJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1991 }
1992 } else {
1993 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1994 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1995 if (!fOutputSingle.IsNull()) {
1996 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1997 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1998 } else {
1999 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 2000 }
2001 }
a8739e8a 2002 }
2003
a8739e8a 2004 // Generate the JDL as a string
2005 TString sjdl = fGridJDL->Generate();
0f389141 2006 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 2007 // Final merge jdl
2008 if (!fMergeDirName.IsNull()) {
2009 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
2010 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
2011 } else {
2012 fMergingJDL->SetOutputDirectory("$1", "Output directory");
2013 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
2014 }
b3e07543 2015 TString sjdl2 = fMergingJDL->Generate();
2016 Int_t index, index1;
95e16f01 2017 sjdl.ReplaceAll("\",\"", "\",\n \"");
a8739e8a 2018 sjdl.ReplaceAll("(member", "\n (member");
2019 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2020 sjdl.ReplaceAll("{", "{\n ");
2021 sjdl.ReplaceAll("};", "\n};");
2022 sjdl.ReplaceAll("{\n \n", "{\n");
2023 sjdl.ReplaceAll("\n\n", "\n");
2024 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2025 sjdl1.ReplaceAll("\",\"", "\",\n \"");
0f389141 2026 sjdl1.ReplaceAll("(member", "\n (member");
2027 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2028 sjdl1.ReplaceAll("{", "{\n ");
2029 sjdl1.ReplaceAll("};", "\n};");
2030 sjdl1.ReplaceAll("{\n \n", "{\n");
2031 sjdl1.ReplaceAll("\n\n", "\n");
2032 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2033 sjdl2.ReplaceAll("\",\"", "\",\n \"");
b3e07543 2034 sjdl2.ReplaceAll("(member", "\n (member");
2035 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2036 sjdl2.ReplaceAll("{", "{\n ");
2037 sjdl2.ReplaceAll("};", "\n};");
2038 sjdl2.ReplaceAll("{\n \n", "{\n");
2039 sjdl2.ReplaceAll("\n\n", "\n");
2040 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 2041 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 2042 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 2043 index = sjdl.Index("JDLVariables");
2044 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 2045 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2046 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 2047 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 2048 index = fJobTag.Index(":");
2049 if (index < 0) index = fJobTag.Length();
2050 TString jobTag = fJobTag;
f47d5cb4 2051 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 2052 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 2053 if (fProductionMode) {
2054 sjdl1.Prepend("# Generated merging jdl (production mode) \
2055 \n# $1 = full alien path to output directory to be merged \
2056 \n# $2 = train number \
2057 \n# $3 = production (like LHC10b) \
2058 \n# $4 = merging stage \
2059 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2060 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2061 sjdl2.Prepend("# Generated merging jdl \
2062 \n# $1 = full alien path to output directory to be merged \
2063 \n# $2 = train number \
2064 \n# $3 = production (like LHC10b) \
2065 \n# $4 = merging stage \
2066 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2067 } else {
2068 sjdl1.Prepend("# Generated merging jdl \
2069 \n# $1 = full alien path to output directory to be merged \
2070 \n# $2 = merging stage \
03335324 2071 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2072 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2073 sjdl2.Prepend("# Generated merging jdl \
2074 \n# $1 = full alien path to output directory to be merged \
2075 \n# $2 = merging stage \
03335324 2076 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2077 }
0f389141 2078 index = sjdl1.Index("JDLVariables");
2079 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 2080 index = sjdl2.Index("JDLVariables");
2081 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 2082 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2083 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
2084 index = sjdl2.Index("Split =");
2085 if (index>=0) {
2086 index1 = sjdl2.Index("\n", index);
2087 sjdl2.Remove(index, index1-index+1);
2088 }
2089 index = sjdl2.Index("SplitMaxInputFileNumber");
2090 if (index>=0) {
2091 index1 = sjdl2.Index("\n", index);
2092 sjdl2.Remove(index, index1-index+1);
2093 }
2094 index = sjdl2.Index("InputDataCollection");
2095 if (index>=0) {
2096 index1 = sjdl2.Index(";", index);
2097 sjdl2.Remove(index, index1-index+1);
2098 }
2099 index = sjdl2.Index("InputDataListFormat");
2100 if (index>=0) {
2101 index1 = sjdl2.Index("\n", index);
2102 sjdl2.Remove(index, index1-index+1);
2103 }
2104 index = sjdl2.Index("InputDataList");
2105 if (index>=0) {
2106 index1 = sjdl2.Index("\n", index);
2107 sjdl2.Remove(index, index1-index+1);
2108 }
f47d5cb4 2109 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2110 // Write jdl to file
a8739e8a 2111 ofstream out;
d2a409b2 2112 out.open(fJDLName.Data(), ios::out);
a8739e8a 2113 if (out.bad()) {
d3b18c4c 2114 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2115 return kFALSE;
2116 }
2117 out << sjdl << endl;
b3e07543 2118 out.close();
0f389141 2119 TString mergeJDLName = fExecutable;
2120 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2121 if (fMergeViaJDL) {
2122 ofstream out1;
2123 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2124 if (out1.bad()) {
d3b18c4c 2125 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2126 return kFALSE;
2127 }
2128 out1 << sjdl1 << endl;
b3e07543 2129 out1.close();
2130 ofstream out2;
2131 TString finalJDL = mergeJDLName;
2132 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2133 out2.open(finalJDL.Data(), ios::out);
2134 if (out2.bad()) {
2135 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2136 return kFALSE;
2137 }
2138 out2 << sjdl2 << endl;
2139 out2.close();
0f389141 2140 }
a8739e8a 2141
2142 // Copy jdl to grid workspace
2143 if (!copy) {
d3b18c4c 2144 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2145 } else {
d2a409b2 2146 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2147 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2148 TString finalJDL = mergeJDLName;
2149 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2150 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2151 if (fProductionMode) {
b5fe9cba 2152 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2153 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2154 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2155 }
d2a409b2 2156 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2157 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2158 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2159 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2160// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2161 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2162 Fatal("","Terminating");
0f389141 2163 if (fMergeViaJDL) {
b3e07543 2164 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2165// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2166// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2167 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2168 Fatal("","Terminating");
2169 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2170 Fatal("","Terminating");
0f389141 2171 }
a8739e8a 2172 }
2173 return kTRUE;
2174}
2175
c57f56b7 2176//______________________________________________________________________________
5513444a 2177Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2178{
2179// Returns true if file exists.
5513444a 2180 if (!gGrid) return kFALSE;
2ea55496 2181 TString slfn = lfn;
2182 slfn.ReplaceAll("alien://","");
2183 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2184 if (!res) return kFALSE;
2185 TMap *map = dynamic_cast<TMap*>(res->At(0));
2186 if (!map) {
2187 delete res;
2188 return kFALSE;
2189 }
2190 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2191 if (!objs || !objs->GetString().Length()) {
2192 delete res;
2193 return kFALSE;
2194 }
2195 delete res;
2196 return kTRUE;
2197}
2198
923e2ca5 2199//______________________________________________________________________________
2200Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2201{
2202// Returns true if directory exists. Can be also a path.
d7aac489 2203// Since there is not API in TAlien, we use the Cd trick:
923e2ca5 2204 if (!gGrid) return kFALSE;
d7aac489 2205 // Backup current path
2206 TString cpath = gGrid->Pwd();
2207 TString command = "cd ";
2208 TString sdir(dirname);
2209 sdir.ReplaceAll("alien://", "");
2210 command += sdir;
2211 TGridResult *res = gGrid->Command(command);
2212 if (!res) {
2213 gGrid->Cd(cpath);
2214 return kFALSE;
2215 }
2216 TMap *map = (TMap*)res->At(0);
2217 if (!map) {
2218 gGrid->Cd(cpath);
2219 delete res;
2220 return kFALSE;
923e2ca5 2221 }
d7aac489 2222 TString sval = map->GetValue("__result__")->GetName();
2223 Bool_t retval = (Bool_t)sval.Atoi();
2224 gGrid->Cd(cpath);
923e2ca5 2225 delete res;
d7aac489 2226 return retval;
2227}
923e2ca5 2228
c57f56b7 2229//______________________________________________________________________________
a2f5fc01 2230void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2231{
2232// Check input data type.
a2f5fc01 2233 isCollection = kFALSE;
2234 isXml = kFALSE;
2235 useTags = kFALSE;
c57f56b7 2236 if (!gGrid) {
2237 Error("CheckDataType", "No connection to grid");
2238 return;
2239 }
a2f5fc01 2240 isCollection = IsCollection(lfn);
c57f56b7 2241 TString msg = "\n##### file: ";
2242 msg += lfn;
a2f5fc01 2243 if (isCollection) {
c57f56b7 2244 msg += " type: raw_collection;";
2245 // special treatment for collections
a2f5fc01 2246 isXml = kFALSE;
c57f56b7 2247 // check for tag files in the collection
2248 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2249 if (!res) {
2250 msg += " using_tags: No (unknown)";
23329835 2251 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2252 return;
2253 }
2254 const char* typeStr = res->GetKey(0, "origLFN");
2255 if (!typeStr || !strlen(typeStr)) {
2256 msg += " using_tags: No (unknown)";
23329835 2257 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2258 return;
2259 }
2260 TString file = typeStr;
a2f5fc01 2261 useTags = file.Contains(".tag");
2262 if (useTags) msg += " using_tags: Yes";
c57f56b7 2263 else msg += " using_tags: No";
23329835 2264 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2265 return;
2266 }
2267 TString slfn(lfn);
2268 slfn.ToLower();
a2f5fc01 2269 isXml = slfn.Contains(".xml");
2270 if (isXml) {
c57f56b7 2271 // Open xml collection and check if there are tag files inside
2272 msg += " type: xml_collection;";
2273 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2274 if (!coll) {
2275 msg += " using_tags: No (unknown)";
23329835 2276 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2277 return;
2278 }
2279 TMap *map = coll->Next();
2280 if (!map) {
2281 msg += " using_tags: No (unknown)";
23329835 2282 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2283 return;
2284 }
2285 map = (TMap*)map->GetValue("");
2286 TString file;
2287 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2288 useTags = file.Contains(".tag");
c57f56b7 2289 delete coll;
a2f5fc01 2290 if (useTags) msg += " using_tags: Yes";
c57f56b7 2291 else msg += " using_tags: No";
23329835 2292 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2293 return;
2294 }
a2f5fc01 2295 useTags = slfn.Contains(".tag");
c57f56b7 2296 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2297 else msg += " type: unknown file;";
a2f5fc01 2298 if (useTags) msg += " using_tags: Yes";
c57f56b7 2299 else msg += " using_tags: No";
23329835 2300 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2301}
2302
4e5c5506 2303//______________________________________________________________________________
2304void AliAnalysisAlien::EnablePackage(const char *package)
2305{
2306// Enables a par file supposed to exist in the current directory.
2307 TString pkg(package);
2308 pkg.ReplaceAll(".par", "");
2309 pkg += ".par";
2310 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2311 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2312 return;
2313 }
2314 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2315 Info("EnablePackage", "AliEn plugin will use .par packages");
2316 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2317 if (!fPackages) {
2318 fPackages = new TObjArray();
2319 fPackages->SetOwner();
2320 }
2321 fPackages->Add(new TObjString(pkg));
2322}
2323
3bdcb562 2324//______________________________________________________________________________
2325TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2326{
2327// Make a tree from files having the location specified in fFileForTestMode.
2328// Inspired from JF's CreateESDChain.
2329 if (fFileForTestMode.IsNull()) {
2330 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2331 return NULL;
2332 }
2333 if (gSystem->AccessPathName(fFileForTestMode)) {
2334 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2335 return NULL;
2336 }
2337 // Open the file
2338 ifstream in;
2339 in.open(fFileForTestMode);
2340 Int_t count = 0;
2341 // Read the input list of files and add them to the chain
b505f1f8 2342 TString line;
d6c60197 2343 TString streeName(treeName);
2344 if (IsUseMCchain()) streeName = "TE";
2345 TChain *chain = new TChain(streeName);
06f67114 2346 TList *friends = new TList();
2347 TChain *cfriend = 0;
2348 if (!fFriendChainName.IsNull()) {
2349 TObjArray *list = fFriendChainName.Tokenize(" ");
2350 TIter next(list);
2351 TObjString *str;
2352 while((str=(TObjString*)next())) {
2353 cfriend = new TChain(streeName, str->GetName());
2354 friends->Add(cfriend);
2355 chain->AddFriend(cfriend);
2356 }
2357 delete list;
2358 }
2359 TString bpath;
2360 TIter nextfriend(friends);
b505f1f8 2361 while (in.good())
2362 {
3bdcb562 2363 in >> line;
ffbe06b9 2364 if (line.IsNull() || line.BeginsWith("#")) continue;
3bdcb562 2365 if (count++ == fNtestFiles) break;
2366 TString esdFile(line);
2367 TFile *file = TFile::Open(esdFile);
b505f1f8 2368 if (file && !file->IsZombie()) {
2369 chain->Add(esdFile);
3bdcb562 2370 file->Close();
b505f1f8 2371 if (!fFriendChainName.IsNull()) {
4b5d9e21 2372 if (esdFile.Index("#") > -1)
2373 esdFile.Remove(esdFile.Index("#"));
06f67114 2374 bpath = gSystem->DirName(esdFile);
2375 bpath += "/";
2376 TString fileFriend;
2377 nextfriend.Reset();
2378 while ((cfriend=(TChain*)nextfriend())) {
2379 fileFriend = bpath;
2380 fileFriend += cfriend->GetTitle();
2381 file = TFile::Open(fileFriend);
2382 if (file && !file->IsZombie()) {
2383 file->Close();
2384 cfriend->Add(fileFriend);
2385 } else {
2386 Fatal("GetChainForTestMode", "Cannot open friend file: %s", fileFriend.Data());
2387 return 0;
2388 }
2389 }
4b5d9e21 2390 }
3bdcb562 2391 } else {
2392 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2393 }
b505f1f8 2394 }
2395 in.close();
2396 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2397 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2398 delete chain;
06f67114 2399 friends->Delete();
2400 delete friends;
3bdcb562 2401 return NULL;
b505f1f8 2402 }
b505f1f8 2403 return chain;
3bdcb562 2404}
2405
d2a409b2 2406//______________________________________________________________________________
2407const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2408{
2409// Get job status for all jobs with jobid>jobidstart.
2410 static char mstatus[20];
2411 mstatus[0] = '\0';
2412 nrunning = 0;
2413 nwaiting = 0;
2414 nerror = 0;
2415 ndone = 0;
2416 TGridJobStatusList *list = gGrid->Ps("");
2417 if (!list) return mstatus;
2418 Int_t nentries = list->GetSize();
2419 TGridJobStatus *status;
2420 Int_t pid;
2421 for (Int_t ijob=0; ijob<nentries; ijob++) {
2422 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2423 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2424 if (pid<jobidstart) continue;
2425 if (pid == lastid) {
8ddad121 2426 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2427 }
2428 switch (status->GetStatus()) {
2429 case TGridJobStatus::kWAITING:
2430 nwaiting++; break;
2431 case TGridJobStatus::kRUNNING:
2432 nrunning++; break;
2433 case TGridJobStatus::kABORTED:
2434 case TGridJobStatus::kFAIL:
2435 case TGridJobStatus::kUNKNOWN:
2436 nerror++; break;
2437 case TGridJobStatus::kDONE:
2438 ndone++;
2439 }
2440 }
2441 list->Delete();
2442 delete list;
2443 return mstatus;
2444}
2445
c57f56b7 2446//______________________________________________________________________________
2447Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2448{
2449// Returns true if file is a collection. Functionality duplicated from
2450// TAlien::Type() because we don't want to directly depend on TAlien.
2451 if (!gGrid) {
2452 Error("IsCollection", "No connection to grid");
2453 return kFALSE;
2454 }
2455 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2456 if (!res) return kFALSE;
2457 const char* typeStr = res->GetKey(0, "type");
2458 if (!typeStr || !strlen(typeStr)) return kFALSE;
2459 if (!strcmp(typeStr, "collection")) return kTRUE;
2460 delete res;
2461 return kFALSE;
2462}
2463
fe2d7fc2 2464//______________________________________________________________________________
2465Bool_t AliAnalysisAlien::IsSingleOutput() const
2466{
2467// Check if single-ouput option is on.
2468 return (!fOutputSingle.IsNull());
2469}
715da79c 2470
2471//______________________________________________________________________________
2472Long64_t AliAnalysisAlien::RunMacroAndExtractLibs(const char* macro, const char *args, TString &libs)
2473{
2474// Tries to run the specified macro and return the libraries that it loads.
2475 TString expname;
2476 if (strlen(macro)) expname = gSystem->ExpandPathName(macro);
2477 if (expname.IsNull() || gSystem->AccessPathName(expname)) {
2478 ::Error("RunMacroAndExtractLibs","Cannot find macro %s in current directory", macro);
2479 return -1;
2480 }
2481 TString oldlibs = gSystem->GetLibraries();
2482 TMacro m(expname);
2483 Int_t error = 0;
2484 Long64_t retval = m.Exec(args, &error);
2485 if (error != TInterpreter::kNoError)
2486 {
2487 ::Error("RunMacroAndExtractLibs", "Macro interpretation %s failed", macro);
2488 return -1;
2489 }
2490 libs = gSystem->GetLibraries();
2491 libs.ReplaceAll(oldlibs, "");
2492 libs.Strip(TString::kLeading);
2493 TObjArray *libTokens = libs.Tokenize(" ");
2494 libs = "";
2495 for (Int_t i=0; i<libTokens->GetEntries(); i++) {
2496 if (!libs.IsNull()) libs += " ";
2497 libs += gSystem->BaseName(libTokens->At(i)->GetName());
2498 }
2499 delete libTokens;
2500 return retval;
2501}
2502
16a4353c 2503//______________________________________________________________________________
2504void AliAnalysisAlien::Print(Option_t *) const
2505{
2506// Print current plugin settings.
84fcd93f 2507 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2508 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2509 if (mgr && mgr->IsProofMode()) {
2510 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2511 if (TestBit(AliAnalysisGrid::kTest))
2512 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2513 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2514 if (!fProofDataSet.IsNull())
2515 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2516 if (fProofReset==1)
2517 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2518 if (fProofReset>1)
2519 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
874de7ea 2520 if (!fROOTVersion.IsNull())
2521 printf("= ROOT version requested________________________ %s\n", fROOTVersion.Data());
d3b18c4c 2522 else
2523 printf("= ROOT version requested________________________ default\n");
2524 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2525 if (!fAliRootMode.IsNull())
2526 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2527 if (fNproofWorkers)
2528 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2529 if (fNproofWorkersPerSlave)
2530 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2531 if (TestSpecialBit(kClearPackages))
2532 printf("= ClearPackages requested...\n");
2533 if (fIncludePath.Data())
2534 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2535 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2536 if (fPackages && fPackages->GetEntries()) {
2537 TIter next(fPackages);
2538 TObject *obj;
2539 TString list;
2540 while ((obj=next())) list += obj->GetName();
2541 printf("= Par files to be used: ________________________ %s\n", list.Data());
2542 }
2543 if (TestSpecialBit(kProofConnectGrid))
2544 printf("= Requested PROOF connection to grid\n");
2545 return;
2546 }
e1c22e21 2547 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2548 if (fOverwriteMode) {
2549 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2550 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2551 }
348be253 2552 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
780ec36d 2553 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO:Print");
84fcd93f 2554 printf("= Production mode:______________________________ %d\n", fProductionMode);
2555 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2556 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2557 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2558 if (fUser.Length())
84fcd93f 2559 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2560 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2561 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
780ec36d 2562 TString basedatadir = fGridDataDir;
2563 TString pattern = fDataPattern;
2564 pattern.Strip();
2565 Int_t ind = pattern.Index(" ");
2566 if (ind>=0) {
2567 basedatadir += "/%run%/";
2568 basedatadir += pattern(0, ind);
2569 pattern = pattern(ind+1, pattern.Length());
2570 }
2571 printf("= Data base directory path requested: __________ %s\n", basedatadir.Data());
2572 printf("= Data search pattern: _________________________ %s\n", pattern.Data());
84fcd93f 2573 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2574 if (fRunNumbers.Length())
84fcd93f 2575 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2576 if (fRunRange[0])
74013535 2577 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2578 if (!fRunRange[0] && !fRunNumbers.Length()) {
2579 TIter next(fInputFiles);
2580 TObject *obj;
2581 TString list;
2582 while ((obj=next())) list += obj->GetName();
84fcd93f 2583 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2584 }
2585 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2586 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2587 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2588 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2589 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2590 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2591 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2592 printf("=====================================================================\n");
2593 printf("= Job price: ___________________________________ %d\n", fPrice);
2594 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2595 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2596 if (fMaxInitFailed>0)
84fcd93f 2597 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2598 if (fMasterResubmitThreshold>0)
84fcd93f 2599 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2600 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2601 if (fNrunsPerMaster>0)
84fcd93f 2602 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2603 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2604 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2605 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2606 if (fArguments.Length())
84fcd93f 2607 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2608 if (fExecutableArgs.Length())
2609 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2610 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2611 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2612 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2613 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2614 if (fDatasetName)
84fcd93f 2615 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2616 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2617 if (fIncludePath.Data())
84fcd93f 2618 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2619 if (fCloseSE.Length())
84fcd93f 2620 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2621 if (fFriendChainName.Length())
84fcd93f 2622 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2623 if (fPackages && fPackages->GetEntries()) {
16a4353c 2624 TIter next(fPackages);
2625 TObject *obj;
2626 TString list;
2627 while ((obj=next())) list += obj->GetName();
84fcd93f 2628 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2629 }
2630}
2631
c57f56b7 2632//______________________________________________________________________________
2633void AliAnalysisAlien::SetDefaults()
2634{
2635// Set default values for everything. What cannot be filled will be left empty.
2636 if (fGridJDL) delete fGridJDL;
2637 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2638 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2639 fPrice = 1;
2640 fTTL = 30000;
2641 fSplitMaxInputFileNumber = 100;
2642 fMaxInitFailed = 0;
2643 fMasterResubmitThreshold = 0;
bb885a9e 2644 fNtestFiles = 10;
149d288c 2645 fNreplicas = 2;
a8739e8a 2646 fRunRange[0] = 0;
2647 fRunRange[1] = 0;
8e3bd06a 2648 fRunPrefix = "%d";
319593fb 2649 fNrunsPerMaster = 1;
16a4353c 2650 fMaxMergeFiles = 100;
c57f56b7 2651 fRunNumbers = "";
2652 fExecutable = "analysis.sh";
5aa94dc3 2653 fExecutableCommand = "root -b -q -x";
c57f56b7 2654 fArguments = "";
631c0b05 2655 fExecutableArgs = "";
c57f56b7 2656 fAnalysisMacro = "myAnalysis.C";
2657 fAnalysisSource = "";
2658 fAdditionalLibs = "";
2659 fSplitMode = "se";
2660 fAPIVersion = "";
2661 fROOTVersion = "";
2662 fAliROOTVersion = "";
2663 fUser = ""; // Your alien user name
2664 fGridWorkingDir = "";
2665 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2666 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2667 fFriendChainName = "";
c57f56b7 2668 fGridOutputDir = "output";
b83f84f0 2669 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2670 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2671 fInputFormat = "xml-single";
2672 fJDLName = "analysis.jdl";
c6cb3634 2673 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2674 fMergeExcludes = "";
0f389141 2675 fMergeViaJDL = 0;
348be253 2676 SetUseCopy(kTRUE);
2677 SetCheckCopy(kTRUE);
149d288c 2678 SetDefaultOutputs(kTRUE);
e1c22e21 2679 fOverwriteMode = 1;
c57f56b7 2680}
2681
a41b2882 2682//______________________________________________________________________________
2683void AliAnalysisAlien::SetFriendChainName(const char *name, const char *libnames)
2684{
2685 // Set file name for the chain of friends and optionally additional libs to be loaded.
2686 // Libs should be separated by blancs.
2687 fFriendChainName = name;
06f67114 2688 fFriendChainName.ReplaceAll(",", " ");
2689 fFriendChainName.Strip();
2690 fFriendChainName.ReplaceAll(" ", " ");
2691
a41b2882 2692 fFriendLibs = libnames;
1a3f255c 2693 if (fFriendLibs.Length()) {
2694 if(!fFriendLibs.Contains(".so"))
2695 Fatal("SetFriendChainName()", "You should provide explicit library names (with extension)");
2696 fFriendLibs.ReplaceAll(",", " ");
2697 fFriendLibs.Strip();
2698 fFriendLibs.ReplaceAll(" ", " ");
a41b2882 2699 }
2700}
2701
874de7ea 2702//______________________________________________________________________________
2703void AliAnalysisAlien::SetRootVersionForProof(const char *version)
2704{
2705// Obsolete method. Use SetROOTVersion instead
2706 Warning("SetRootVersionForProof", "Obsolete. Use SetROOTVersion instead");
95e16f01 2707 if (fROOTVersion.IsNull()) SetROOTVersion(version);
2708 else Error("SetRootVersionForProof", "ROOT version already set to %s", fROOTVersion.Data());
874de7ea 2709}
2710
0f389141 2711//______________________________________________________________________________
b3e07543 2712Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2713{
b3e07543 2714// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2715 // First check if the result is already in the output directory.
2716 if (FileExists(Form("%s/%s",aliendir,filename))) {
2717 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2718 return kFALSE;
2719 }
b3e07543 2720 // Now check the last stage done.
2721 Int_t stage = 0;
2722 while (1) {
2723 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2724 stage++;
2725 }
2726 // Next stage of merging
2727 stage++;
2728 TString pattern = "*root_archive.zip";
2729 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2730 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2731 if (res) delete res;
2732 // Write standard output to file
2733 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2734 // Count the number of files inside
2735 ifstream ifile;
2736 ifile.open(Form("Stage_%d.xml",stage));
2737 if (!ifile.good()) {
2738 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2739 return kFALSE;
b3e07543 2740 }
2741 TString line;
2742 Int_t nfiles = 0;
2743 while (!ifile.eof()) {
2744 ifile >> line;
2745 if (line.Contains("/event")) nfiles++;
7c2cd90a 2746 }
b3e07543 2747 ifile.close();
2748 if (!nfiles) {
2749 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2750 return kFALSE;
2751 } else {
2752 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2753 }
b3e07543 2754 // Copy the file in the output directory
2755 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2756// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2757 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2758 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2759 // Check if this is the last stage to be done.
2760 Bool_t laststage = (nfiles<nperchunk);
2761 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
0ad3ea1f 2762 Int_t jobId = 0;
b3e07543 2763 if (laststage) {
2764 printf("### Submiting final merging stage %d\n", stage);
2765 TString finalJDL = jdl;
2766 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2767 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
0ad3ea1f 2768 jobId = SubmitSingleJob(query);
b3e07543 2769 } else {
2770 printf("### Submiting merging stage %d\n", stage);
03335324 2771 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
0ad3ea1f 2772 jobId = SubmitSingleJob(query);
7c2cd90a 2773 }
0ad3ea1f 2774 if (!jobId) return kFALSE;
2775
2776 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
2777 fGridJobIDs.Append(Form("%d", jobId));
2778 if (!fGridStages.IsNull()) fGridStages.Append(" ");
2779 fGridStages.Append(Form("%s_merge_stage%d",
2780 laststage ? "final" : "partial", stage));
2781
b3e07543 2782 return kTRUE;
2783}
7c2cd90a 2784
105631b5 2785//______________________________________________________________________________
2786AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2787{
2788// Loat the analysis manager from a file.
2789 TFile *file = TFile::Open(fname);
2790 if (!file) {
2791 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2792 return 0;
2793 }
2794 TIter nextkey(file->GetListOfKeys());
2795 AliAnalysisManager *mgr = 0;
2796 TKey *key;
2797 while ((key=(TKey*)nextkey())) {
2798 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2799 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2800 }
2801 if (!mgr)
2802 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2803 return mgr;
2804}
2805
7c2cd90a 2806//______________________________________________________________________________
2807Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2808{
2809// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2810 if (!gGrid) return 0;
2811 printf("=> %s ------> ",query);
2812 TGridResult *res = gGrid->Command(query);
2813 if (!res) return 0;
2814 TString jobId = res->GetKey(0,"jobId");
2815 delete res;
2816 if (jobId.IsNull()) {
2817 printf("submission failed. Reason:\n");
2818 gGrid->Stdout();
2819 gGrid->Stderr();
2820 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2821 return 0;
2822 }
0ad3ea1f 2823 Int_t ijobId = jobId.Atoi();
2824 printf(" Job id: '%s' (%d)\n", jobId.Data(), ijobId);
2825 return ijobId;
7c2cd90a 2826}
2827
ec5acdce 2828//______________________________________________________________________________
2829Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2830{
2831// Merges a collection of output files using concatenation.
d66aa2a9 2832 TString scoll(collection);
2833 if (!scoll.Contains(".xml")) return kFALSE;
ec5acdce 2834 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2835 if (!coll) {
2836 ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2837 return kFALSE;
2838 }
2839 // Iterate grid collection
2840 TString outtmp;
2841 Bool_t merged = kFALSE;
2842 Int_t ifile = 0;
2843 while (coll->Next()) {
2844 TString fname = gSystem->DirName(coll->GetTURL());
2845 fname += "/";
2846 fname += output;
2847 outtmp = Form("%d_%s", ifile, output);
2848 if (!TFile::Cp(fname, outtmp)) {
2849 ::Error("MergeInfo", "Could not copy %s", fname.Data());
2850 continue;
2851 }
2852 ifile++;
2853 if (ifile<2) {
2854 gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2855 continue;
2856 }
2857 gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2858 gSystem->Exec("cp tempmerged lastmerged");
2859 }
2860 if (ifile) {
2861 gSystem->Exec(Form("cp lastmerged %s", output));
2862 gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2863 merged = kTRUE;
2864 }
2865 return merged;
2866}
2867
7c2cd90a 2868//______________________________________________________________________________
b3e07543 2869Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2870{
b3e07543 2871// Merge given output files from basedir. Basedir can be an alien output directory
2872// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2873// files in a group (ignored for xml input). Merging can be done in stages:
2874// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2875// stage=1 : works with an xml of all root_archive.zip in the output directory
2876// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2877 TString outputFile = output;
0f389141 2878 TString command;
a2f5fc01 2879 TString outputChunk;
2880 TString previousChunk = "";
b3e07543 2881 TObjArray *listoffiles = new TObjArray();
2882// listoffiles->SetOwner();
a2f5fc01 2883 Int_t countChunk = 0;
2884 Int_t countZero = nmaxmerge;
0f389141 2885 Bool_t merged = kTRUE;
d66aa2a9 2886 Bool_t isGrid = kTRUE;
a2f5fc01 2887 Int_t index = outputFile.Index("@");
2888 if (index > 0) outputFile.Remove(index);
7c2cd90a 2889 TString inputFile = outputFile;
b3e07543 2890 TString sbasedir = basedir;
2891 if (sbasedir.Contains(".xml")) {
2892 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2893 nmaxmerge = 9999999;
2894 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2895 if (!coll) {
2896 ::Error("MergeOutput", "Input XML collection empty.");
2897 return kFALSE;
2898 }
2899 // Iterate grid collection
2900 while (coll->Next()) {
2901 TString fname = gSystem->DirName(coll->GetTURL());
2902 fname += "/";
2903 fname += inputFile;
2904 listoffiles->Add(new TNamed(fname.Data(),""));
2905 }
ffbe06b9 2906 } else if (sbasedir.Contains(".txt")) {
d66aa2a9 2907 // The file having the .txt extension is expected to contain a list of
2908 // folders where the output files will be looked. For alien folders,
2909 // the full folder LFN is expected (starting with alien://)
ffbe06b9 2910 // Assume lfn's on each line
2911 TString line;
2912 ifstream in;
2913 in.open(sbasedir);
d66aa2a9 2914 if (in.fail()) {
2915 ::Error("MergeOutput", "File %s cannot be opened. Merging stopped." ,sbasedir.Data());
2916 return kTRUE;
2917 }
ffbe06b9 2918 Int_t nfiles = 0;
2919 while (in.good()) {
2920 in >> line;
2921 if (line.IsNull() || line.BeginsWith("#")) continue;
d66aa2a9 2922 line.Strip();
2923 if (!line.Contains("alien:")) isGrid = kFALSE;
2924 line += "/";
2925 line += outputFile;
ffbe06b9 2926 nfiles++;
2927 listoffiles->Add(new TNamed(line.Data(),""));
2928 }
d66aa2a9 2929 in.close();
ffbe06b9 2930 if (!nfiles) {
2931 ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data());
2932 delete listoffiles;
2933 return kFALSE;
2934 }
b3e07543 2935 } else {
2936 command = Form("find %s/ *%s", basedir, inputFile.Data());
2937 printf("command: %s\n", command.Data());
2938 TGridResult *res = gGrid->Command(command);
2939 if (!res) {
2940 ::Error("MergeOutput","No result for the find command\n");
2941 delete listoffiles;
2942 return kFALSE;
2943 }
2944 TIter nextmap(res);
2945 TMap *map = 0;
2946 while ((map=(TMap*)nextmap())) {
2947 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2948 if (!objs || !objs->GetString().Length()) {
2949 // Nothing found - skip this output
2950 delete res;
2951 delete listoffiles;
2952 return kFALSE;
2953 }
2954 listoffiles->Add(new TNamed(objs->GetName(),""));
2955 }
2956 delete res;
2957 }
2958 if (!listoffiles->GetEntries()) {
7c2cd90a 2959 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2960 delete listoffiles;
0f389141 2961 return kFALSE;
2962 }
2963
2964 TFileMerger *fm = 0;
b3e07543 2965 TIter next0(listoffiles);
2966 TObjArray *listoffilestmp = new TObjArray();
2967 listoffilestmp->SetOwner();
2968 TObject *nextfile;
2969 TString snextfile;
2970 // Keep only the files at upper level
2971 Int_t countChar = 0;
2972 while ((nextfile=next0())) {
2973 snextfile = nextfile->GetName();
2974 Int_t crtCount = snextfile.CountChar('/');
2975 if (nextfile == listoffiles->First()) countChar = crtCount;
2976 if (crtCount < countChar) countChar = crtCount;
2977 }
2978 next0.Reset();
2979 while ((nextfile=next0())) {
2980 snextfile = nextfile->GetName();
2981 Int_t crtCount = snextfile.CountChar('/');
2982 if (crtCount > countChar) {
2983 delete nextfile;
2984 continue;
2985 }
2986 listoffilestmp->Add(nextfile);
2987 }
2988 delete listoffiles;
2989 listoffiles = listoffilestmp; // Now contains 'good' files
2990 listoffiles->Print();
2991 TIter next(listoffiles);
7c2cd90a 2992 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2993 outputChunk = outputFile;
2994 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2995 // Check for existent temporary merge files
e1c22e21 2996 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2997 // Preserve old merging functionality for stage 0.
2998 if (stage==0) {
2999 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
3000 while (1) {
3001 // Skip as many input files as in a chunk
2ea55496 3002 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 3003 nextfile = next();
3004 if (!nextfile) {
2ea55496 3005 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 3006 delete listoffiles;
2ea55496 3007 return kFALSE;
3008 }
b3e07543 3009 snextfile = nextfile->GetName();
7c2cd90a 3010 }
3011 outputChunk = outputFile;
3012 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
3013 countChunk++;
3014 if (gSystem->AccessPathName(outputChunk)) continue;
3015 // Merged file with chunks up to <countChunk> found
3016 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
3017 previousChunk = outputChunk;
3018 break;
3019 }
3020 }
3021 countZero = nmaxmerge;
3022
b3e07543 3023 while ((nextfile=next())) {
3024 snextfile = nextfile->GetName();
2ea55496 3025 // Loop 'find' results and get next LFN
7c2cd90a 3026 if (countZero == nmaxmerge) {
3027 // First file in chunk - create file merger and add previous chunk if any.
d66aa2a9 3028 fm = new TFileMerger(isGrid);
7c2cd90a 3029 fm->SetFastMethod(kTRUE);
3030 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
3031 outputChunk = outputFile;
3032 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
3033 }
3034 // If last file found, put merged results in the output file
b3e07543 3035 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 3036 // Add file to be merged and decrement chunk counter.
b3e07543 3037 fm->AddFile(snextfile);
7c2cd90a 3038 countZero--;
b3e07543 3039 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 3040 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3041 // Nothing found - skip this output
3042 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 3043 merged = kFALSE;
3044 break;
7c2cd90a 3045 }
3046 fm->OutputFile(outputChunk);
3047 // Merge the outputs, then go to next chunk
3048 if (!fm->Merge()) {
3049 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 3050 merged = kFALSE;
3051 break;
7c2cd90a 3052 } else {
3053 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
3054 gSystem->Unlink(previousChunk);
3055 }
b3e07543 3056 if (nextfile == listoffiles->Last()) break;
7c2cd90a 3057 countChunk++;
3058 countZero = nmaxmerge;
3059 previousChunk = outputChunk;
0f389141 3060 }
0f389141 3061 }
b3e07543 3062 delete listoffiles;
3063 delete fm;
7c2cd90a 3064 return merged;
3065 }
3066 // Merging stage different than 0.
3067 // Move to the begining of the requested chunk.
d66aa2a9 3068 fm = new TFileMerger(isGrid);
7c2cd90a 3069 fm->SetFastMethod(kTRUE);
b3e07543 3070 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
3071 delete listoffiles;
7c2cd90a 3072 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3073 // Nothing found - skip this output
3074 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
3075 delete fm;
3076 return kFALSE;
3077 }
b3e07543 3078 fm->OutputFile(outputFile);
7c2cd90a 3079 // Merge the outputs
3080 if (!fm->Merge()) {
3081 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
3082 delete fm;
3083 return kFALSE;
3084 } else {
b3e07543 3085 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 3086 }
7c2cd90a 3087 delete fm;
3088 return kTRUE;
0f389141 3089}
3090
c57f56b7 3091//______________________________________________________________________________
3092Bool_t AliAnalysisAlien::MergeOutputs()
3093{
3094// Merge analysis outputs existing in the AliEn space.
3095 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
3096 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3097 if (!Connect()) {
3098 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
3099 return kFALSE;
0f389141 3100 }
c9e8f7fd 3101 if (fMergeViaJDL) {
3102 if (!TestBit(AliAnalysisGrid::kMerge)) {
3103 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
3104 return kFALSE;
3105 }
3106 if (fProductionMode) {
3107 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
3108 return kFALSE;
3109 }
0f389141 3110 Info("MergeOutputs", "Submitting merging JDL");
a03be957 3111 if (!SubmitMerging()) return kFALSE;
0f389141 3112 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 3113 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 3114 return kFALSE;
c57f56b7 3115 }
3116 // Get the output path
dc030a24 3117 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 3118 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 3119 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
3120 return kFALSE;
3121 }
3122 if (!fOutputFiles.Length()) {
3123 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3124 return kFALSE;
0f389141 3125 }
3126 // Check if fast read option was requested
e1c22e21 3127 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
3128 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 3129 if (fFastReadOption) {
d3339be3 3130 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
3131 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 3132 gEnv->SetValue("XNet.ConnectTimeout",50);
3133 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 3134 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 3135 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 3136 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 3137 }
e8b839ab 3138 // Make sure we change the temporary directory
3139 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 3140 // Set temporary compilation directory to current one
3141 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 3142 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3143 TIter next(list);
3144 TObjString *str;
a2f5fc01 3145 TString outputFile;
c57f56b7 3146 Bool_t merged = kTRUE;
3147 while((str=(TObjString*)next())) {
a2f5fc01 3148 outputFile = str->GetString();
3149 Int_t index = outputFile.Index("@");
3150 if (index > 0) outputFile.Remove(index);
3151 TString outputChunk = outputFile;
3152 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 3153 // Skip already merged outputs
a2f5fc01 3154 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 3155 if (fOverwriteMode) {
a2f5fc01 3156 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
3157 gSystem->Unlink(outputFile);
3158 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3159 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3160 outputChunk.Data());
3161 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3162 }
3163 } else {
a2f5fc01 3164 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 3165 continue;
3166 }
3167 } else {
a2f5fc01 3168 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3169 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3170 outputChunk.Data());
3171 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3172 }
3173 }
37d21c01 3174 if (fMergeExcludes.Contains(outputFile.Data()) ||
3175 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 3176 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 3177 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 3178 if (!merged) {
3179 Error("MergeOutputs", "Terminate() will NOT be executed");
09d5920f 3180 delete list;
3181 return kFALSE;
ff07ec61 3182 }
3183 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
3184 if (fileOpened) fileOpened->Close();
c57f56b7 3185 }
09d5920f 3186 delete list;
0f389141 3187 return kTRUE;
c57f56b7 3188}
3189
bb885a9e 3190//______________________________________________________________________________
3191void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
3192{
3193// Use the output files connected to output containers from the analysis manager
3194// rather than the files defined by SetOutputFiles
3195 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 3196 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 3197 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
3198}
3199
149d288c 3200//______________________________________________________________________________
3201void AliAnalysisAlien::SetOutputFiles(const char *list)
3202{
3203// Manually set the output files list.
3204// Removes duplicates. Not allowed if default outputs are not disabled.
3205 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3206 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
3207 return;
3208 }
3209 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
3210 fOutputFiles = "";
3211 TString slist = list;
3212 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
3213 TObjArray *arr = slist.Tokenize(" ");
3214 TObjString *os;
3215 TIter next(arr);
3216 TString sout;
3217 while ((os=(TObjString*)next())) {
3218 sout = os->GetString();
3219 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
3220 if (fOutputFiles.Contains(sout)) continue;
3221 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
3222 fOutputFiles += sout;
3223 }
3224 delete arr;
f790bc1b 3225}
149d288c 3226
3227//______________________________________________________________________________
3228void AliAnalysisAlien::SetOutputArchive(const char *list)
3229{
3230// Manually set the output archive list. Free text - you are on your own...
3231// Not allowed if default outputs are not disabled.
3232 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3233 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
3234 return;
3235 }
3236 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
3237 fOutputArchive = list;
3238}
3239
3240//______________________________________________________________________________
3241void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
3242{
3243// Setting a prefered output SE is not allowed anymore.
3244 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
3245}
3246
28ce6c9c 3247//______________________________________________________________________________
3248void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
3249{
3250// Set some PROOF special parameter.
3251 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3252 if (pair) {
3253 TObject *old = pair->Key();
3254 TObject *val = pair->Value();
3255 fProofParam.Remove(old);
3256 delete old;
3257 delete val;
3258 }
3259 fProofParam.Add(new TObjString(pname), new TObjString(value));
3260}
3261
3262//______________________________________________________________________________
3263const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
3264{
3265// Returns a special PROOF parameter.
3266 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3267 if (!pair) return 0;
3268 return pair->Value()->GetName();
3269}
3270
5513444a 3271//______________________________________________________________________________
3272Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 3273{
3274// Start remote grid analysis.
3bdcb562 3275 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3276 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
3277 if (!mgr || !mgr->IsInitialized()) {
3278 Error("StartAnalysis", "You need an initialized analysis manager for this");
3279 return kFALSE;
3280 }
3281 // Are we in PROOF mode ?
3282 if (mgr->IsProofMode()) {
baa34407 3283 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3284 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3285 if (fProofCluster.IsNull()) {
3286 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3287 return kFALSE;
3288 }
3289 if (fProofDataSet.IsNull() && !testMode) {
3290 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3291 return kFALSE;
3292 }
3293 // Set the needed environment
3294 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3295 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3296 if (fProofReset && !testMode) {
3297 if (fProofReset==1) {
3298 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3299 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3300 } else {
3301 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3302 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3303 }
3304 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3305 return kFALSE;
3306 }
baa34407 3307
3308 if (!testMode) {
3309 // Check if there is an old active session
3310 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3311 if (nsessions) {
3312 Error("StartAnalysis","You have to reset your old session first\n");
3313 return kFALSE;
3314 }
53911ca6 3315 }
3bdcb562 3316 // Do we need to change the ROOT version ? The success of this cannot be checked.
874de7ea 3317 if (!fROOTVersion.IsNull() && !testMode) {
cbccef47 3318 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"VO_ALICE@ROOT::%s\");",
874de7ea 3319 fProofCluster.Data(), fROOTVersion.Data()));
3bdcb562 3320 }
3321 // Connect to PROOF and check the status
3322 Long_t proof = 0;
27734f0e 3323 TString sworkers;
3324 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3325 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3326 if (!testMode) {
27734f0e 3327 if (!sworkers.IsNull())
3328 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3329 else
3330 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3331 } else {
3332 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3333 if (!proof) {
3334 Error("StartAnalysis", "Could not start PROOF in test mode");
3335 return kFALSE;
3336 }
3337 }
3338 if (!proof) {
3339 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3340 return kFALSE;
3341 }
27734f0e 3342 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3343 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3344 // Set proof special parameters if any
3345 TIter nextpp(&fProofParam);
3346 TObject *proofparam;
3347 while ((proofparam=nextpp())) {
3348 TString svalue = GetProofParameter(proofparam->GetName());
3349 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3350 }
3bdcb562 3351 // Is dataset existing ?
3352 if (!testMode) {
3353 TString dataset = fProofDataSet;
3354 Int_t index = dataset.Index("#");
3355 if (index>=0) dataset.Remove(index);
830acc4c 3356// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3357// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3358// return kFALSE;
3359// }
3360// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3361 }
3362 // Is ClearPackages() needed ?
3363 if (TestSpecialBit(kClearPackages)) {
3364 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3365 gROOT->ProcessLine("gProof->ClearPackages();");
3366 }
3367 // Is a given aliroot mode requested ?
3368 TList optionsList;
1ac846e9 3369 TString parLibs;
830acc4c 3370 if (!fAliRootMode.IsNull()) {
3bdcb562 3371 TString alirootMode = fAliRootMode;
3372 if (alirootMode == "default") alirootMode = "";
3373 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3374 optionsList.SetOwner();
3375 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3376 // Check the additional libs to be loaded
3377 TString extraLibs;
1ac846e9 3378 Bool_t parMode = kFALSE;
dc1ffbcb 3379 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3380 // Parse the extra libs for .so
3381 if (fAdditionalLibs.Length()) {
a41b2882 3382 TString additionalLibs = fAdditionalLibs;
3383 additionalLibs.Strip();
3384 if (additionalLibs.Length() && fFriendLibs.Length())
3385 additionalLibs += " ";
3386 additionalLibs += fFriendLibs;
3387 TObjArray *list = additionalLibs.Tokenize(" ");
3bdcb562 3388 TIter next(list);
3389 TObjString *str;
1ac846e9 3390 while((str=(TObjString*)next())) {
3391 if (str->GetString().Contains(".so")) {
3392 if (parMode) {
3393 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3394 break;
3395 }
3396 TString stmp = str->GetName();
3397 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3398 stmp.ReplaceAll(".so","");
3399 if (!extraLibs.IsNull()) extraLibs += ":";
3400 extraLibs += stmp;
3401 continue;
3402 }
3403 if (str->GetString().Contains(".par")) {
3404 // The first par file found in the list will not allow any further .so
3405 parMode = kTRUE;
3406 if (!parLibs.IsNull()) parLibs += ":";
3407 parLibs += str->GetName();
3408 continue;
3409 }
3bdcb562 3410 }
3411 if (list) delete list;
3412 }
a41b2882 3413 if (!extraLibs.IsNull()) {
3414 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3415 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3416 }
830acc4c 3417 // Check extra includes
3418 if (!fIncludePath.IsNull()) {
3419 TString includePath = fIncludePath;
3420 includePath.ReplaceAll(" ",":");
dc030a24 3421 includePath.ReplaceAll("$ALICE_ROOT/","");
3422 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3423 includePath.ReplaceAll("-I","");
dc030a24 3424 includePath.Remove(TString::kTrailing, ':');
830acc4c 3425 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3426 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3427 }
27734f0e 3428 // Check if connection to grid is requested
3429 if (TestSpecialBit(kProofConnectGrid))
3430 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3431 // Enable AliRoot par
3432 if (testMode) {
3433 // Enable proof lite package
3434 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3435 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3436 TNamed *obj = (TNamed*)optionsList.At(i);
3437 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3438 }
3439 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3440 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3441 Info("StartAnalysis", "AliRootProofLite enabled");
3442 } else {
3443 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3444 return kFALSE;
3445 }
3446 } else {
dc030a24 3447 if ( ! fAliROOTVersion.IsNull() ) {
3448 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3449 fAliROOTVersion.Data(), &optionsList))) {
3450 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3451 return kFALSE;
3452 }
3453 }
27734f0e 3454 }
1ac846e9 3455 // Enable first par files from fAdditionalLibs
3456 if (!parLibs.IsNull()) {
3457 TObjArray *list = parLibs.Tokenize(":");
3458 TIter next(list);
3459 TObjString *package;
3460 while((package=(TObjString*)next())) {
40501bb7 3461 TString spkg = package->GetName();
3462 spkg.ReplaceAll(".par", "");
3463 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3464 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3465 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3466 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3467 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3468 return kFALSE;
3469 }
3470 } else {
3471 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3472 return kFALSE;
3473 }
3474 }
3475 if (list) delete list;
3476 }
3bdcb562 3477 } else {
3478 if (fAdditionalLibs.Contains(".so") && !testMode) {
3479 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3480 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3481 return kFALSE;
3482 }
3483 }
3484 // Enable par files if requested
3485 if (fPackages && fPackages->GetEntries()) {
3486 TIter next(fPackages);
3487 TObject *package;
3488 while ((package=next())) {
1ac846e9 3489 // Skip packages already enabled
3490 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3491 TString spkg = package->GetName();
3492 spkg.ReplaceAll(".par", "");
3493 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3494 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3495 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3496 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3497 return kFALSE;
3498 }
3499 } else {
3500 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3501 return kFALSE;
3502 }
3503 }
3504 }
3505 // Do we need to load analysis source files ?
3506 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3507 if (fAnalysisSource.Length()) {
3508 TObjArray *list = fAnalysisSource.Tokenize(" ");
3509 TIter next(list);
3510 TObjString *str;
3511 while((str=(TObjString*)next())) {
3512 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3513 }
3bdcb562 3514 if (list) delete list;
3515 }
830acc4c 3516 if (testMode) {
3517 // Register dataset to proof lite.
3518 if (fFileForTestMode.IsNull()) {
3519 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3520 return kFALSE;
3521 }
3522 if (gSystem->AccessPathName(fFileForTestMode)) {
3523 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3524 return kFALSE;
3525 }
3526 TFileCollection *coll = new TFileCollection();
3527 coll->AddFromFile(fFileForTestMode);
8ddad121 3528 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3529 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3530 }
3bdcb562 3531 return kTRUE;
3532 }
c57f56b7 3533
43da816a 3534 // Check if output files have to be taken from the analysis manager
3535 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3536 // Add output files and AOD files
3537 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3538 // Add extra files registered to the analysis manager
f790bc1b 3539 TString extra = GetListOfFiles("ext");
3540 if (!extra.IsNull()) {
149d288c 3541 extra.ReplaceAll(".root", "*.root");
f790bc1b 3542 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3543 fOutputFiles += extra;
84fcd93f 3544 }
149d288c 3545 // Compose the output archive.
b83f84f0 3546 fOutputArchive = "log_archive.zip:std*@disk=1 ";
ec5acdce 3547 if (mgr->IsCollectThroughput())
95e16f01 3548 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",fOutputFiles.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 3549 else
3550 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3551 }
f7b1cbc2 3552// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3553 if (TestBit(AliAnalysisGrid::kOffline)) {
3554 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3555 \n there nor any job run. You can revise the JDL and analysis \
3556 \n macro then run the same in \"submit\" mode.");
3557 } else if (TestBit(AliAnalysisGrid::kTest)) {
3558 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3559 \n dataset.");
3560 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3561 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3562 \n space and job submitted.");
3563 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3564 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3565 if (fMergeViaJDL) CheckInputData();
5513444a 3566 return kTRUE;
c57f56b7 3567 } else {
3568 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3569 }
3570
348be253 3571 Print();
c57f56b7 3572 if (!Connect()) {
3573 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3574 return kFALSE;
16a4353c 3575 }
d3b18c4c 3576 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3577 if (!CheckInputData()) {
3578 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3579 return kFALSE;
c57f56b7 3580 }
d3339be3 3581 if (!CreateDataset(fDataPattern)) {
3582 TString serror;
3583 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3584 if (fRunNumbers.Length()) serror = "run numbers";
3585 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3586 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3587 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3588 return kFALSE;
3589 }
21ca8e59 3590 WriteAnalysisFile();
c57f56b7 3591 WriteAnalysisMacro();
3592 WriteExecutable();
3593 WriteValidationScript();
0f389141 3594 if (fMergeViaJDL) {
3595 WriteMergingMacro();
3596 WriteMergeExecutable();
3597 WriteValidationScript(kTRUE);
3598 }
5513444a 3599 if (!CreateJDL()) return kFALSE;
3600 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3601 if (testMode) {
c57f56b7 3602 // Locally testing the analysis
3603 Info("StartAnalysis", "\n_______________________________________________________________________ \
3604 \n Running analysis script in a daughter shell as on a worker node \
3605 \n_______________________________________________________________________");
149d288c 3606 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3607 TIter next(list);
3608 TObjString *str;
a2f5fc01 3609 TString outputFile;
c57f56b7 3610 while((str=(TObjString*)next())) {
a2f5fc01 3611 outputFile = str->GetString();
3612 Int_t index = outputFile.Index("@");
3613 if (index > 0) outputFile.Remove(index);
3614 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3615 }
3616 delete list;
3617 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3618 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3619// gSystem->Exec("cat stdout");
5513444a 3620 return kFALSE;
c57f56b7 3621 }
5513444a 3622 // Check if submitting is managed by LPM manager
a3e84053 3623 if (fProductionMode) {
7ac61826 3624 //TString prodfile = fJDLName;
3625 //prodfile.ReplaceAll(".jdl", ".prod");
3626 //WriteProductionFile(prodfile);
5513444a 3627 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3628 return kFALSE;
3629 }
a8739e8a 3630 // Submit AliEn job(s)
d2a409b2 3631 gGrid->Cd(fGridOutputDir);
a8739e8a 3632 TGridResult *res;
c57f56b7 3633 TString jobID = "";
0ad3ea1f 3634 fGridJobIDs = "";
3635 fGridStages = "";
d2a409b2 3636 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3637 // Submit a given xml or a set of runs
a8739e8a 3638 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3639 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3640 if (res) {
3641 const char *cjobId = res->GetKey(0,"jobId");
3642 if (!cjobId) {
a03be957 3643 gGrid->Stdout();
3644 gGrid->Stderr();
a8739e8a 3645 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3646 return kFALSE;
a8739e8a 3647 } else {
3648 Info("StartAnalysis", "\n_______________________________________________________________________ \
3649 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3650 \n_______________________________________________________________________",
3651 fJDLName.Data(), cjobId);
3652 jobID = cjobId;
0ad3ea1f 3653 if (jobID.Atoi()) {
3654 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3655 fGridJobIDs.Append(jobID);
3656 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3657 fGridStages.Append("full");
3658 }
a8739e8a 3659 }
3660 delete res;
a03be957 3661 } else {
3662 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3663 return kFALSE;
a8739e8a 3664 }
3665 } else {
d2a409b2 3666 // Submit for a range of enumeration of runs.
a03be957 3667 if (!Submit()) return kFALSE;
0ad3ea1f 3668 jobID = fGridJobIDs;
c57f56b7 3669 }
a8739e8a 3670
d88e4159 3671 if (fDropToShell) {
3672 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3673 \n You may exit at any time and terminate the job later using the option <terminate> \
3674 \n ##################################################################################", jobID.Data());
3675 gSystem->Exec("aliensh");
3676 } else {
3677 Info("StartAnalysis", "\n#### SUBMITTED JOB %s TO ALIEN QUEUE #### \
3678 \n Remember to terminate the job later using the option <terminate> \
3679 \n ##################################################################################", jobID.Data());
3680 }
5513444a 3681 return kTRUE;
c57f56b7 3682}
3683
f790bc1b 3684//______________________________________________________________________________
3685const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3686{
3687// Get a comma-separated list of output files of the requested type.
3688// Type can be (case unsensitive):
3689// aod - list of aod files (std, extensions and filters)
3690// out - list of output files connected to containers (but not aod's or extras)
3691// ext - list of extra files registered to the manager
3692// ter - list of files produced in terminate
3693 static TString files;
3694 files = "";
3695 TString stype = type;
3696 stype.ToLower();
3697 TString aodfiles, extra;
3698 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3699 if (!mgr) {
3700 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3701 return files.Data();
3702 }
3703 if (mgr->GetOutputEventHandler()) {
6be131bd 3704 aodfiles = "";
3705 if (mgr->GetOutputEventHandler()->GetFillAOD())
3706 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
f6eaa73b 3707 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs(kTRUE);
6be131bd 3708 if (!extraaod.IsNull() && mgr->GetOutputEventHandler()->GetFillExtension()) {
0c0825e4 3709 if (!aodfiles.IsNull()) aodfiles += ",";
f790bc1b 3710 aodfiles += extraaod;
3711 }
3712 }
3713 if (stype.Contains("aod")) {
3714 files = aodfiles;
3715 if (stype == "aod") return files.Data();
3716 }
3717 // Add output files that are not in the list of AOD files
3718 TString outputfiles = "";
3719 TIter next(mgr->GetOutputs());
3720 AliAnalysisDataContainer *output;
3721 const char *filename = 0;
3722 while ((output=(AliAnalysisDataContainer*)next())) {
3723 filename = output->GetFileName();
3724 if (!(strcmp(filename, "default"))) continue;
3725 if (outputfiles.Contains(filename)) continue;
3726 if (aodfiles.Contains(filename)) continue;
0c0825e4 3727 if (!outputfiles.IsNull() && strlen(filename)) outputfiles += ",";
f790bc1b 3728 outputfiles += filename;
3729 }
3730 if (stype.Contains("out")) {
0c0825e4 3731 if (!files.IsNull() && !outputfiles.IsNull()) files += ",";
f790bc1b 3732 files += outputfiles;
3733 if (stype == "out") return files.Data();
3734 }
3735 // Add extra files registered to the analysis manager
3736 TString sextra;
3737 extra = mgr->GetExtraFiles();
3738 if (!extra.IsNull()) {
3739 extra.Strip();
3740 extra.ReplaceAll(" ", ",");
3741 TObjArray *fextra = extra.Tokenize(",");
3742 TIter nextx(fextra);
3743 TObject *obj;
3744 while ((obj=nextx())) {
3745 if (aodfiles.Contains(obj->GetName())) continue;
3746 if (outputfiles.Contains(obj->GetName())) continue;
3747 if (sextra.Contains(obj->GetName())) continue;
3748 if (!sextra.IsNull()) sextra += ",";
3749 sextra += obj->GetName();
3750 }
3751 delete fextra;
3752 if (stype.Contains("ext")) {
0c0825e4 3753 if (!files.IsNull() && !sextra.IsNull()) files += ",";
f790bc1b 3754 files += sextra;
3755 }
3756 }
3757 if (stype == "ext") return files.Data();
3758 TString termfiles;
3759 if (!fTerminateFiles.IsNull()) {
3760 fTerminateFiles.Strip();
3761 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3762 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3763 TIter nextx(fextra);
3764 TObject *obj;
3765 while ((obj=nextx())) {
3766 if (aodfiles.Contains(obj->GetName())) continue;
3767 if (outputfiles.Contains(obj->GetName())) continue;
3768 if (termfiles.Contains(obj->GetName())) continue;
3769 if (sextra.Contains(obj->GetName())) continue;
3770 if (!termfiles.IsNull()) termfiles += ",";
3771 termfiles += obj->GetName();
3772 }
3773 delete fextra;
3774 }
3775 if (stype.Contains("ter")) {
3776 if (!files.IsNull() && !termfiles.IsNull()) {
3777 files += ",";
3778 files += termfiles;
3779 }
3780 }
3781 return files.Data();
3782}
3783
d2a409b2 3784//______________________________________________________________________________
a03be957 3785Bool_t AliAnalysisAlien::Submit()
d2a409b2 3786{
3787// Submit all master jobs.
3788 Int_t nmasterjobs = fInputFiles->GetEntries();
3789 Long_t tshoot = gSystem->Now();
a03be957 3790 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3791 while (fNsubmitted < nmasterjobs) {
3792 Long_t now = gSystem->Now();
3793 if ((now-tshoot)>30000) {
3794 tshoot = now;
a03be957 3795 if (!SubmitNext()) return kFALSE;
d2a409b2 3796 }
3797 }
a03be957 3798 return kTRUE;
d2a409b2 3799}
3800
0f389141 3801//______________________________________________________________________________
a03be957 3802Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3803{
3804// Submit all merging jobs.
dc030a24 3805 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3806 gGrid->Cd(fGridOutputDir);
3807 TString mergeJDLName = fExecutable;
3808 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3809 if (!fInputFiles) {
3810 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3811 return kFALSE;
3812 }
0f389141 3813 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3814 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3815 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3816 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3817 if (fOutputToRunNo) {
3818 // The output directory is the run number
3819 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3820 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3821 } else {
17b4e9a4 3822 if (!fRunNumbers.Length() && !fRunRange[0]) {
3823 // The output directory is the grid outdir
3824 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3825 runOutDir = fGridOutputDir;
3826 } else {
3827 // The output directory is the master number in 3 digits format
3828 printf("### Submitting merging job for master <%03d>\n", i);
3829 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3830 }
7c2cd90a 3831 }
3832 // Check now the number of merging stages.
2b222f7c 3833 TObjArray *list = fOutputFiles.Tokenize(",");
3834 TIter next(list);
3835 TObjString *str;
3836 TString outputFile;
3837 while((str=(TObjString*)next())) {
3838 outputFile = str->GetString();
3839 Int_t index = outputFile.Index("@");
3840 if (index > 0) outputFile.Remove(index);
37d21c01 3841 if (!fMergeExcludes.Contains(outputFile) &&
3842 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3843 }
3844 delete list;
b3e07543 3845 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3846 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3847 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3848 }
a03be957 3849 if (!ntosubmit) return kTRUE;
d88e4159 3850 if (fDropToShell) {
3851 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3852 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3853 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3854 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3855 \n ################################################################################################################");
3856 gSystem->Exec("aliensh");
3857 } else {
3858 Info("StartAnalysis", "\n #### STARTED MERGING JOBS FOR YOU #### \
3859 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL') \
3860 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3861 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3862 \n ################################################################################################################");
3863 }
a03be957 3864 return kTRUE;
0f389141 3865}
3866
d2a409b2 3867//______________________________________________________________________________
a03be957 3868Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3869{
08d5b699 3870// Submit next bunch of master jobs if the queue is free. The first master job is
3871// submitted right away, while the next will not be unless the previous was split.
3872// The plugin will not submit new master jobs if there are more that 500 jobs in
3873// waiting phase.
d2a409b2 3874 static Bool_t iscalled = kFALSE;
3875 static Int_t firstmaster = 0;
3876 static Int_t lastmaster = 0;
3877 static Int_t npermaster = 0;
a03be957 3878 if (iscalled) return kTRUE;
d2a409b2 3879 iscalled = kTRUE;
3880 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3881 Int_t ntosubmit = 0;
3882 TGridResult *res;
3883 TString jobID = "";
3bdcb562 3884 Int_t nmasterjobs = fInputFiles->GetEntries();
3885 if (!fNsubmitted) {
3886 ntosubmit = 1;
3887 if (!IsUseSubmitPolicy()) {
90d50a8c 3888 if (nmasterjobs>5)
3bdcb562 3889 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3890 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3891 ntosubmit = nmasterjobs;
3892 }
3893 } else {
d2a409b2 3894 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3895 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3896 // If last master not split, just return
a03be957 3897 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3898 // No more than 100 waiting jobs
08d5b699 3899 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3900 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3901 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3902 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3903 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3904 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3905 }
d2a409b2 3906 for (Int_t i=0; i<ntosubmit; i++) {
3907 // Submit for a range of enumeration of runs.
a03be957 3908 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3909 TString query;
cd11251e 3910 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3911 runOutDir.ReplaceAll(".xml", "");
3912 if (fOutputToRunNo)
3913 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3914 else
3915 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3916 printf("********* %s\n",query.Data());
d2a409b2 3917 res = gGrid->Command(query);
3918 if (res) {
98ca124f 3919 TString cjobId1 = res->GetKey(0,"jobId");
3920 if (!cjobId1.Length()) {
d2a409b2 3921 iscalled = kFALSE;
a03be957 3922 gGrid->Stdout();
3923 gGrid->Stderr();
3924 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3925 return kFALSE;
d2a409b2 3926 } else {
3927 Info("StartAnalysis", "\n_______________________________________________________________________ \
3928 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3929 \n_______________________________________________________________________",
98ca124f 3930 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
0ad3ea1f 3931 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3932 fGridJobIDs.Append(cjobId1);
3933 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3934 fGridStages.Append("full");
d2a409b2 3935 jobID += cjobId1;
3936 jobID += " ";
98ca124f 3937 lastmaster = cjobId1.Atoi();
d2a409b2 3938 if (!firstmaster) firstmaster = lastmaster;
3939 fNsubmitted++;
3940 }
3941 delete res;
a03be957 3942 } else {
3943 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3944 return kFALSE;
d2a409b2 3945 }
3946 }
3947 iscalled = kFALSE;
a03be957 3948 return kTRUE;
d2a409b2 3949}
3950
c57f56b7 3951//______________________________________________________________________________
3952void AliAnalysisAlien::WriteAnalysisFile()
3953{
f10e8481 3954// Write current analysis manager into the file <analysisFile>
3955 TString analysisFile = fExecutable;
3956 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3957 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3958 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3959 if (!mgr || !mgr->IsInitialized()) {
3960 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3961 return;
3962 }
3963 // Check analysis type
3964 TObject *handler;
3965 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3966 handler = (TObject*)mgr->GetInputEventHandler();
3967 if (handler) {
65eb22f8 3968 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3969 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3970 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3971 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3972 } else {
3973 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3974 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3975 }
c57f56b7 3976 }
3977 TDirectory *cdir = gDirectory;
f10e8481 3978 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3979 if (file) {
e16a394c 3980 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3981 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3982 // Unless merging makes no sense
3983 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3984 mgr->Write();
3985 delete file;
fe2d7fc2 3986 // Enable termination for local jobs
3987 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3988 }
3989 if (cdir) cdir->cd();
f10e8481 3990 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3991 }
3992 Bool_t copy = kTRUE;
d3b18c4c 3993 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3994 if (copy) {
3995 CdWork();
3996 TString workdir = gGrid->GetHomeDirectory();
3997 workdir += fGridWorkingDir;
d3b18c4c 3998 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3999 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 4000 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
4001 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 4002 }
4003}
4004
4005//______________________________________________________________________________
4006void AliAnalysisAlien::WriteAnalysisMacro()
4007{
4008// Write the analysis macro that will steer the analysis in grid mode.
4009 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4010 ofstream out;
4011 out.open(fAnalysisMacro.Data(), ios::out);
4012 if (!out.good()) {
4013 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4014 return;
4015 }
5b273635 4016 Bool_t hasSTEERBase = kFALSE;
4017 Bool_t hasESD = kFALSE;
4018 Bool_t hasAOD = kFALSE;
4019 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4020 Bool_t hasOADB = kFALSE;
5b273635 4021 Bool_t hasANALYSISalice = kFALSE;
4022 Bool_t hasCORRFW = kFALSE;
c57f56b7 4023 TString func = fAnalysisMacro;
4024 TString type = "ESD";
4025 TString comment = "// Analysis using ";
715da79c 4026 if (fMCLoop) {
4027 type = "MCGEN";
4028 comment += "MCGEN";
4029 } else {
4030 if (IsUseMCchain()) {
4031 type = "MC";
4032 comment += "MC";
4033 } else {
4034 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
4035 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
4036 type = "AOD";
4037 comment += "AOD";
4038 }
4039 }
4040 }
0df6ccf2 4041 if (type!="AOD" && fFriendChainName!="") {
4042 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
4043 return;
4044 }
c57f56b7 4045 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
4046 else comment += " data";
4047 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
4048 func.ReplaceAll(".C", "");
4049 out << "void " << func.Data() << "()" << endl;
4050 out << "{" << endl;
4051 out << comment.Data() << endl;
4052 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 4053 out << " TStopwatch timer;" << endl;
4054 out << " timer.Start();" << endl << endl;
13f0c00b 4055 // Change temp directory to current one
fdee57f1 4056 if (!IsLocalTest()) {
4057 out << "// connect to AliEn and make the chain" << endl;
4058 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
4059 }
13f0c00b 4060 out << "// Set temporary merging directory to current one" << endl;
4061 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4062 out << "// Set temporary compilation directory to current one" << endl;
4063 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 4064 // Reset existing include path
4065 out << "// Reset existing include path and add current directory first in the search" << endl;
4066 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 4067 if (!fExecutableCommand.Contains("aliroot")) {
4068 out << "// load base root libraries" << endl;
4069 out << " gSystem->Load(\"libTree\");" << endl;
4070 out << " gSystem->Load(\"libGeom\");" << endl;
4071 out << " gSystem->Load(\"libVMC\");" << endl;
4072 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4073 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4074 }
d5c6455a 4075 if (fAdditionalRootLibs.Length()) {
47a4137d 4076 // in principle libtree /lib geom libvmc etc. can go into this list, too
4077 out << "// Add aditional libraries" << endl;
4078 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4079 TIter next(list);
4080 TObjString *str;
4081 while((str=(TObjString*)next())) {
4082 if (str->GetString().Contains(".so"))
4083 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 4084 }
47a4137d 4085 if (list) delete list;
d5c6455a 4086 }
57377eb5 4087 out << "// Load analysis framework libraries" << endl;
652561d5 4088 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 4089 if (!fPackages) {
f790bc1b 4090 if (!fExecutableCommand.Contains("aliroot")) {
4091 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4092 out << " gSystem->Load(\"libESD\");" << endl;
4093 out << " gSystem->Load(\"libAOD\");" << endl;
4094 }
4e5c5506 4095 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4096 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 4097 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4098 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 4099 } else {
4e5c5506 4100 TIter next(fPackages);
4101 TObject *obj;
57377eb5 4102 TString pkgname;
57377eb5 4103 while ((obj=next())) {
4104 pkgname = obj->GetName();
4478e6f1 4105 if (pkgname == "STEERBase" ||
4106 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4107 if (pkgname == "ESD" ||
4108 pkgname == "ESD.par") hasESD = kTRUE;
4109 if (pkgname == "AOD" ||
4110 pkgname == "AOD.par") hasAOD = kTRUE;
4111 if (pkgname == "ANALYSIS" ||
4112 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4113 if (pkgname == "OADB" ||
4114 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 4115 if (pkgname == "ANALYSISalice" ||
4116 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4117 if (pkgname == "CORRFW" ||
4118 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 4119 }
4120 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 4121 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4122 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 4123 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4124 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 4125 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4126 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 4127 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4128 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4129 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4130 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 4131 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4132 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 4133 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4134 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 4135 out << "// Compile other par packages" << endl;
4136 next.Reset();
fcc9bb6f 4137 while ((obj=next())) {
4138 pkgname = obj->GetName();
4478e6f1 4139 if (pkgname == "STEERBase" ||
4140 pkgname == "STEERBase.par" ||
4141 pkgname == "ESD" ||
4142 pkgname == "ESD.par" ||
4143 pkgname == "AOD" ||
4144 pkgname == "AOD.par" ||
4145 pkgname == "ANALYSIS" ||
4146 pkgname == "ANALYSIS.par" ||
7ea40ac8 4147 pkgname == "OADB" ||
4148 pkgname == "OADB.par" ||
4478e6f1 4149 pkgname == "ANALYSISalice" ||
4150 pkgname == "ANALYSISalice.par" ||
4151 pkgname == "CORRFW" ||
4152 pkgname == "CORRFW.par") continue;
5b273635 4153 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 4154 }
4e5c5506 4155 }
b3e07543 4156 out << "// include path" << endl;
4157 // Get the include path from the interpreter and remove entries pointing to AliRoot
4158 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4159 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4160 out << " TIter nextpath(listpaths);" << endl;
4161 out << " TObjString *pname;" << endl;
4162 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4163 out << " TString current = pname->GetName();" << endl;
4164 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4165 out << " gSystem->AddIncludePath(current);" << endl;
4166 out << " }" << endl;
4167 out << " if (listpaths) delete listpaths;" << endl;
4168 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4169 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4170 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
715da79c 4171 if (fMCLoop && !fGeneratorLibs.IsNull()) {
4172 out << "// MC generator libraries" << endl;
4173 TObjArray *list = fGeneratorLibs.Tokenize(" ");
4174 TIter next(list);
4175 TObjString *str;
4176 while((str=(TObjString*)next())) {
4177 out << " gSystem->Load(\"" << str->GetName() << "\");" << endl;
4178 }
4179 delete list;
4180 }
6da75e0b 4181 if (fAdditionalLibs.Length()) {
4182 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4183 TString additionalLibs = fAdditionalLibs;
4184 additionalLibs.Strip();
4185 if (additionalLibs.Length() && fFriendLibs.Length())
4186 additionalLibs += " ";
4187 additionalLibs += fFriendLibs;
4188 TObjArray *list = additionalLibs.Tokenize(" ");
6da75e0b 4189 TIter next(list);
4190 TObjString *str;
4191 while((str=(TObjString*)next())) {
4192 if (str->GetString().Contains(".so"))
4193 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 4194 if (str->GetString().Contains(".par"))
4195 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 4196 }
715da79c 4197 delete list;
6da75e0b 4198 }
4199 out << endl;
c57f56b7 4200 out << "// analysis source to be compiled at runtime (if any)" << endl;
4201 if (fAnalysisSource.Length()) {
4202 TObjArray *list = fAnalysisSource.Tokenize(" ");
4203 TIter next(list);
4204 TObjString *str;
4205 while((str=(TObjString*)next())) {
4206 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4207 }
4208 if (list) delete list;
4209 }
4210 out << endl;
b3e07543 4211// out << " printf(\"Currently load libraries:\\n\");" << endl;
4212// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 4213 if (fFastReadOption) {
d3339be3 4214 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
4215 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 4216 out << "// fast xrootd reading enabled" << endl;
4217 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4218 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4219 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4220 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4221 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4222 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 4223 }
c57f56b7 4224 out << "// read the analysis manager from file" << endl;
f10e8481 4225 TString analysisFile = fExecutable;
4226 analysisFile.ReplaceAll(".sh", ".root");
105631b5 4227 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4228 << analysisFile << "\");" << endl;
4229 out << " if (!mgr) return;" << endl;
21ca8e59 4230 if (IsLocalTest()) {
4231 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
4232 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 4233 if (fFileForTestMode.IsNull())
4234 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
4235 else
4236 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 4237 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4b5d9e21 4238 if (!fFriendChainName.IsNull())
b9f1efa5 4239 out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\",\"" << fFriendLibs << "\");" << endl;
024dfa52 4240 if (IsUseMCchain())
4241 out << " plugin->SetUseMCchain();" << endl;
715da79c 4242 if (fMCLoop)
4243 out << " plugin->SetMCLoop(kTRUE);" << endl;
21ca8e59 4244 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 4245 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 4246 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 4247 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
4248 } else {
7a366e0b 4249 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 4250 out << " mgr->SetNSysInfo(100);" << endl;
4251 }
21ca8e59 4252 }
c57f56b7 4253 out << " mgr->PrintStatus();" << endl;
52b6a92b 4254 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 4255 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 4256 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 4257 } else {
9a8288ff 4258 if (TestBit(AliAnalysisGrid::kTest))
4259 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4260 else
4261 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 4262 }
4263 }
21ca8e59 4264 if (!IsLocalTest()) {
715da79c 4265 if (fMCLoop) {
4266 out << " mgr->SetCacheSize(0);" << endl;
4267 out << " mgr->EventLoop(" << fNMCevents << ");" << endl;
4268 } else {
4269 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
4270 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
4271 }
21ca8e59 4272 } else {
715da79c 4273 if (fMCLoop) {
4274 out << " mgr->SetCacheSize(0);" << endl;
4275 out << " mgr->EventLoop(" << fNMCevents << ");" << endl;
7435f77f
AG		 4276 } else {
4277 out << " mgr->StartAnalysis(\"localfile\");" << endl;
4278 }
242accb2 4279 }
f7498086 4280 out << " timer.Stop();" << endl;
4281 out << " timer.Print();" << endl;
c57f56b7 4282 out << "}" << endl << endl;
715da79c 4283 if (!IsLocalTest() && !fMCLoop) {
fcc9bb6f 4284 out <<"//________________________________________________________________________________" << endl;
c57f56b7 4285 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
4286 out << "{" << endl;
4287 out << "// Create a chain using url's from xml file" << endl;
242accb2 4288 out << " TString filename;" << endl;
4289 out << " Int_t run = 0;" << endl;
c85cfc0f 4290 if (IsUseMCchain()) {
4291 out << " TString treename = \"TE\";" << endl;
4292 } else {
780ec36d 4293 if (!fTreeName.IsNull()) {
4294 out << " TString treename = \"" << fTreeName << "\";" << endl;
4295 } else {
4296 out << " TString treename = type;" << endl;
4297 out << " treename.ToLower();" << endl;
4298 out << " treename += \"Tree\";" << endl;
4299 }
c85cfc0f 4300 }
e02fee64 4301 out << " printf(\"***************************************\\n\");" << endl;
4302 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
4303 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 4304 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
4305 out << " if (!coll) {" << endl;
4306 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
4307 out << " return NULL;" << endl;
4308 out << " }" << endl;
242accb2 4309 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 4310 out << " TChain *chain = new TChain(treename);" << endl;
06f67114 4311 if(!fFriendChainName.IsNull()) {
4312 out << " TList *friends = new TList();" << endl;
4313 out << " TIter nextfriend(friends);" << endl;
4314 out << " TChain *cfriend = 0;" << endl;
4315 TObjArray *list = fFriendChainName.Tokenize(" ");
4316 TIter next(list);
4317 TObjString *str;
4318 while((str=(TObjString*)next())) {
4319 out << " cfriend = new TChain(treename, \"" << str->GetName() << "\");" << endl;
4320 out << " friends->Add(cfriend);" << endl;
4321 out << " chain->AddFriend(cfriend);" << endl;
4322 }
4323 delete list;
4324// out << " TChain *chainFriend = new TChain(treename);" << endl;
0df6ccf2 4325 }
c57f56b7 4326 out << " coll->Reset();" << endl;
0df6ccf2 4327 out << " while (coll->Next()) {" << endl;
242accb2 4328 out << " filename = coll->GetTURL("");" << endl;
4329 out << " if (mgr) {" << endl;
4330 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
4331 out << " if (nrun && nrun != run) {" << endl;
4332 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
4333 out << " mgr->SetRunFromPath(nrun);" << endl;
4334 out << " run = nrun;" << endl;
4335 out << " }" << endl;
4336 out << " }" << endl;
4337 out << " chain->Add(filename);" << endl;
06f67114 4338 if(!fFriendChainName.IsNull()) {
4339 out << " TString bpath=coll->GetTURL(\"\");" << endl;
4340 out << " if (bpath.Index(\"#\") > -1) bpath.Remove(bpath.Index(\"#\"));" << endl;
4341 out << " bpath = gSystem->DirName(bpath);" << endl;
4342 out << " bpath += \"/\";" << endl;
4343 out << " TString fileFriend;" << endl;
4344 out << " nextfriend.Reset();" << endl;
4345 out << " while ((cfriend=(TChain*)nextfriend())) {" << endl;
4346 out << " fileFriend = bpath;" << endl;
4347 out << " fileFriend += cfriend->GetTitle();" << endl;
4348 out << " TFile *file = TFile::Open(fileFriend);" << endl;
4349 out << " if (file) {" << endl;
4350 out << " file->Close();" << endl;
4351 out << " cfriend->Add(fileFriend.Data());" << endl;
4352 out << " } else {" << endl;
4353 out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
4354 out << " return 0;" << endl;
4355 out << " }" << endl;
4b5d9e21 4356 out << " }" << endl;
0df6ccf2 4357 }
4358 out << " }" << endl;
c57f56b7 4359 out << " if (!chain->GetNtrees()) {" << endl;
4360 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
4361 out << " return NULL;" << endl;
4362 out << " }" << endl;
4363 out << " return chain;" << endl;
fcc9bb6f 4364 out << "}" << endl << endl;
c57f56b7 4365 }
5b273635 4366 if (hasANALYSISalice) {
4367 out <<"//________________________________________________________________________________" << endl;
4368 out << "Bool_t SetupPar(const char *package) {" << endl;
4369 out << "// Compile the package and set it up." << endl;
4370 out << " TString pkgdir = package;" << endl;
4371 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4372 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4373 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4374 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4375 out << " // Check for BUILD.sh and execute" << endl;
4376 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4377 out << " printf(\"*******************************\\n\");" << endl;
4378 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4379 out << " printf(\"*******************************\\n\");" << endl;
4380 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4381 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4382 out << " gSystem->ChangeDirectory(cdir);" << endl;
4383 out << " return kFALSE;" << endl;
4384 out << " }" << endl;
4385 out << " } else {" << endl;
4386 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4387 out << " gSystem->ChangeDirectory(cdir);" << endl;
4388 out << " return kFALSE;" << endl;
4389 out << " }" << endl;
4390 out << " // Check for SETUP.C and execute" << endl;
4391 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4392 out << " printf(\"*******************************\\n\");" << endl;
4393 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4394 out << " printf(\"*******************************\\n\");" << endl;
4395 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4396 out << " } else {" << endl;
4397 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4398 out << " gSystem->ChangeDirectory(cdir);" << endl;
4399 out << " return kFALSE;" << endl;
4400 out << " }" << endl;
4401 out << " // Restore original workdir" << endl;
4402 out << " gSystem->ChangeDirectory(cdir);" << endl;
4403 out << " return kTRUE;" << endl;
4404 out << "}" << endl;
4405 }
c57f56b7 4406 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4407 }
4408 Bool_t copy = kTRUE;
d3b18c4c 4409 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4410 if (copy) {
4411 CdWork();
4412 TString workdir = gGrid->GetHomeDirectory();
4413 workdir += fGridWorkingDir;
4414 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4415 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4416// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4417 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4418 Form("alien://%s/%s", workdir.Data(),
4419 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4420 }
4421}
4422
0f389141 4423//______________________________________________________________________________
4424void AliAnalysisAlien::WriteMergingMacro()
4425{
4426// Write a macro to merge the outputs per master job.
4427 if (!fMergeViaJDL) return;
4428 if (!fOutputFiles.Length()) {
4429 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4430 return;
4431 }
ec5acdce 4432 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
0f389141 4433 TString mergingMacro = fExecutable;
4434 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4435 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4436 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4437 ofstream out;
4438 out.open(mergingMacro.Data(), ios::out);
4439 if (!out.good()) {
4440 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4441 return;
4442 }
5b273635 4443 Bool_t hasSTEERBase = kFALSE;
4444 Bool_t hasESD = kFALSE;
4445 Bool_t hasAOD = kFALSE;
4446 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4447 Bool_t hasOADB = kFALSE;
5b273635 4448 Bool_t hasANALYSISalice = kFALSE;
4449 Bool_t hasCORRFW = kFALSE;
0f389141 4450 TString func = mergingMacro;
4451 TString comment;
4452 func.ReplaceAll(".C", "");
f47d5cb4 4453 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4454 out << "{" << endl;
4455 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4456 out << " TStopwatch timer;" << endl;
4457 out << " timer.Start();" << endl << endl;
b3e07543 4458 // Reset existing include path
4459 out << "// Reset existing include path and add current directory first in the search" << endl;
4460 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4461 if (!fExecutableCommand.Contains("aliroot")) {
4462 out << "// load base root libraries" << endl;
4463 out << " gSystem->Load(\"libTree\");" << endl;
4464 out << " gSystem->Load(\"libGeom\");" << endl;
4465 out << " gSystem->Load(\"libVMC\");" << endl;
4466 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4467 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4468 }
0f389141 4469 if (fAdditionalRootLibs.Length()) {
4470 // in principle libtree /lib geom libvmc etc. can go into this list, too
4471 out << "// Add aditional libraries" << endl;
4472 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4473 TIter next(list);
4474 TObjString *str;
4475 while((str=(TObjString*)next())) {
4476 if (str->GetString().Contains(".so"))
4477 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4478 }
4479 if (list) delete list;
4480 }
0f389141 4481 out << "// Load analysis framework libraries" << endl;
4482 if (!fPackages) {
7c2cd90a 4483 if (!fExecutableCommand.Contains("aliroot")) {
4484 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4485 out << " gSystem->Load(\"libESD\");" << endl;
4486 out << " gSystem->Load(\"libAOD\");" << endl;
4487 }
0f389141 4488 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4489 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4490 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4491 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4492 } else {
4493 TIter next(fPackages);
4494 TObject *obj;
4495 TString pkgname;
5b273635 4496 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4497 while ((obj=next())) {
4498 pkgname = obj->GetName();
4499 if (pkgname == "STEERBase" ||
4500 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4501 if (pkgname == "ESD" ||
4502 pkgname == "ESD.par") hasESD = kTRUE;
4503 if (pkgname == "AOD" ||
4504 pkgname == "AOD.par") hasAOD = kTRUE;
4505 if (pkgname == "ANALYSIS" ||
4506 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4507 if (pkgname == "OADB" ||
4508 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4509 if (pkgname == "ANALYSISalice" ||
4510 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4511 if (pkgname == "CORRFW" ||
4512 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4513 }
5b273635 4514 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4515 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4516 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4517 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4518 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4519 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4520 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4521 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4522 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4523 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4524 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4525 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4526 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4527 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4528 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4529 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4530 out << "// Compile other par packages" << endl;
4531 next.Reset();
4532 while ((obj=next())) {
4533 pkgname = obj->GetName();
4534 if (pkgname == "STEERBase" ||
4535 pkgname == "STEERBase.par" ||
4536 pkgname == "ESD" ||
4537 pkgname == "ESD.par" ||
4538 pkgname == "AOD" ||
4539 pkgname == "AOD.par" ||
4540 pkgname == "ANALYSIS" ||
4541 pkgname == "ANALYSIS.par" ||
7ea40ac8 4542 pkgname == "OADB" ||
4543 pkgname == "OADB.par" ||
0f389141 4544 pkgname == "ANALYSISalice" ||
4545 pkgname == "ANALYSISalice.par" ||
4546 pkgname == "CORRFW" ||
4547 pkgname == "CORRFW.par") continue;
5b273635 4548 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4549 }
4550 }
b3e07543 4551 out << "// include path" << endl;
4552 // Get the include path from the interpreter and remove entries pointing to AliRoot
4553 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4554 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4555 out << " TIter nextpath(listpaths);" << endl;
4556 out << " TObjString *pname;" << endl;
4557 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4558 out << " TString current = pname->GetName();" << endl;
4559 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4560 out << " gSystem->AddIncludePath(current);" << endl;
4561 out << " }" << endl;
4562 out << " if (listpaths) delete listpaths;" << endl;
4563 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4564 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4565 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0d239a91
AG		 4566 if (fMCLoop && !fGeneratorLibs.IsNull()) {
4567 out << "// MC generator libraries" << endl;
4568 TObjArray *list = fGeneratorLibs.Tokenize(" ");
4569 TIter next(list);
4570 TObjString *str;
4571 while((str=(TObjString*)next())) {
4572 out << " gSystem->Load(\"" << str->GetName() << "\");" << endl;
4573 }
4574 delete list;
4575 }
0f389141 4576 if (fAdditionalLibs.Length()) {
4577 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4578 TString additionalLibs = fAdditionalLibs;
4579 additionalLibs.Strip();
4580 if (additionalLibs.Length() && fFriendLibs.Length())
4581 additionalLibs += " ";
4582 additionalLibs += fFriendLibs;
4583 TObjArray *list = additionalLibs.Tokenize(" ");
0f389141 4584 TIter next(list);
4585 TObjString *str;
4586 while((str=(TObjString*)next())) {
4587 if (str->GetString().Contains(".so"))
4588 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4589 }
4590 if (list) delete list;
4591 }
4592 out << endl;
4593 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4594 if (fAnalysisSource.Length()) {
4595 TObjArray *list = fAnalysisSource.Tokenize(" ");
4596 TIter next(list);
4597 TObjString *str;
4598 while((str=(TObjString*)next())) {
4599 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4600 }
4601 if (list) delete list;
4602 }
149d288c 4603 out << endl;
4604
0f389141 4605 if (fFastReadOption) {
4606 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4607 out << "// fast xrootd reading enabled" << endl;
4608 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4609 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4610 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4611 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4612 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4613 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4614 }
4615 // Change temp directory to current one
fdee57f1 4616 out << "// Connect to AliEn" << endl;
4617 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
e8b839ab 4618 out << "// Set temporary merging directory to current one" << endl;
4619 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4620 out << "// Set temporary compilation directory to current one" << endl;
4621 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
7c2cd90a 4622 out << " TString outputDir = dir;" << endl;
d87f72d5 4623 if (IsMergeAOD())
4624 out << " TString outputFiles = \"" << GetListOfFiles("outaod") << "\";" << endl;
4625 else
4626 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4627 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4628 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4629 out << " TIter *iter = new TIter(list);" << endl;
4630 out << " TObjString *str;" << endl;
a2f5fc01 4631 out << " TString outputFile;" << endl;
0f389141 4632 out << " Bool_t merged = kTRUE;" << endl;
ec5acdce 4633 TString analysisFile = fExecutable;
4634 analysisFile.ReplaceAll(".sh", ".root");
4635 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4636 << analysisFile << "\");" << endl;
4637 out << " if (!mgr) {" << endl;
4638 out << " printf(\"ERROR: Analysis manager could not be extracted from file \");" << endl;
4639 out << " return;" << endl;
4640 out << " }" << endl;
d66aa2a9 4641 if (IsLocalTest()) {
0c0825e4 4642 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4643 out << " printf(\"Testing merging...\\n\");" << endl;
0c0825e4 4644 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4645 }
0f389141 4646 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4647 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4648 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4649 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4650 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4651 out << " // Skip already merged outputs" << endl;
a2f5fc01 4652 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
d66aa2a9 4653 out << " printf(\"Output file <%s> found. Not merging again.\\n\",outputFile.Data());" << endl;
0f389141 4654 out << " continue;" << endl;
4655 out << " }" << endl;
a2f5fc01 4656 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4657 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4658 out << " if (!merged) {" << endl;
a2f5fc01 4659 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4660 out << " return;" << endl;
0f389141 4661 out << " }" << endl;
4662 out << " }" << endl;
d66aa2a9 4663 if (mgr && mgr->IsCollectThroughput() && !IsLocalTest()) {
ec5acdce 4664 out << " TString infolog = \"" << mgr->GetFileInfoLog() << "\";" << endl;
4665 out << " AliAnalysisAlien::MergeInfo(infolog, outputDir);" << endl;
4666 }
7c2cd90a 4667 out << " // all outputs merged, validate" << endl;
4668 out << " ofstream out;" << endl;
4669 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4670 out << " out.close();" << endl;
4671 out << " // read the analysis manager from file" << endl;
d66aa2a9 4672 if (IsLocalTest()) {
0c0825e4 4673 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4674 out << " printf(\"Testing Terminate()...\\n\");" << endl;
0c0825e4 4675 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4676 } else {
4677 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
4678 }
5c4250fc 4679 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4680 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4681 out << " mgr->PrintStatus();" << endl;
ec5acdce 4682 if (mgr) {
4683 if (mgr->GetDebugLevel()>3) {
f866cba5 4684 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4685 } else {
9a8288ff 4686 if (TestBit(AliAnalysisGrid::kTest))
4687 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4688 else
4689 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4690 }
4691 }
9a8288ff 4692 out << " TTree *tree = NULL;" << endl;
4693 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4694 out << "}" << endl << endl;
5b273635 4695 if (hasANALYSISalice) {
4696 out <<"//________________________________________________________________________________" << endl;
4697 out << "Bool_t SetupPar(const char *package) {" << endl;
4698 out << "// Compile the package and set it up." << endl;
4699 out << " TString pkgdir = package;" << endl;
4700 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4701 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4702 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4703 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4704 out << " // Check for BUILD.sh and execute" << endl;
4705 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4706 out << " printf(\"*******************************\\n\");" << endl;
4707 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4708 out << " printf(\"*******************************\\n\");" << endl;
4709 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4710 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4711 out << " gSystem->ChangeDirectory(cdir);" << endl;
4712 out << " return kFALSE;" << endl;
4713 out << " }" << endl;
4714 out << " } else {" << endl;
4715 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4716 out << " gSystem->ChangeDirectory(cdir);" << endl;
4717 out << " return kFALSE;" << endl;
4718 out << " }" << endl;
4719 out << " // Check for SETUP.C and execute" << endl;
4720 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4721 out << " printf(\"*******************************\\n\");" << endl;
4722 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4723 out << " printf(\"*******************************\\n\");" << endl;
4724 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4725 out << " } else {" << endl;
4726 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4727 out << " gSystem->ChangeDirectory(cdir);" << endl;
4728 out << " return kFALSE;" << endl;
4729 out << " }" << endl;
4730 out << " // Restore original workdir" << endl;
4731 out << " gSystem->ChangeDirectory(cdir);" << endl;
4732 out << " return kTRUE;" << endl;
4733 out << "}" << endl;
4734 }
0f389141 4735 }
4736 Bool_t copy = kTRUE;
d3b18c4c 4737 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4738 if (copy) {
4739 CdWork();
4740 TString workdir = gGrid->GetHomeDirectory();
4741 workdir += fGridWorkingDir;
4742 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4743 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4744// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4745 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4746 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4747 }
4748}
4749
4750//______________________________________________________________________________
4751Bool_t AliAnalysisAlien::SetupPar(const char *package)
4752{
205b201f 4753// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4754// Note that for loading the compiled library. The current directory should have precedence in
4755// LD_LIBRARY_PATH
4756 TString pkgdir = package;
4757 pkgdir.ReplaceAll(".par","");
b50a41c3 4758 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4759 TString cdir = gSystem->WorkingDirectory();
4760 gSystem->ChangeDirectory(pkgdir);
4761 // Check for BUILD.sh and execute
4762 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4763 printf("**************************************************\n");
4764 printf("*** Building PAR archive %s\n", package);
4765 printf("**************************************************\n");
4766 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4767 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4768 gSystem->ChangeDirectory(cdir);
4769 return kFALSE;
4770 }
4771 } else {
4772 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4773 gSystem->ChangeDirectory(cdir);
4774 return kFALSE;
4775 }
4776 // Check for SETUP.C and execute
4777 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4778 printf("**************************************************\n");
4779 printf("*** Setup PAR archive %s\n", package);
4780 printf("**************************************************\n");
4781 gROOT->Macro("PROOF-INF/SETUP.C");
4782 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4783 } else {
4784 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4785 gSystem->ChangeDirectory(cdir);
4786 return kFALSE;
4787 }
4788 // Restore original workdir
4789 gSystem->ChangeDirectory(cdir);
4790 return kTRUE;
4791}
4792
c57f56b7 4793//______________________________________________________________________________
4794void AliAnalysisAlien::WriteExecutable()
4795{
4796// Generate the alien executable script.
5aa94dc3 4797 // Patch executable with -x to catch error code
4798 if (fExecutableCommand.Contains("root") &&
4799 fExecutableCommand.Contains("-q") &&
4800 !fExecutableCommand.Contains("-x")) fExecutableCommand += " -x";
c57f56b7 4801 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4802 ofstream out;
4803 out.open(fExecutable.Data(), ios::out);
4804 if (out.bad()) {
5513444a 4805 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4806 return;
4807 }
4808 out << "#!/bin/bash" << endl;
2a4d5166 4809 // Make sure we can properly compile par files
4810 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4811 out << "echo \"=========================================\"" << endl;
4812 out << "echo \"############## PATH : ##############\"" << endl;
4813 out << "echo $PATH" << endl;
4814 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4815 out << "echo $LD_LIBRARY_PATH" << endl;
4816 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4817 out << "echo $ROOTSYS" << endl;
4818 out << "echo \"############## which root : ##############\"" << endl;
4819 out << "which root" << endl;
4820 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4821 out << "echo $ALICE_ROOT" << endl;
4822 out << "echo \"############## which aliroot : ##############\"" << endl;
4823 out << "which aliroot" << endl;
9c5ddadc 4824 out << "echo \"############## system limits : ##############\"" << endl;
4825 out << "ulimit -a" << endl;
4826 out << "echo \"############## memory : ##############\"" << endl;
4827 out << "free -m" << endl;
c57f56b7 4828 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4829 out << fExecutableCommand << " ";
370c9880 4830 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4831 out << "RET=$?" << endl;
4832 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4833 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4834 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4835 out << " let sig=\"$RET - 128\""<<endl;
4836 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4837 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4838 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4839 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4840 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4841 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4842 out << " fi"<<endl;
4843 out << " exit $RET"<< endl;
4844 out << "fi" << endl << endl ;
4845 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4846 out << "echo \"############## memory after: ##############\"" << endl;
4847 out << "free -m" << endl;
c57f56b7 4848 }
4849 Bool_t copy = kTRUE;
d3b18c4c 4850 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4851 if (copy) {
4852 CdWork();
4853 TString workdir = gGrid->GetHomeDirectory();
4854 workdir += fGridWorkingDir;
6da4839e 4855 TString executable = TString::Format("%s/%s", workdir.Data(), fExecutable.Data());
c57f56b7 4856 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4857 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4858 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4859 executable.Data())) Fatal("","Terminating");
c57f56b7 4860 }
4861}
4862
0f389141 4863//______________________________________________________________________________
4864void AliAnalysisAlien::WriteMergeExecutable()
4865{
4866// Generate the alien executable script for the merging job.
4867 if (!fMergeViaJDL) return;
4868 TString mergeExec = fExecutable;
4869 mergeExec.ReplaceAll(".sh", "_merge.sh");
4870 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4871 ofstream out;
4872 out.open(mergeExec.Data(), ios::out);
4873 if (out.bad()) {
4874 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4875 return;
4876 }
4877 out << "#!/bin/bash" << endl;
2a4d5166 4878 // Make sure we can properly compile par files
4879 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4880 out << "echo \"=========================================\"" << endl;
4881 out << "echo \"############## PATH : ##############\"" << endl;
4882 out << "echo $PATH" << endl;
4883 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4884 out << "echo $LD_LIBRARY_PATH" << endl;
4885 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4886 out << "echo $ROOTSYS" << endl;
4887 out << "echo \"############## which root : ##############\"" << endl;
4888 out << "which root" << endl;
4889 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4890 out << "echo $ALICE_ROOT" << endl;
4891 out << "echo \"############## which aliroot : ##############\"" << endl;
4892 out << "which aliroot" << endl;
4893 out << "echo \"############## system limits : ##############\"" << endl;
4894 out << "ulimit -a" << endl;
4895 out << "echo \"############## memory : ##############\"" << endl;
4896 out << "free -m" << endl;
4897 out << "echo \"=========================================\"" << endl << endl;
0f389141 4898 TString mergeMacro = fExecutable;
4899 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4900 if (IsOneStageMerging())
4901 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4902 else
f47d5cb4 4903 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4904 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4905 out << "RET=$?" << endl;
4906 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4907 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4908 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4909 out << " let sig=\"$RET - 128\""<<endl;
4910 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4911 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4912 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4913 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4914 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4915 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4916 out << " fi"<<endl;
4917 out << " exit $RET"<< endl;
4918 out << "fi" << endl << endl ;
0f389141 4919 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4920 out << "echo \"############## memory after: ##############\"" << endl;
4921 out << "free -m" << endl;
0f389141 4922 }
4923 Bool_t copy = kTRUE;
d3b18c4c 4924 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4925 if (copy) {
4926 CdWork();
4927 TString workdir = gGrid->GetHomeDirectory();
0f389141 4928 workdir += fGridWorkingDir;
6da4839e 4929 TString executable = TString::Format("%s/%s", workdir.Data(), mergeExec.Data());
0f389141 4930 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4931 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6da4839e
AG		 4932 if (!copyLocal2Alien("WriteMergeExecutable",mergeExec.Data(),
4933 executable.Data())) Fatal("","Terminating");
0f389141 4934 }
4935}
4936
c57f56b7 4937//______________________________________________________________________________
5513444a 4938void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4939{
4940// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4941// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4942// Next lines: full_path_to_dataset XXX (XXX is a string)
4943// To submit, one has to: submit jdl XXX for all lines
4944 ofstream out;
4945 out.open(filename, ios::out);
4946 if (out.bad()) {
4947 Error("WriteProductionFile", "Bad file name: %s", filename);
4948 return;
4949 }
d3b18c4c 4950 TString workdir;
4951 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4952 workdir = gGrid->GetHomeDirectory();
5513444a 4953 workdir += fGridWorkingDir;
f5e8c702 4954 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4955 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4956 out << locjdl << " " << njobspermaster << endl;
5513444a 4957 Int_t nmasterjobs = fInputFiles->GetEntries();
4958 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4959 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4960 runOutDir.ReplaceAll(".xml", "");
4961 if (fOutputToRunNo)
4962 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4963 else
4964 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4965 }
d3b18c4c 4966 if (gGrid) {
4967 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4968 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4969// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4970 if (!copyLocal2Alien("WriteProductionFile", filename,
4971 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4972 }
5513444a 4973}
4974
4975//______________________________________________________________________________
0f389141 4976void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4977{
4978// Generate the alien validation script.
4979 // Generate the validation script
4980 TObjString *os;
d3b18c4c 4981 if (fValidationScript.IsNull()) {
4982 fValidationScript = fExecutable;
4983 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4984 }
4985 TString validationScript = fValidationScript;
4986 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4987 if (!Connect()) {
4988 Error("WriteValidationScript", "Alien connection required");
4989 return;
4990 }
d3b18c4c 4991 if (!fTerminateFiles.IsNull()) {
4992 fTerminateFiles.Strip();
4993 fTerminateFiles.ReplaceAll(" ",",");
4994 }
a2f5fc01 4995 TString outStream = "";
4996 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4997 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4998 ofstream out;
0d5d317c 4999 out.open(validationScript, ios::out);
c57f56b7 5000 out << "#!/bin/bash" << endl;
5001 out << "##################################################" << endl;
5002 out << "validateout=`dirname $0`" << endl;
5003 out << "validatetime=`date`" << endl;
5004 out << "validated=\"0\";" << endl;
5005 out << "error=0" << endl;
5006 out << "if [ -z $validateout ]" << endl;
5007 out << "then" << endl;
5008 out << " validateout=\".\"" << endl;
5009 out << "fi" << endl << endl;
5010 out << "cd $validateout;" << endl;
5011 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 5012 out << "echo \"*******************************************************\"" << outStream << endl;
5013 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 5014 out << "" << endl;
a2f5fc01 5015 out << "echo \"* Time: $validatetime \"" << outStream << endl;
5016 out << "echo \"* Dir: $validateout\"" << outStream << endl;
5017 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
5018 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5019 out << "ls -la ./" << outStream << endl;
5020 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 5021 out << "##################################################" << endl;
ebec370a 5022 out << "" << endl;
5023
5024 out << "if [ ! -f stderr ] ; then" << endl;
5025 out << " error=1" << endl;
a2f5fc01 5026 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
5027 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5028 out << "fi" << endl;
5029
b34c9f51 5030 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
5031 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
5032 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
5033 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
5034 out << "" << endl;
5035
ebec370a 5036 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
5037 out << " error=1" << endl;
a2f5fc01 5038 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
5039 out << " echo \"$parArch\" " << outStream << endl;
5040 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5041 out << "fi" << endl;
5042
5043 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
5044 out << " error=1" << endl;
a2f5fc01 5045 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
5046 out << " echo \"$segViol\" " << outStream << endl;
5047 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5048 out << "fi" << endl;
5049
5050 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
5051 out << " error=1" << endl;
a2f5fc01 5052 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
5053 out << " echo \"$segFault\" " << outStream << endl;
5054 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5055 out << "fi" << endl;
5056
b34c9f51 5057 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
5058 out << " error=1" << endl;
a2f5fc01 5059 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
5060 out << " echo \"$glibcErr\" " << outStream << endl;
5061 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 5062 out << "fi" << endl;
5063
ebec370a 5064 // Part dedicated to the specific analyses running into the train
5065
d3b18c4c 5066 TString outputFiles = fOutputFiles;
5067 if (merge && !fTerminateFiles.IsNull()) {
0c0825e4 5068 if (!outputFiles.IsNull()) outputFiles += ",";
d3b18c4c 5069 outputFiles += fTerminateFiles;
5070 }
5071 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 5072 TIter next1(arr);
a2f5fc01 5073 TString outputFile;
bbd88db8 5074 while (!merge && (os=(TObjString*)next1())) {
5075 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 5076 outputFile = os->GetString();
5077 Int_t index = outputFile.Index("@");
5078 if (index > 0) outputFile.Remove(index);
bbd88db8 5079 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 5080 if (outputFile.Contains("*")) continue;
5081 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 5082 out << " error=1" << endl;
7c2cd90a 5083 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
5084 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 5085 out << "fi" << endl;
5086 }
5087 delete arr;
7c2cd90a 5088 out << "if ! [ -f outputs_valid ] ; then" << endl;
5089 out << " error=1" << endl;
5090 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
5091 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
5092 out << "fi" << endl;
923e2ca5 5093
c57f56b7 5094 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 5095 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 5096 if (!IsKeepLogs()) {
5097 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 5098 outStream = "";
149d288c 5099 out << " rm -f std*" << endl;
5100 }
c57f56b7 5101 out << "fi" << endl;
5102
a2f5fc01 5103 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5104 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 5105 out << "cd -" << endl;
5106 out << "exit $error" << endl;
5107 }
5108 Bool_t copy = kTRUE;
d3b18c4c 5109 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 5110 if (copy) {
5111 CdWork();
5112 TString workdir = gGrid->GetHomeDirectory();
5113 workdir += fGridWorkingDir;
d3b18c4c 5114 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 5115 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 5116// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
5117 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
5118 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 5119 }
5120}