]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Fixed wrong streaming of fRandom
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
715da79c 32#include "TInterpreter.h"
c57f56b7 33#include "TFile.h"
830acc4c 34#include "TFileCollection.h"
3bdcb562 35#include "TChain.h"
c57f56b7 36#include "TObjString.h"
37#include "TObjArray.h"
77f34eae 38#include "TMacro.h"
c57f56b7 39#include "TGrid.h"
40#include "TGridResult.h"
41#include "TGridCollection.h"
42#include "TGridJDL.h"
d2a409b2 43#include "TGridJobStatusList.h"
44#include "TGridJobStatus.h"
c57f56b7 45#include "TFileMerger.h"
46#include "AliAnalysisManager.h"
77f34eae 47#include "AliAnalysisTaskCfg.h"
bb885a9e 48#include "AliVEventHandler.h"
49#include "AliAnalysisDataContainer.h"
65eb22f8 50#include "AliMultiInputEventHandler.h"
c57f56b7 51
c82bb898 52using std::ofstream;
53using std::ifstream;
54using std::ios;
55using std::endl;
c57f56b7 56ClassImp(AliAnalysisAlien)
6c395669 57#if 0
58;
59#endif
c57f56b7 60
6c395669 61namespace {
62 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
63 {
64 TString sl(Form("file:%s", loc));
65 TString sr(Form("alien://%s", rem));
66 Bool_t ret = TFile::Cp(sl, sr);
67 if (!ret) {
68 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
69 }
70 return ret;
71 }
72}
73
c57f56b7 74//______________________________________________________________________________
75AliAnalysisAlien::AliAnalysisAlien()
76 :AliAnalysisGrid(),
77 fGridJDL(NULL),
0f389141 78 fMergingJDL(NULL),
c57f56b7 79 fPrice(0),
80 fTTL(0),
81 fSplitMaxInputFileNumber(0),
82 fMaxInitFailed(0),
83 fMasterResubmitThreshold(0),
bb885a9e 84 fNtestFiles(0),
319593fb 85 fNrunsPerMaster(0),
16a4353c 86 fMaxMergeFiles(0),
b3e07543 87 fMaxMergeStages(0),
d2a409b2 88 fNsubmitted(0),
a3e84053 89 fProductionMode(0),
cd11251e 90 fOutputToRunNo(0),
0f389141 91 fMergeViaJDL(0),
92 fFastReadOption(0),
e1c22e21 93 fOverwriteMode(1),
149d288c 94 fNreplicas(2),
3bdcb562 95 fNproofWorkers(0),
27734f0e 96 fNproofWorkersPerSlave(0),
3bdcb562 97 fProofReset(0),
715da79c 98 fNMCevents(0),
99 fNMCjobs(0),
c57f56b7 100 fRunNumbers(),
101 fExecutable(),
0a1c1f7f 102 fExecutableCommand(),
c57f56b7 103 fArguments(),
631c0b05 104 fExecutableArgs(),
c57f56b7 105 fAnalysisMacro(),
106 fAnalysisSource(),
d3b18c4c 107 fValidationScript(),
d5c6455a 108 fAdditionalRootLibs(),
c57f56b7 109 fAdditionalLibs(),
715da79c 110 fGeneratorLibs(),
c57f56b7 111 fSplitMode(),
112 fAPIVersion(),
113 fROOTVersion(),
114 fAliROOTVersion(),
648174cf 115 fExternalPackages(),
c57f56b7 116 fUser(),
117 fGridWorkingDir(),
118 fGridDataDir(),
119 fDataPattern(),
120 fGridOutputDir(),
121 fOutputArchive(),
122 fOutputFiles(),
123 fInputFormat(),
e7c71df0 124 fDatasetName(),
c57f56b7 125 fJDLName(),
d3b18c4c 126 fTerminateFiles(),
bb885a9e 127 fMergeExcludes(),
37d21c01 128 fRegisterExcludes(),
f965131e 129 fIncludePath(),
bb885a9e 130 fCloseSE(),
0df6ccf2 131 fFriendChainName(),
c6cb3634 132 fJobTag(),
648174cf 133 fOutputSingle(),
5fce53f4 134 fRunPrefix(),
3bdcb562 135 fProofCluster(),
136 fProofDataSet(),
137 fFileForTestMode(),
3bdcb562 138 fAliRootMode(),
d88e4159 139 fProofProcessOpt(),
f47d5cb4 140 fMergeDirName(),
4e5c5506 141 fInputFiles(0),
28ce6c9c 142 fPackages(0),
77f34eae 143 fModules(0),
d88e4159 144 fProofParam(),
a41b2882 145 fDropToShell(true),
715da79c 146 fMCLoop(false),
a41b2882 147 fGridJobIDs(""),
148 fGridStages(""),
780ec36d 149 fFriendLibs(""),
150 fTreeName()
c57f56b7 151{
152// Dummy ctor.
153 SetDefaults();
154}
155
156//______________________________________________________________________________
157AliAnalysisAlien::AliAnalysisAlien(const char *name)
158 :AliAnalysisGrid(name),
159 fGridJDL(NULL),
0f389141 160 fMergingJDL(NULL),
c57f56b7 161 fPrice(0),
162 fTTL(0),
163 fSplitMaxInputFileNumber(0),
164 fMaxInitFailed(0),
165 fMasterResubmitThreshold(0),
bb885a9e 166 fNtestFiles(0),
319593fb 167 fNrunsPerMaster(0),
16a4353c 168 fMaxMergeFiles(0),
b3e07543 169 fMaxMergeStages(0),
d2a409b2 170 fNsubmitted(0),
a3e84053 171 fProductionMode(0),
cd11251e 172 fOutputToRunNo(0),
0f389141 173 fMergeViaJDL(0),
174 fFastReadOption(0),
e1c22e21 175 fOverwriteMode(1),
149d288c 176 fNreplicas(2),
3bdcb562 177 fNproofWorkers(0),
27734f0e 178 fNproofWorkersPerSlave(0),
3bdcb562 179 fProofReset(0),
715da79c 180 fNMCevents(0),
181 fNMCjobs(0),
c57f56b7 182 fRunNumbers(),
183 fExecutable(),
0a1c1f7f 184 fExecutableCommand(),
c57f56b7 185 fArguments(),
631c0b05 186 fExecutableArgs(),
c57f56b7 187 fAnalysisMacro(),
188 fAnalysisSource(),
d3b18c4c 189 fValidationScript(),
d5c6455a 190 fAdditionalRootLibs(),
c57f56b7 191 fAdditionalLibs(),
715da79c 192 fGeneratorLibs(),
c57f56b7 193 fSplitMode(),
194 fAPIVersion(),
195 fROOTVersion(),
196 fAliROOTVersion(),
648174cf 197 fExternalPackages(),
c57f56b7 198 fUser(),
199 fGridWorkingDir(),
200 fGridDataDir(),
201 fDataPattern(),
202 fGridOutputDir(),
203 fOutputArchive(),
204 fOutputFiles(),
205 fInputFormat(),
e7c71df0 206 fDatasetName(),
c57f56b7 207 fJDLName(),
d3b18c4c 208 fTerminateFiles(),
bb885a9e 209 fMergeExcludes(),
37d21c01 210 fRegisterExcludes(),
f965131e 211 fIncludePath(),
bb885a9e 212 fCloseSE(),
0df6ccf2 213 fFriendChainName(),
c6cb3634 214 fJobTag(),
648174cf 215 fOutputSingle(),
5fce53f4 216 fRunPrefix(),
3bdcb562 217 fProofCluster(),
218 fProofDataSet(),
219 fFileForTestMode(),
3bdcb562 220 fAliRootMode(),
d88e4159 221 fProofProcessOpt(),
f47d5cb4 222 fMergeDirName(),
4e5c5506 223 fInputFiles(0),
28ce6c9c 224 fPackages(0),
77f34eae 225 fModules(0),
d88e4159 226 fProofParam(),
0ad3ea1f 227 fDropToShell(true),
715da79c 228 fMCLoop(false),
0ad3ea1f 229 fGridJobIDs(""),
a41b2882 230 fGridStages(""),
780ec36d 231 fFriendLibs(""),
232 fTreeName()
c57f56b7 233{
234// Default ctor.
235 SetDefaults();
236}
237
238//______________________________________________________________________________
239AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
240 :AliAnalysisGrid(other),
241 fGridJDL(NULL),
0f389141 242 fMergingJDL(NULL),
c57f56b7 243 fPrice(other.fPrice),
244 fTTL(other.fTTL),
245 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
246 fMaxInitFailed(other.fMaxInitFailed),
247 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 248 fNtestFiles(other.fNtestFiles),
319593fb 249 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 250 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 251 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 252 fNsubmitted(other.fNsubmitted),
a3e84053 253 fProductionMode(other.fProductionMode),
cd11251e 254 fOutputToRunNo(other.fOutputToRunNo),
0f389141 255 fMergeViaJDL(other.fMergeViaJDL),
256 fFastReadOption(other.fFastReadOption),
257 fOverwriteMode(other.fOverwriteMode),
149d288c 258 fNreplicas(other.fNreplicas),
3bdcb562 259 fNproofWorkers(other.fNproofWorkers),
27734f0e 260 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 261 fProofReset(other.fProofReset),
715da79c 262 fNMCevents(other.fNMCevents),
263 fNMCjobs(other.fNMCjobs),
c57f56b7 264 fRunNumbers(other.fRunNumbers),
265 fExecutable(other.fExecutable),
0a1c1f7f 266 fExecutableCommand(other.fExecutableCommand),
c57f56b7 267 fArguments(other.fArguments),
631c0b05 268 fExecutableArgs(other.fExecutableArgs),
c57f56b7 269 fAnalysisMacro(other.fAnalysisMacro),
270 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 271 fValidationScript(other.fValidationScript),
d5c6455a 272 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 273 fAdditionalLibs(other.fAdditionalLibs),
715da79c 274 fGeneratorLibs(other.fGeneratorLibs),
c57f56b7 275 fSplitMode(other.fSplitMode),
276 fAPIVersion(other.fAPIVersion),
277 fROOTVersion(other.fROOTVersion),
278 fAliROOTVersion(other.fAliROOTVersion),
648174cf 279 fExternalPackages(other.fExternalPackages),
c57f56b7 280 fUser(other.fUser),
281 fGridWorkingDir(other.fGridWorkingDir),
282 fGridDataDir(other.fGridDataDir),
283 fDataPattern(other.fDataPattern),
284 fGridOutputDir(other.fGridOutputDir),
285 fOutputArchive(other.fOutputArchive),
286 fOutputFiles(other.fOutputFiles),
287 fInputFormat(other.fInputFormat),
e7c71df0 288 fDatasetName(other.fDatasetName),
c57f56b7 289 fJDLName(other.fJDLName),
d3b18c4c 290 fTerminateFiles(other.fTerminateFiles),
bb885a9e 291 fMergeExcludes(other.fMergeExcludes),
37d21c01 292 fRegisterExcludes(other.fRegisterExcludes),
f965131e 293 fIncludePath(other.fIncludePath),
bb885a9e 294 fCloseSE(other.fCloseSE),
0df6ccf2 295 fFriendChainName(other.fFriendChainName),
c6cb3634 296 fJobTag(other.fJobTag),
648174cf 297 fOutputSingle(other.fOutputSingle),
5fce53f4 298 fRunPrefix(other.fRunPrefix),
3bdcb562 299 fProofCluster(other.fProofCluster),
300 fProofDataSet(other.fProofDataSet),
301 fFileForTestMode(other.fFileForTestMode),
3bdcb562 302 fAliRootMode(other.fAliRootMode),
d88e4159 303 fProofProcessOpt(other.fProofProcessOpt),
f47d5cb4 304 fMergeDirName(other.fMergeDirName),
4e5c5506 305 fInputFiles(0),
28ce6c9c 306 fPackages(0),
77f34eae 307 fModules(0),
d88e4159 308 fProofParam(),
0ad3ea1f 309 fDropToShell(other.fDropToShell),
715da79c 310 fMCLoop(other.fMCLoop),
0ad3ea1f 311 fGridJobIDs(other.fGridJobIDs),
a41b2882 312 fGridStages(other.fGridStages),
780ec36d 313 fFriendLibs(other.fFriendLibs),
314 fTreeName(other.fTreeName)
c57f56b7 315{
316// Copy ctor.
317 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 318 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 319 fRunRange[0] = other.fRunRange[0];
320 fRunRange[1] = other.fRunRange[1];
c57f56b7 321 if (other.fInputFiles) {
322 fInputFiles = new TObjArray();
323 TIter next(other.fInputFiles);
324 TObject *obj;
325 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
326 fInputFiles->SetOwner();
327 }
4e5c5506 328 if (other.fPackages) {
329 fPackages = new TObjArray();
330 TIter next(other.fPackages);
331 TObject *obj;
332 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
333 fPackages->SetOwner();
334 }
77f34eae 335 if (other.fModules) {
336 fModules = new TObjArray();
337 fModules->SetOwner();
338 TIter next(other.fModules);
339 AliAnalysisTaskCfg *mod, *crt;
340 while ((crt=(AliAnalysisTaskCfg*)next())) {
341 mod = new AliAnalysisTaskCfg(*crt);
342 fModules->Add(mod);
343 }
344 }
c57f56b7 345}
346
347//______________________________________________________________________________
348AliAnalysisAlien::~AliAnalysisAlien()
349{
350// Destructor.
77f34eae 351 delete fGridJDL;
352 delete fMergingJDL;
353 delete fInputFiles;
354 delete fPackages;
355 delete fModules;
28ce6c9c 356 fProofParam.DeleteAll();
c57f56b7 357}
358
359//______________________________________________________________________________
360AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
361{
362// Assignment.
363 if (this != &other) {
364 AliAnalysisGrid::operator=(other);
365 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 366 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 367 fPrice = other.fPrice;
368 fTTL = other.fTTL;
369 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
370 fMaxInitFailed = other.fMaxInitFailed;
371 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 372 fNtestFiles = other.fNtestFiles;
a3e84053 373 fNrunsPerMaster = other.fNrunsPerMaster;
374 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 375 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 376 fNsubmitted = other.fNsubmitted;
377 fProductionMode = other.fProductionMode;
cd11251e 378 fOutputToRunNo = other.fOutputToRunNo;
0f389141 379 fMergeViaJDL = other.fMergeViaJDL;
380 fFastReadOption = other.fFastReadOption;
381 fOverwriteMode = other.fOverwriteMode;
149d288c 382 fNreplicas = other.fNreplicas;
3bdcb562 383 fNproofWorkers = other.fNproofWorkers;
27734f0e 384 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 385 fProofReset = other.fProofReset;
715da79c 386 fNMCevents = other.fNMCevents;
387 fNMCjobs = other.fNMCjobs;
c57f56b7 388 fRunNumbers = other.fRunNumbers;
389 fExecutable = other.fExecutable;
0a1c1f7f 390 fExecutableCommand = other.fExecutableCommand;
c57f56b7 391 fArguments = other.fArguments;
631c0b05 392 fExecutableArgs = other.fExecutableArgs;
c57f56b7 393 fAnalysisMacro = other.fAnalysisMacro;
394 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 395 fValidationScript = other.fValidationScript;
d5c6455a 396 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 397 fAdditionalLibs = other.fAdditionalLibs;
715da79c 398 fGeneratorLibs = other.fGeneratorLibs;
c57f56b7 399 fSplitMode = other.fSplitMode;
400 fAPIVersion = other.fAPIVersion;
401 fROOTVersion = other.fROOTVersion;
402 fAliROOTVersion = other.fAliROOTVersion;
648174cf 403 fExternalPackages = other.fExternalPackages;
c57f56b7 404 fUser = other.fUser;
405 fGridWorkingDir = other.fGridWorkingDir;
406 fGridDataDir = other.fGridDataDir;
407 fDataPattern = other.fDataPattern;
408 fGridOutputDir = other.fGridOutputDir;
409 fOutputArchive = other.fOutputArchive;
410 fOutputFiles = other.fOutputFiles;
411 fInputFormat = other.fInputFormat;
e7c71df0 412 fDatasetName = other.fDatasetName;
c57f56b7 413 fJDLName = other.fJDLName;
d3b18c4c 414 fTerminateFiles = other.fTerminateFiles;
bb885a9e 415 fMergeExcludes = other.fMergeExcludes;
37d21c01 416 fRegisterExcludes = other.fRegisterExcludes;
f965131e 417 fIncludePath = other.fIncludePath;
bb885a9e 418 fCloseSE = other.fCloseSE;
0df6ccf2 419 fFriendChainName = other.fFriendChainName;
c6cb3634 420 fJobTag = other.fJobTag;
648174cf 421 fOutputSingle = other.fOutputSingle;
5fce53f4 422 fRunPrefix = other.fRunPrefix;
3bdcb562 423 fProofCluster = other.fProofCluster;
424 fProofDataSet = other.fProofDataSet;
425 fFileForTestMode = other.fFileForTestMode;
3bdcb562 426 fAliRootMode = other.fAliRootMode;
d88e4159 427 fProofProcessOpt = other.fProofProcessOpt;
f47d5cb4 428 fMergeDirName = other.fMergeDirName;
d88e4159 429 fDropToShell = other.fDropToShell;
715da79c 430 fMCLoop = other.fMCLoop;
0ad3ea1f 431 fGridJobIDs = other.fGridJobIDs;
432 fGridStages = other.fGridStages;
a41b2882 433 fFriendLibs = other.fFriendLibs;
780ec36d 434 fTreeName = other.fTreeName;
c57f56b7 435 if (other.fInputFiles) {
436 fInputFiles = new TObjArray();
437 TIter next(other.fInputFiles);
438 TObject *obj;
439 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
440 fInputFiles->SetOwner();
441 }
4e5c5506 442 if (other.fPackages) {
443 fPackages = new TObjArray();
444 TIter next(other.fPackages);
445 TObject *obj;
446 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
447 fPackages->SetOwner();
448 }
77f34eae 449 if (other.fModules) {
450 fModules = new TObjArray();
451 fModules->SetOwner();
452 TIter next(other.fModules);
453 AliAnalysisTaskCfg *mod, *crt;
454 while ((crt=(AliAnalysisTaskCfg*)next())) {
455 mod = new AliAnalysisTaskCfg(*crt);
456 fModules->Add(mod);
457 }
458 }
c57f56b7 459 }
460 return *this;
461}
462
37d21c01 463//______________________________________________________________________________
464void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
465{
466// Add a single additional library to be loaded. Extension must be present.
467 TString lib(name);
468 if (!lib.Contains(".")) {
469 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
470 return;
471 }
472 if (fAdditionalLibs.Contains(name)) {
473 Warning("AddAdditionalLibrary", "Library %s already added.", name);
474 return;
475 }
476 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
477 fAdditionalLibs += lib;
478}
479
77f34eae 480//______________________________________________________________________________
481void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
482{
483// Adding a module. Checks if already existing. Becomes owned by this.
484 if (!module) return;
485 if (GetModule(module->GetName())) {
486 Error("AddModule", "A module having the same name %s already added", module->GetName());
487 return;
488 }
489 if (!fModules) {
490 fModules = new TObjArray();
491 fModules->SetOwner();
492 }
493 fModules->Add(module);
494}
495
496//______________________________________________________________________________
497void AliAnalysisAlien::AddModules(TObjArray *list)
498{
499// Adding a list of modules. Checks if already existing. Becomes owned by this.
500 TIter next(list);
501 AliAnalysisTaskCfg *module;
502 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
503}
504
505//______________________________________________________________________________
506Bool_t AliAnalysisAlien::CheckDependencies()
507{
508// Check if all dependencies are satisfied. Reorder modules if needed.
509 Int_t nmodules = GetNmodules();
510 if (!nmodules) {
511 Warning("CheckDependencies", "No modules added yet to check their dependencies");
512 return kTRUE;
513 }
514 AliAnalysisTaskCfg *mod = 0;
515 AliAnalysisTaskCfg *dep = 0;
516 TString depname;
517 Int_t i, j, k;
518 for (i=0; i<nmodules; i++) {
519 mod = (AliAnalysisTaskCfg*) fModules->At(i);
520 Int_t ndeps = mod->GetNdeps();
521 Int_t istart = i;
522 for (j=0; j<ndeps; j++) {
523 depname = mod->GetDependency(j);
524 dep = GetModule(depname);
525 if (!dep) {
526 Error("CheckDependencies","Dependency %s not added for module %s",
527 depname.Data(), mod->GetName());
528 return kFALSE;
529 }
530 if (dep->NeedsDependency(mod->GetName())) {
531 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
532 mod->GetName(), dep->GetName());
533 return kFALSE;
534 }
535 Int_t idep = fModules->IndexOf(dep);
536 // The dependency task must come first
537 if (idep>i) {
538 // Remove at idep and move all objects below up one slot
539 // down to index i included.
540 fModules->RemoveAt(idep);
3e40fd4c 541 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 542 fModules->AddAt(dep, i++);
543 }
544 //Redo from istart if dependencies were inserted
545 if (i>istart) i=istart-1;
546 }
547 }
548 return kTRUE;
549}
550
3e40fd4c 551//______________________________________________________________________________
552AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
553{
554// Create the analysis manager and optionally execute the macro in filename.
555 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
715da79c 556 if (mgr) {
557 mgr->SetMCLoop(fMCLoop);
558 return mgr;
559 }
3e40fd4c 560 mgr = new AliAnalysisManager(name);
561 mgr->SetGridHandler((AliAnalysisGrid*)this);
715da79c 562 mgr->SetMCLoop(fMCLoop);
3e40fd4c 563 if (strlen(filename)) {
564 TString line = gSystem->ExpandPathName(filename);
565 line.Prepend(".x ");
566 gROOT->ProcessLine(line.Data());
567 }
568 return mgr;
569}
570
77f34eae 571//______________________________________________________________________________
572Int_t AliAnalysisAlien::GetNmodules() const
573{
574// Get number of modules.
575 if (!fModules) return 0;
576 return fModules->GetEntries();
577}
578
579//______________________________________________________________________________
580AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
581{
582// Get a module by name.
583 if (!fModules) return 0;
584 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
585}
586
587//______________________________________________________________________________
588Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
589{
590// Load a given module.
591 if (mod->IsLoaded()) return kTRUE;
192ddca2 592 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
593 if (!mgr) {
594 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
595 return kFALSE;
596 }
77f34eae 597 Int_t ndeps = mod->GetNdeps();
598 TString depname;
599 for (Int_t j=0; j<ndeps; j++) {
600 depname = mod->GetDependency(j);
601 AliAnalysisTaskCfg *dep = GetModule(depname);
602 if (!dep) {
603 Error("LoadModule","Dependency %s not existing for module %s",
604 depname.Data(), mod->GetName());
605 return kFALSE;
606 }
607 if (!LoadModule(dep)) {
608 Error("LoadModule","Dependency %s for module %s could not be loaded",
609 depname.Data(), mod->GetName());
610 return kFALSE;
611 }
612 }
613 // Load libraries for the module
614 if (!mod->CheckLoadLibraries()) {
615 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
616 return kFALSE;
617 }
192ddca2 618 // Check if a custom file name was requested
619 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
620
621 // Check if a custom terminate file name was requested
622 if (strlen(mod->GetTerminateFileName())) {
623 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
624 fTerminateFiles += mod->GetTerminateFileName();
625 }
626
77f34eae 627 // Execute the macro
628 if (mod->ExecuteMacro()<0) {
629 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
630 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
631 return kFALSE;
632 }
633 // Configure dependencies
634 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
635 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
636 mod->GetConfigMacro()->GetTitle(), mod->GetName());
637 return kFALSE;
638 }
5e8c7c22 639 // Adjust extra libraries
640 Int_t nlibs = mod->GetNlibs();
641 TString lib;
642 for (Int_t i=0; i<nlibs; i++) {
643 lib = mod->GetLibrary(i);
5e8c7c22 644 lib = Form("lib%s.so", lib.Data());
c9d6bd12 645 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 646 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
647 fAdditionalLibs += lib;
648 }
77f34eae 649 return kTRUE;
650}
651
139fe6ba 652//______________________________________________________________________________
653Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
654{
655// Generate the full train.
656 fAdditionalLibs = "";
657 if (!LoadModules()) return kFALSE;
658 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
659 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 660 mgr->RunLocalInit();
139fe6ba 661 mgr->PrintStatus();
662 Int_t productionMode = fProductionMode;
663 SetProductionMode();
664 TString macro = fAnalysisMacro;
665 TString executable = fExecutable;
666 TString validation = fValidationScript;
667 TString execCommand = fExecutableCommand;
668 SetAnalysisMacro(Form("%s.C", name));
669 SetExecutable(Form("%s.sh", name));
a8e31fb0 670// SetExecutableCommand("aliroot -b -q ");
139fe6ba 671 SetValidationScript(Form("%s_validation.sh", name));
672 StartAnalysis();
673 SetProductionMode(productionMode);
674 fAnalysisMacro = macro;
675 fExecutable = executable;
676 fExecutableCommand = execCommand;
677 fValidationScript = validation;
678 return kTRUE;
679}
680
77f34eae 681//______________________________________________________________________________
21ca8e59 682Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 683{
684// Generate test macros for a single module or for the full train.
5e8c7c22 685 fAdditionalLibs = "";
77f34eae 686 if (strlen(modname)) {
21ca8e59 687 if (!CheckDependencies()) return kFALSE;
688 AliAnalysisTaskCfg *mod = GetModule(modname);
689 if (!mod) {
690 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
691 return kFALSE;
692 }
693 if (!LoadModule(mod)) return kFALSE;
a41b2882 694 if (!LoadFriendLibs()) return kFALSE;
21ca8e59 695 } else if (!LoadModules()) return kFALSE;
696 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
697 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 698 mgr->RunLocalInit();
21ca8e59 699 mgr->PrintStatus();
700 SetLocalTest(kTRUE);
701 Int_t productionMode = fProductionMode;
702 SetProductionMode();
703 TString macro = fAnalysisMacro;
704 TString executable = fExecutable;
705 TString validation = fValidationScript;
706 TString execCommand = fExecutableCommand;
707 SetAnalysisMacro(Form("%s.C", name));
708 SetExecutable(Form("%s.sh", name));
d66aa2a9 709 fOutputFiles = GetListOfFiles("outaod");
710 // Add extra files registered to the analysis manager
711 TString extra = GetListOfFiles("ext");
712 if (!extra.IsNull()) {
713 extra.ReplaceAll(".root", "*.root");
714 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
715 fOutputFiles += extra;
716 }
a8e31fb0 717// SetExecutableCommand("aliroot -b -q ");
21ca8e59 718 SetValidationScript(Form("%s_validation.sh", name));
719 WriteAnalysisFile();
720 WriteAnalysisMacro();
721 WriteExecutable();
722 WriteValidationScript();
ffbe06b9 723 WriteMergingMacro();
724 WriteMergeExecutable();
725 WriteValidationScript(kTRUE);
21ca8e59 726 SetLocalTest(kFALSE);
727 SetProductionMode(productionMode);
728 fAnalysisMacro = macro;
729 fExecutable = executable;
730 fExecutableCommand = execCommand;
731 fValidationScript = validation;
77f34eae 732 return kTRUE;
733}
734
735//______________________________________________________________________________
736Bool_t AliAnalysisAlien::LoadModules()
737{
738// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 739 fAdditionalLibs = "";
21ca8e59 740 Int_t nmodules = GetNmodules();
741 if (!nmodules) {
742 Warning("LoadModules", "No module to be loaded");
743 return kTRUE;
744 }
3e40fd4c 745 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
746 if (!mgr) {
747 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
748 return kFALSE;
749 }
77f34eae 750 if (!CheckDependencies()) return kFALSE;
21ca8e59 751 nmodules = GetNmodules();
77f34eae 752 AliAnalysisTaskCfg *mod;
21ca8e59 753 for (Int_t imod=0; imod<nmodules; imod++) {
754 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 755 if (!LoadModule(mod)) return kFALSE;
756 }
a41b2882 757 // Load additional friend libraries
758 return LoadFriendLibs();
77f34eae 759}
760
a41b2882 761//______________________________________________________________________________
762Bool_t AliAnalysisAlien::LoadFriendLibs() const
763{
764// Load libraries required for reading friends.
765 if (fFriendLibs.Length()) {
766 TObjArray *list = 0;
767 TString lib;
768 if (fFriendLibs.Contains(",")) list = fFriendLibs.Tokenize(",");
769 else list = fFriendLibs.Tokenize(" ");
770 for (Int_t ilib=0; ilib<list->GetEntriesFast(); ilib++) {
771 lib = list->At(ilib)->GetName();
772 lib.ReplaceAll(".so","");
773 lib.ReplaceAll(" ","");
774 if (lib.BeginsWith("lib")) lib.Remove(0, 3);
775 lib.Prepend("lib");
776 Int_t loaded = strlen(gSystem->GetLibraries(lib,"",kFALSE));
777 if (!loaded) loaded = gSystem->Load(lib);
778 if (loaded < 0) {
779 Error("LoadModules", "Cannot load library for friends %s", lib.Data());
780 return kFALSE;
781 }
782 }
783 delete list;
784 }
785 return kTRUE;
786}
787
74013535 788//______________________________________________________________________________
789void AliAnalysisAlien::SetRunPrefix(const char *prefix)
790{
791// Set the run number format. Can be a prefix or a format like "%09d"
792 fRunPrefix = prefix;
793 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
794}
795
f965131e 796//______________________________________________________________________________
797void AliAnalysisAlien::AddIncludePath(const char *path)
798{
799// Add include path in the remote analysis macro.
800 TString p(path);
801 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
802 else fIncludePath += Form("-I%s ", path);
803}
804
c57f56b7 805//______________________________________________________________________________
806void AliAnalysisAlien::AddRunNumber(Int_t run)
807{
808// Add a run number to the list of runs to be processed.
809 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 810 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 811}
812
98690895 813//______________________________________________________________________________
814void AliAnalysisAlien::AddRunList(const char* runList)
815{
816// Add several runs into the list of runs; they are expected to be separated by a blank character.
817 TString sList = runList;
818 TObjArray *list = sList.Tokenize(" ");
819 Int_t n = list->GetEntries();
820 for (Int_t i = 0; i < n; i++) {
821 TObjString *os = (TObjString*)list->At(i);
822 AddRunNumber(os->GetString().Atoi());
823 }
824 delete list;
825}
826
ee75cfc3 827//______________________________________________________________________________
828void AliAnalysisAlien::AddRunNumber(const char* run)
829{
830// Add a run number to the list of runs to be processed.
7aa5d539 831 TString runs = run;
832 TObjString *os;
833 TObjArray *arr = runs.Tokenize(" ");
834 TIter next(arr);
835 TString prefix;
836 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
837 while ((os=(TObjString*)next())){
838 if (fRunNumbers.Length()) fRunNumbers += " ";
839 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
840 }
841 delete arr;
ee75cfc3 842}
843
c57f56b7 844//______________________________________________________________________________
845void AliAnalysisAlien::AddDataFile(const char *lfn)
846{
847// Adds a data file to the input to be analysed. The file should be a valid LFN
848// or point to an existing file in the alien workdir.
849 if (!fInputFiles) fInputFiles = new TObjArray();
850 fInputFiles->Add(new TObjString(lfn));
851}
648174cf 852
853//______________________________________________________________________________
854void AliAnalysisAlien::AddExternalPackage(const char *package)
855{
856// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
857 if (fExternalPackages) fExternalPackages += " ";
858 fExternalPackages += package;
859}
860
c57f56b7 861//______________________________________________________________________________
862Bool_t AliAnalysisAlien::Connect()
863{
864// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
865 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 866 if (fProductionMode) return kTRUE;
c57f56b7 867 if (!gGrid) {
868 Info("Connect", "Trying to connect to AliEn ...");
869 TGrid::Connect("alien://");
870 }
871 if (!gGrid || !gGrid->IsConnected()) {
872 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
873 return kFALSE;
874 }
875 fUser = gGrid->GetUser();
876 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
877 return kTRUE;
878}
879
880//______________________________________________________________________________
881void AliAnalysisAlien::CdWork()
882{
883// Check validity of alien workspace. Create directory if possible.
884 if (!Connect()) {
885 Error("CdWork", "Alien connection required");
886 return;
887 }
888 TString homedir = gGrid->GetHomeDirectory();
889 TString workdir = homedir + fGridWorkingDir;
923e2ca5 890 if (DirectoryExists(workdir)) {
891 gGrid->Cd(workdir);
892 return;
893 }
894 // Work directory not existing - create it
895 gGrid->Cd(homedir);
b93f8109 896 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 897 gGrid->Cd(fGridWorkingDir);
d3b18c4c 898 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 899 } else {
d3b18c4c 900 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 901 workdir.Data(), homedir.Data());
902 fGridWorkingDir = "";
903 }
c57f56b7 904}
905
348be253 906//______________________________________________________________________________
907Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
908{
909// Check if file copying is possible.
d3b18c4c 910 if (fProductionMode) return kTRUE;
b5ae4fe0 911 TString salienpath(alienpath);
912 if (salienpath.Contains(" ")) {
913 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
914 return kFALSE;
915 }
348be253 916 if (!Connect()) {
917 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
918 return kFALSE;
919 }
d3339be3 920 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
921 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 922 // Check if alien_CLOSE_SE is defined
923 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
924 if (!closeSE.IsNull()) {
925 Info("CheckFileCopy", "Your current close storage is pointing to: \
926 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
927 } else {
928 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
929 }
930 // Check if grid directory exists.
931 if (!DirectoryExists(alienpath)) {
932 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
933 return kFALSE;
934 }
b5ae4fe0 935 TString stest = "plugin_test_copy";
936 TFile f(stest, "RECREATE");
348be253 937 // User may not have write permissions to current directory
938 if (f.IsZombie()) {
939 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
940 gSystem->WorkingDirectory());
941 return kFALSE;
942 }
943 f.Close();
b5ae4fe0 944 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
945 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 946 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 947 \n# 1. Make sure you have write permissions there. If this is the case: \
948 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
949 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
950 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
951 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 952 gSystem->Unlink(stest.Data());
348be253 953 return kFALSE;
954 }
b5ae4fe0 955 gSystem->Unlink(stest.Data());
956 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 957 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 958 return kTRUE;
959}
960
c57f56b7 961//______________________________________________________________________________
962Bool_t AliAnalysisAlien::CheckInputData()
963{
964// Check validity of input data. If necessary, create xml files.
d3b18c4c 965 if (fProductionMode) return kTRUE;
a8739e8a 966 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 967 if (!fGridDataDir.Length()) {
968 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
969 return kFALSE;
970 }
9f020952 971 if (fMergeViaJDL) {
972 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
973 return kFALSE;
974 }
d2a409b2 975 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 976 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
977 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 978 return kTRUE;
c57f56b7 979 }
980 // Process declared files
a2f5fc01 981 Bool_t isCollection = kFALSE;
982 Bool_t isXml = kFALSE;
983 Bool_t useTags = kFALSE;
c57f56b7 984 Bool_t checked = kFALSE;
d3b18c4c 985 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 986 TString file;
987 TString workdir = gGrid->GetHomeDirectory();
988 workdir += fGridWorkingDir;
989 if (fInputFiles) {
990 TObjString *objstr;
991 TIter next(fInputFiles);
992 while ((objstr=(TObjString*)next())) {
993 file = workdir;
994 file += "/";
995 file += objstr->GetString();
996 // Store full lfn path
997 if (FileExists(file)) objstr->SetString(file);
998 else {
999 file = objstr->GetName();
1000 if (!FileExists(objstr->GetName())) {
1001 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
1002 objstr->GetName(), workdir.Data());
1003 return kFALSE;
1004 }
1005 }
1006 Bool_t iscoll, isxml, usetags;
1007 CheckDataType(file, iscoll, isxml, usetags);
1008 if (!checked) {
1009 checked = kTRUE;
a2f5fc01 1010 isCollection = iscoll;
1011 isXml = isxml;
1012 useTags = usetags;
1013 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1014 } else {
a2f5fc01 1015 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 1016 Error("CheckInputData", "Some conflict was found in the types of inputs");
1017 return kFALSE;
1018 }
1019 }
1020 }
1021 }
1022 // Process requested run numbers
a8739e8a 1023 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 1024 // Check validity of alien data directory
1025 if (!fGridDataDir.Length()) {
1026 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
1027 return kFALSE;
1028 }
923e2ca5 1029 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 1030 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
1031 return kFALSE;
1032 }
a2f5fc01 1033 if (isCollection) {
c57f56b7 1034 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
1035 return kFALSE;
1036 }
1037
a2f5fc01 1038 if (checked && !isXml) {
c57f56b7 1039 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
1040 return kFALSE;
1041 }
1042 // Check validity of run number(s)
1043 TObjArray *arr;
1044 TObjString *os;
74013535 1045 TString format;
319593fb 1046 Int_t nruns = 0;
904f9f5f 1047 TString schunk, schunk2;
c57f56b7 1048 TString path;
1049 if (!checked) {
1050 checked = kTRUE;
a2f5fc01 1051 useTags = fDataPattern.Contains("tag");
1052 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1053 }
a2f5fc01 1054 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 1055 Error("CheckInputData", "Cannot mix input files using/not using tags");
1056 return kFALSE;
1057 }
1058 if (fRunNumbers.Length()) {
a8739e8a 1059 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 1060 arr = fRunNumbers.Tokenize(" ");
1061 TIter next(arr);
1062 while ((os=(TObjString*)next())) {
1063 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1064 if (!DirectoryExists(path)) {
1065 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 1066 continue;
c57f56b7 1067 }
1068 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
1069 TString msg = "\n##### file: ";
1070 msg += path;
1071 msg += " type: xml_collection;";
a2f5fc01 1072 if (useTags) msg += " using_tags: Yes";
c57f56b7 1073 else msg += " using_tags: No";
23329835 1074 Info("CheckDataType", "%s", msg.Data());
319593fb 1075 if (fNrunsPerMaster<2) {
d2a409b2 1076 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 1077 } else {
1078 nruns++;
1079 if (((nruns-1)%fNrunsPerMaster) == 0) {
1080 schunk = os->GetString();
1081 }
1082 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1083 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1084 AddDataFile(schunk);
319593fb 1085 }
c57f56b7 1086 }
1087 delete arr;
a8739e8a 1088 } else {
1089 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1090 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1091 format = Form("%%s/%s ", fRunPrefix.Data());
1092 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1093 if (!DirectoryExists(path)) {
a8739e8a 1094 continue;
1095 }
74013535 1096 format = Form("%%s/%s.xml", fRunPrefix.Data());
1097 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1098 TString msg = "\n##### file: ";
1099 msg += path;
1100 msg += " type: xml_collection;";
a2f5fc01 1101 if (useTags) msg += " using_tags: Yes";
a8739e8a 1102 else msg += " using_tags: No";
23329835 1103 Info("CheckDataType", "%s", msg.Data());
319593fb 1104 if (fNrunsPerMaster<2) {
74013535 1105 format = Form("%s.xml", fRunPrefix.Data());
1106 AddDataFile(Form(format.Data(),irun));
319593fb 1107 } else {
1108 nruns++;
1109 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1110 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1111 }
74013535 1112 format = Form("_%s.xml", fRunPrefix.Data());
1113 schunk2 = Form(format.Data(), irun);
319593fb 1114 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1115 schunk += schunk2;
d2a409b2 1116 AddDataFile(schunk);
319593fb 1117 }
a8739e8a 1118 }
904f9f5f 1119 if (!fInputFiles) {
1120 schunk += schunk2;
1121 AddDataFile(schunk);
1122 }
c57f56b7 1123 }
1124 return kTRUE;
1125}
1126
21ca8e59 1127//______________________________________________________________________________
4a8b667c 1128Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1129{
1130// Copy data from the given grid directory according a pattern and make a local
1131// dataset.
2ba4c287 1132// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1133 if (!Connect()) {
1134 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
4a8b667c 1135 return 0;
21ca8e59 1136 }
1137 if (!DirectoryExists(griddir)) {
1138 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
4a8b667c 1139 return 0;
21ca8e59 1140 }
1141 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1142 printf("Running command: %s\n", command.Data());
1143 TGridResult *res = gGrid->Command(command);
1144 Int_t nfound = res->GetEntries();
1145 if (!nfound) {
1146 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
4a8b667c 1147 return 0;
21ca8e59 1148 }
1149 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1150
1151 // archives
1152 TObjArray* additionalArchives = 0;
1153 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1154 additionalArchives = TString(archivefile).Tokenize(";");
1155 archivefile = additionalArchives->At(0)->GetName();
1156 additionalArchives->RemoveAt(0);
1157 additionalArchives->Compress();
1158 }
1159
21ca8e59 1160 // Copy files locally
1161 ofstream out;
1162 out.open(output, ios::out);
1163 TMap *map;
1164 TString turl, dirname, filename, temp;
1165 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1166 gSystem->MakeDirectory(outputdir);
1167 gSystem->ChangeDirectory(outputdir);
4a8b667c 1168 Int_t ncopied = 0;
21ca8e59 1169 for (Int_t i=0; i<nfound; i++) {
1170 map = (TMap*)res->At(i);
1171 turl = map->GetValue("turl")->GetName();
1172 filename = gSystem->BaseName(turl.Data());
1173 dirname = gSystem->DirName(turl.Data());
1174 dirname = gSystem->BaseName(dirname.Data());
1175 gSystem->MakeDirectory(dirname);
43b807f6 1176
1177 TString source(turl);
1178 TString targetFileName(filename);
1179
1180 if (strlen(archivefile) > 0) {
4a8b667c 1181// TODO here the archive in which the file resides should be determined
1182// however whereis returns only a guid, and guid2lfn does not work
1183// Therefore we use the one provided as argument for now
1184 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1185 targetFileName = archivefile;
43b807f6 1186 }
1187 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
4a8b667c 1188 Bool_t success = kTRUE;
1189 if (additionalArchives) {
1190 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1191 TString target;
1192 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1193 gSystem->MakeDirectory(gSystem->DirName(target));
1194 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1195 }
1196 }
2ba4c287 1197
4a8b667c 1198 if (success) {
1199 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1200 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1201 ncopied++;
1202 }
21ca8e59 1203 }
1204 }
1205 gSystem->ChangeDirectory(cdir);
1206 delete res;
4a8b667c 1207 delete additionalArchives;
1208 return ncopied;
21ca8e59 1209}
1210
c57f56b7 1211//______________________________________________________________________________
1212Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1213{
1214// Create dataset for the grid data directory + run number.
3af861ff 1215 const Int_t gMaxEntries = 15000;
d3b18c4c 1216 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1217 if (!Connect()) {
1218 Error("CreateDataset", "Cannot create dataset with no grid connection");
1219 return kFALSE;
1220 }
1221
1222 // Cd workspace
d3b18c4c 1223 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1224 TString workdir = gGrid->GetHomeDirectory();
1225 workdir += fGridWorkingDir;
1226
1227 // Compose the 'find' command arguments
74013535 1228 TString format;
c57f56b7 1229 TString command;
780ec36d 1230 TString delimiter = pattern;
1231 delimiter.Strip();
1232 if (delimiter.Contains(" ")) delimiter = "";
1233 else delimiter = " ";
c57f56b7 1234 TString options = "-x collection ";
bb885a9e 1235 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1236 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1237 TString conditions = "";
3af861ff 1238 Int_t nstart = 0;
1239 Int_t ncount = 0;
1240 Int_t stage = 0;
c57f56b7 1241 TString file;
1242 TString path;
319593fb 1243 Int_t nruns = 0;
904f9f5f 1244 TString schunk, schunk2;
ab254fd1 1245 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1246 if (!fRunNumbers.Length() && !fRunRange[0]) {
1247 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1248 // Make a single data collection from data directory.
1249 path = fGridDataDir;
923e2ca5 1250 if (!DirectoryExists(path)) {
d2a409b2 1251 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1252 return kFALSE;
780ec36d 1253 }
923e2ca5 1254// CdWork();
d2a409b2 1255 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1256 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1257 while (1) {
1258 ncount = 0;
1259 stage++;
1260 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1261 command = "find ";
1262 command += Form("%s -o %d ",options.Data(), nstart);
1263 command += path;
780ec36d 1264 command += delimiter;
3af861ff 1265 command += pattern;
1266 command += conditions;
1267 printf("command: %s\n", command.Data());
1268 TGridResult *res = gGrid->Command(command);
1269 if (res) delete res;
1270 // Write standard output to file
1271 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1272 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1273 Bool_t nullFile = kFALSE;
1274 if (!hasGrep) {
1275 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1276 } else {
1277 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1278 if (nullFile) {
1279 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1280 gSystem->Exec("rm -f __tmp*");
1281 return kFALSE;
1282 }
1283 TString line;
1284 ifstream in;
1285 in.open("__tmp__");
1286 in >> line;
1287 in.close();
1288 gSystem->Exec("rm -f __tmp__");
1289 ncount = line.Atoi();
1290 }
1291 }
1292 if (ncount == gMaxEntries) {
1293 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1294 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1295 if (!cbase) cbase = cadd;
1296 else {
1297 cbase->Add(cadd);
1298 delete cadd;
defd7a3a 1299 }
3af861ff 1300 nstart += ncount;
1301 } else {
1302 if (cbase) {
1303 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1304 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1305 cbase->Add(cadd);
1306 delete cadd;
1307 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1308 delete cbase; cbase = 0;
1309 } else {
1310 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1311 }
1312 gSystem->Exec("rm -f __tmp*");
1313 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1314 break;
1315 }
0f389141 1316 }
1317 Bool_t fileExists = FileExists(file);
1318 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1319 // Copy xml file to alien space
0f389141 1320 if (fileExists) gGrid->Rm(file);
d2a409b2 1321 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1322 if (!FileExists(file)) {
1323 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1324 return kFALSE;
1325 }
1326 // Update list of files to be processed.
1327 }
1328 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1329 return kTRUE;
1330 }
c57f56b7 1331 // Several runs
a2f5fc01 1332 Bool_t nullResult = kTRUE;
a8739e8a 1333 if (fRunNumbers.Length()) {
1334 TObjArray *arr = fRunNumbers.Tokenize(" ");
1335 TObjString *os;
1336 TIter next(arr);
1337 while ((os=(TObjString*)next())) {
3af861ff 1338 nstart = 0;
1339 stage = 0;
0dab9d6d 1340 path = Form("%s/%s", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1341 if (!DirectoryExists(path)) continue;
1342// CdWork();
a8739e8a 1343 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1344 else file = Form("%s.xml", os->GetString().Data());
319593fb 1345 // If local collection file does not exist, create it via 'find' command.
3af861ff 1346 while (1) {
1347 ncount = 0;
1348 stage++;
1349 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1350 command = "find ";
1351 command += Form("%s -o %d ",options.Data(), nstart);
1352 command += path;
780ec36d 1353 command += delimiter;
3af861ff 1354 command += pattern;
1355 command += conditions;
1356 TGridResult *res = gGrid->Command(command);
1357 if (res) delete res;
1358 // Write standard output to file
1359 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1360 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1361 Bool_t nullFile = kFALSE;
1362 if (!hasGrep) {
1363 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1364 } else {
1365 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1366 if (nullFile) {
1367 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1368 gSystem->Exec("rm -f __tmp*");
1369 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1370 break;
3af861ff 1371 }
1372 TString line;
1373 ifstream in;
1374 in.open("__tmp__");
1375 in >> line;
1376 in.close();
1377 gSystem->Exec("rm -f __tmp__");
1378 ncount = line.Atoi();
1379 }
1380 nullResult = kFALSE;
1381 }
1382 if (ncount == gMaxEntries) {
1383 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1384 if (fNrunsPerMaster > 1) {
1385 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1386 file.Data(),gMaxEntries);
1387 return kFALSE;
1388 }
1389 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1390 if (!cbase) cbase = cadd;
1391 else {
1392 cbase->Add(cadd);
1393 delete cadd;
defd7a3a 1394 }
3af861ff 1395 nstart += ncount;
1396 } else {
c89a84bc 1397 if (cbase && fNrunsPerMaster<2) {
3af861ff 1398 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1399 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1400 cbase->Add(cadd);
1401 delete cadd;
1402 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1403 delete cbase; cbase = 0;
1404 } else {
1405 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1406 }
1407 gSystem->Exec("rm -f __tmp*");
1408 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1409 break;
d3339be3 1410 }
3af861ff 1411 }
a8739e8a 1412 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1413 // Check if there is one run per master job.
1414 if (fNrunsPerMaster<2) {
1415 if (FileExists(file)) {
0f389141 1416 if (fOverwriteMode) gGrid->Rm(file);
1417 else {
1418 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1419 continue;
1420 }
319593fb 1421 }
1422 // Copy xml file to alien space
1423 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1424 if (!FileExists(file)) {
1425 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1426 delete arr;
1427 return kFALSE;
1428 }
1429 } else {
1430 nruns++;
1431 if (((nruns-1)%fNrunsPerMaster) == 0) {
1432 schunk = os->GetString();
1433 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1434 } else {
1435 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1436 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1437 cbase->Add(cadd);
1438 delete cadd;
1439 }
1440 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1441 continue;
1442 }
1443 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1444 if (FileExists(schunk)) {
1445 if (fOverwriteMode) gGrid->Rm(file);
1446 else {
1447 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1448 continue;
1449 }
319593fb 1450 }
84fcd93f 1451 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1452 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1453 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1454 if (!FileExists(schunk)) {
1455 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1456 delete arr;
1457 return kFALSE;
1458 }
d3339be3 1459 }
a8739e8a 1460 }
1461 delete arr;
a2f5fc01 1462 if (nullResult) {
d3339be3 1463 Error("CreateDataset", "No valid dataset corresponding to the query!");
1464 return kFALSE;
1465 }
a8739e8a 1466 } else {
1467 // Process a full run range.
1468 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
0dab9d6d 1469 format = Form("%%s/%s", fRunPrefix.Data());
3af861ff 1470 nstart = 0;
1471 stage = 0;
74013535 1472 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1473 if (!DirectoryExists(path)) continue;
1474// CdWork();
74013535 1475 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1476 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1477 else file = Form(format.Data(), irun);
0f389141 1478 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1479 if (fOverwriteMode) gGrid->Rm(file);
1480 else {
1481 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1482 continue;
1483 }
a8739e8a 1484 }
319593fb 1485 // If local collection file does not exist, create it via 'find' command.
3af861ff 1486 while (1) {
1487 ncount = 0;
1488 stage++;
1489 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1490 command = "find ";
1491 command += Form("%s -o %d ",options.Data(), nstart);
1492 command += path;
780ec36d 1493 command += delimiter;
3af861ff 1494 command += pattern;
1495 command += conditions;
1496 TGridResult *res = gGrid->Command(command);
1497 if (res) delete res;
1498 // Write standard output to file
1499 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1500 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1501 Bool_t nullFile = kFALSE;
1502 if (!hasGrep) {
1503 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1504 } else {
1505 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1506 if (nullFile) {
1507 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1508 gSystem->Exec("rm -f __tmp*");
1604c9ed 1509 break;
3af861ff 1510 }
1511 TString line;
1512 ifstream in;
1513 in.open("__tmp__");
1514 in >> line;
1515 in.close();
1516 gSystem->Exec("rm -f __tmp__");
1517 ncount = line.Atoi();
1518 }
1519 nullResult = kFALSE;
1520 }
1521 if (ncount == gMaxEntries) {
1522 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1523 if (fNrunsPerMaster > 1) {
1524 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1525 file.Data(),gMaxEntries);
1526 return kFALSE;
1527 }
1528 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1529 if (!cbase) cbase = cadd;
1530 else {
1531 cbase->Add(cadd);
1532 delete cadd;
defd7a3a 1533 }
3af861ff 1534 nstart += ncount;
1535 } else {
c89a84bc 1536 if (cbase && fNrunsPerMaster<2) {
3af861ff 1537 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1538 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1539 cbase->Add(cadd);
1540 delete cadd;
1541 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1542 delete cbase; cbase = 0;
1543 } else {
1544 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1545 }
1546 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1547 break;
d3339be3 1548 }
319593fb 1549 }
a8739e8a 1550 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1551 // Check if there is one run per master job.
1552 if (fNrunsPerMaster<2) {
1553 if (FileExists(file)) {
0f389141 1554 if (fOverwriteMode) gGrid->Rm(file);
1555 else {
1556 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1557 continue;
1558 }
319593fb 1559 }
1560 // Copy xml file to alien space
1561 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1562 if (!FileExists(file)) {
1563 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1564 return kFALSE;
1565 }
1566 } else {
1567 nruns++;
95e5b448 1568 // Check if the collection for the chunk exist locally.
1569 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1570 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1571 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1572 else continue;
1573 }
84fcd93f 1574 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1575 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1576 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1577 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1578 } else {
1579 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1580 cbase->Add(cadd);
1581 delete cadd;
1582 }
74013535 1583 format = Form("%%s_%s.xml", fRunPrefix.Data());
1584 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1585 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1586 continue;
1587 }
904f9f5f 1588 schunk = schunk2;
319593fb 1589 if (FileExists(schunk)) {
0f389141 1590 if (fOverwriteMode) gGrid->Rm(schunk);
1591 else {
1592 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1593 continue;
1594 }
319593fb 1595 }
84fcd93f 1596 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1597 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1598 if (FileExists(schunk)) {
0f389141 1599 if (fOverwriteMode) gGrid->Rm(schunk);
1600 else {
1601 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1602 continue;
1603 }
95e5b448 1604 }
319593fb 1605 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1606 if (!FileExists(schunk)) {
1607 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1608 return kFALSE;
1609 }
1610 }
c57f56b7 1611 }
a2f5fc01 1612 if (nullResult) {
d3339be3 1613 Error("CreateDataset", "No valid dataset corresponding to the query!");
1614 return kFALSE;
1615 }
a8739e8a 1616 }
c57f56b7 1617 return kTRUE;
1618}
1619
1620//______________________________________________________________________________
1621Bool_t AliAnalysisAlien::CreateJDL()
1622{
1623// Generate a JDL file according to current settings. The name of the file is
1624// specified by fJDLName.
ec5acdce 1625 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
c57f56b7 1626 Bool_t error = kFALSE;
1627 TObjArray *arr = 0;
1628 Bool_t copy = kTRUE;
d3b18c4c 1629 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1630 Bool_t generate = kTRUE;
1631 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1632 if (!Connect()) {
1633 Error("CreateJDL", "Alien connection required");
1634 return kFALSE;
1635 }
1636 // Check validity of alien workspace
d3b18c4c 1637 TString workdir;
1638 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1639 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1640 workdir += fGridWorkingDir;
1641 if (generate) {
1642 TObjString *os;
1643 if (!fInputFiles) {
1644 Error("CreateJDL()", "Define some input files for your analysis.");
1645 error = kTRUE;
1646 }
1647 // Compose list of input files
1648 // Check if output files were defined
1649 if (!fOutputFiles.Length()) {
1650 Error("CreateJDL", "You must define at least one output file");
1651 error = kTRUE;
1652 }
1653 // Check if an output directory was defined and valid
1654 if (!fGridOutputDir.Length()) {
1655 Error("CreateJDL", "You must define AliEn output directory");
1656 error = kTRUE;
1657 } else {
d3b18c4c 1658 if (!fProductionMode) {
1659 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1660 if (!DirectoryExists(fGridOutputDir)) {
1661 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1662 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1663 } else {
1664 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1665 // error = kTRUE;
1666 }
67a3b254 1667 } else {
1668 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1669 }
1670 gGrid->Cd(workdir);
1671 }
c57f56b7 1672 }
1673 // Exit if any error up to now
1674 if (error) return kFALSE;
1675 // Set JDL fields
0f389141 1676 if (!fUser.IsNull()) {
1677 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1678 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1679 }
1680 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1681 TString mergeExec = fExecutable;
1682 mergeExec.ReplaceAll(".sh", "_merge.sh");
1683 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1684 mergeExec.ReplaceAll(".sh", ".C");
1685 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1686 if (!fArguments.IsNull())
1687 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1688 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1689 else {
1690 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1691 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1692 }
b3e07543 1693
1f0d1ca2 1694 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1695 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1696 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1697 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1698
0f389141 1699 if (fMaxInitFailed > 0) {
c57f56b7 1700 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1701 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1702 }
1703 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1704 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1705 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1706 }
1707 if (!IsOneStageMerging()) {
f47d5cb4 1708 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1709 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1710 }
1711 if (fSplitMode.Length()) {
c57f56b7 1712 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1713 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1714 }
1715 fMergingJDL->SetValue("Split", "\"se\"");
1716 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1717 if (!fAliROOTVersion.IsNull()) {
1718 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1719 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1720 }
1721 if (!fROOTVersion.IsNull()) {
c57f56b7 1722 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1723 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1724 }
1725 if (!fAPIVersion.IsNull()) {
c57f56b7 1726 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1727 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1728 }
648174cf 1729 if (!fExternalPackages.IsNull()) {
1730 arr = fExternalPackages.Tokenize(" ");
1731 TIter next(arr);
1732 while ((os=(TObjString*)next())) {
1733 TString pkgname = os->GetString();
1734 Int_t index = pkgname.Index("::");
1735 TString pkgversion = pkgname(index+2, pkgname.Length());
1736 pkgname.Remove(index);
1737 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1738 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1739 }
1740 delete arr;
1741 }
0f389141 1742 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1743 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1744 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1745 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1746 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1747 TString analysisFile = fExecutable;
1748 analysisFile.ReplaceAll(".sh", ".root");
1749 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1750 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1751 if (fAdditionalLibs.Length()) {
1752 arr = fAdditionalLibs.Tokenize(" ");
1753 TIter next(arr);
1754 while ((os=(TObjString*)next())) {
1755 if (os->GetString().Contains(".so")) continue;
1756 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1757 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1758 }
1759 delete arr;
1760 }
4e5c5506 1761 if (fPackages) {
1762 TIter next(fPackages);
1763 TObject *obj;
0f389141 1764 while ((obj=next())) {
4e5c5506 1765 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1766 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1767 }
4e5c5506 1768 }
95e16f01 1769 const char *comment = "List of output files and archives";
c57f56b7 1770 if (fOutputArchive.Length()) {
e4d9a069 1771 TString outputArchive = fOutputArchive;
1772 if (!fRegisterExcludes.IsNull()) {
1773 arr = fRegisterExcludes.Tokenize(" ");
1774 TIter next1(arr);
1775 while ((os=(TObjString*)next1())) {
1776 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1777 outputArchive.ReplaceAll(os->GetString(),"");
1778 }
1779 delete arr;
1780 }
1781 arr = outputArchive.Tokenize(" ");
c57f56b7 1782 TIter next(arr);
0f389141 1783 Bool_t first = kTRUE;
0f389141 1784 while ((os=(TObjString*)next())) {
0f389141 1785 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1786 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
0f389141 1787 else
95e16f01 1788 fGridJDL->AddToSet("Output", os->GetString());
1789 if (first) fGridJDL->AddToSetDescription("Output", comment);
0f389141 1790 first = kFALSE;
1791 }
c57f56b7 1792 delete arr;
f790bc1b 1793 // Output archive for the merging jdl
f790bc1b 1794 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1795 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1796 // Add normal output files, extra files + terminate files
1797 TString files = GetListOfFiles("outextter");
37d21c01 1798 // Do not register files in fRegisterExcludes
1799 if (!fRegisterExcludes.IsNull()) {
1800 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1801 TIter next1(arr);
1802 while ((os=(TObjString*)next1())) {
1803 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1804 files.ReplaceAll(os->GetString(),"");
1805 }
66d71516 1806 delete arr;
0f389141 1807 }
f790bc1b 1808 files.ReplaceAll(".root", "*.root");
ec5acdce 1809
1810 if (mgr->IsCollectThroughput())
95e16f01 1811 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",files.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 1812 else
1813 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1814 } else {
95f84779 1815 TString files = fOutputArchive;
1816 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1817 outputArchive = files;
f790bc1b 1818 }
0f389141 1819 arr = outputArchive.Tokenize(" ");
705adb3e 1820 TIter next2(arr);
0f389141 1821 first = kTRUE;
705adb3e 1822 while ((os=(TObjString*)next2())) {
0cdf65a8 1823 TString currentfile = os->GetString();
0cdf65a8 1824 if (!currentfile.Contains("@") && fCloseSE.Length())
95e16f01 1825 fMergingJDL->AddToSet("Output", Form("%s@%s",currentfile.Data(), fCloseSE.Data()));
0f389141 1826 else
95e16f01 1827 fMergingJDL->AddToSet("Output", currentfile);
1828 if (first) fMergingJDL->AddToSetDescription("Output", comment);
0f389141 1829 first = kFALSE;
1830 }
1831 delete arr;
c57f56b7 1832 }
149d288c 1833 arr = fOutputFiles.Tokenize(",");
c57f56b7 1834 TIter next(arr);
0f389141 1835 Bool_t first = kTRUE;
43da816a 1836 while ((os=(TObjString*)next())) {
e1eaf596 1837 // Ignore ouputs in jdl that are also in outputarchive
1838 TString sout = os->GetString();
f790bc1b 1839 sout.ReplaceAll("*", "");
1840 sout.ReplaceAll(".root", "");
e1eaf596 1841 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1842 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1843 // Ignore fRegisterExcludes
1844 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1845 if (!first) comment = NULL;
43da816a 1846 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1847 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
43da816a 1848 else
95e16f01 1849 fGridJDL->AddToSet("Output", os->GetString());
1850 if (first) fGridJDL->AddToSetDescription("Output", comment);
f790bc1b 1851 if (fMergeExcludes.Contains(sout)) continue;
1852 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1853 fMergingJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
f790bc1b 1854 else
95e16f01 1855 fMergingJDL->AddToSet("Output", os->GetString());
1856 if (first) fMergingJDL->AddToSetDescription("Output", comment);
1857 first = kFALSE;
43da816a 1858 }
c57f56b7 1859 delete arr;
0f389141 1860 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1861 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1862 TString validationScript = fValidationScript;
0f389141 1863 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1864 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1865 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1866 if (fMasterResubmitThreshold) {
1867 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1868 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1869 }
d2a409b2 1870 // Write a jdl with 2 input parameters: collection name and output dir name.
1871 WriteJDL(copy);
c57f56b7 1872 }
1873 // Copy jdl to grid workspace
a8739e8a 1874 if (copy) {
b5e4aaa7 1875 // Check if an output directory was defined and valid
1876 if (!fGridOutputDir.Length()) {
1877 Error("CreateJDL", "You must define AliEn output directory");
1878 return kFALSE;
1879 } else {
1880 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1881 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1882 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1883 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1884 } else {
1885 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1886 return kFALSE;
1887 }
1888 }
1889 gGrid->Cd(workdir);
1890 }
648174cf 1891 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1892 TString mergeJDLName = fExecutable;
1893 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1894 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1895 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1896 if (fProductionMode) {
648174cf 1897 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1898 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1899 }
648174cf 1900 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1901 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1902 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1903 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1904 Fatal("","Terminating");
1905// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1906 if (fMergeViaJDL) {
1907 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1908// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1909 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1910 Fatal("","Terminating");
0f389141 1911 }
648174cf 1912 }
c57f56b7 1913 if (fAdditionalLibs.Length()) {
e7c71df0 1914 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1915 TObjString *os;
1916 TIter next(arr);
1917 while ((os=(TObjString*)next())) {
c57f56b7 1918 if (os->GetString().Contains(".so")) continue;
4e5c5506 1919 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1920 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1921// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1922 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1923 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1924 Fatal("","Terminating");
c57f56b7 1925 }
1926 delete arr;
1927 }
4e5c5506 1928 if (fPackages) {
1929 TIter next(fPackages);
1930 TObject *obj;
1931 while ((obj=next())) {
fdbbc7be 1932 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1933 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1934// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1935 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1936 Form("%s/%s", workdir.Data(), obj->GetName())))
1937 Fatal("","Terminating");
4e5c5506 1938 }
1939 }
c57f56b7 1940 }
1941 return kTRUE;
1942}
1943
a8739e8a 1944//______________________________________________________________________________
d2a409b2 1945Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1946{
1947// Writes one or more JDL's corresponding to findex. If findex is negative,
1948// all run numbers are considered in one go (jdl). For non-negative indices
1949// they correspond to the indices in the array fInputFiles.
1950 if (!fInputFiles) return kFALSE;
d3b18c4c 1951 TObject *os;
1952 TString workdir;
1953 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1954 workdir += fGridWorkingDir;
f47d5cb4 1955 TString stageName = "$2";
1956 if (fProductionMode) stageName = "$4";
1957 if (!fMergeDirName.IsNull()) {
1958 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1959 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1960 } else {
1961 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1962 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1963 }
d3b18c4c 1964 if (fProductionMode) {
a8739e8a 1965 TIter next(fInputFiles);
b3e07543 1966 while ((os=next())) {
d3b18c4c 1967 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1968 }
56754bf6 1969 if (!fOutputToRunNo)
1970 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1971 else
1972 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1973 } else {
1974 if (!fRunNumbers.Length() && !fRunRange[0]) {
1975 // One jdl with no parameters in case input data is specified by name.
1976 TIter next(fInputFiles);
1977 while ((os=next()))
1978 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1979 if (!fOutputSingle.IsNull())
1980 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1981 else {
1982 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1983 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1984 }
1985 } else {
1986 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1987 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1988 if (!fOutputSingle.IsNull()) {
1989 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1990 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1991 } else {
1992 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1993 }
1994 }
a8739e8a 1995 }
1996
a8739e8a 1997 // Generate the JDL as a string
1998 TString sjdl = fGridJDL->Generate();
0f389141 1999 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 2000 // Final merge jdl
2001 if (!fMergeDirName.IsNull()) {
2002 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
2003 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
2004 } else {
2005 fMergingJDL->SetOutputDirectory("$1", "Output directory");
2006 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
2007 }
b3e07543 2008 TString sjdl2 = fMergingJDL->Generate();
2009 Int_t index, index1;
95e16f01 2010 sjdl.ReplaceAll("\",\"", "\",\n \"");
a8739e8a 2011 sjdl.ReplaceAll("(member", "\n (member");
2012 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2013 sjdl.ReplaceAll("{", "{\n ");
2014 sjdl.ReplaceAll("};", "\n};");
2015 sjdl.ReplaceAll("{\n \n", "{\n");
2016 sjdl.ReplaceAll("\n\n", "\n");
2017 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2018 sjdl1.ReplaceAll("\",\"", "\",\n \"");
0f389141 2019 sjdl1.ReplaceAll("(member", "\n (member");
2020 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2021 sjdl1.ReplaceAll("{", "{\n ");
2022 sjdl1.ReplaceAll("};", "\n};");
2023 sjdl1.ReplaceAll("{\n \n", "{\n");
2024 sjdl1.ReplaceAll("\n\n", "\n");
2025 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2026 sjdl2.ReplaceAll("\",\"", "\",\n \"");
b3e07543 2027 sjdl2.ReplaceAll("(member", "\n (member");
2028 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2029 sjdl2.ReplaceAll("{", "{\n ");
2030 sjdl2.ReplaceAll("};", "\n};");
2031 sjdl2.ReplaceAll("{\n \n", "{\n");
2032 sjdl2.ReplaceAll("\n\n", "\n");
2033 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 2034 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 2035 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 2036 index = sjdl.Index("JDLVariables");
2037 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 2038 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2039 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 2040 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 2041 index = fJobTag.Index(":");
2042 if (index < 0) index = fJobTag.Length();
2043 TString jobTag = fJobTag;
f47d5cb4 2044 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 2045 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 2046 if (fProductionMode) {
2047 sjdl1.Prepend("# Generated merging jdl (production mode) \
2048 \n# $1 = full alien path to output directory to be merged \
2049 \n# $2 = train number \
2050 \n# $3 = production (like LHC10b) \
2051 \n# $4 = merging stage \
2052 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2053 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2054 sjdl2.Prepend("# Generated merging jdl \
2055 \n# $1 = full alien path to output directory to be merged \
2056 \n# $2 = train number \
2057 \n# $3 = production (like LHC10b) \
2058 \n# $4 = merging stage \
2059 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2060 } else {
2061 sjdl1.Prepend("# Generated merging jdl \
2062 \n# $1 = full alien path to output directory to be merged \
2063 \n# $2 = merging stage \
03335324 2064 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2065 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2066 sjdl2.Prepend("# Generated merging jdl \
2067 \n# $1 = full alien path to output directory to be merged \
2068 \n# $2 = merging stage \
03335324 2069 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2070 }
0f389141 2071 index = sjdl1.Index("JDLVariables");
2072 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 2073 index = sjdl2.Index("JDLVariables");
2074 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 2075 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2076 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
2077 index = sjdl2.Index("Split =");
2078 if (index>=0) {
2079 index1 = sjdl2.Index("\n", index);
2080 sjdl2.Remove(index, index1-index+1);
2081 }
2082 index = sjdl2.Index("SplitMaxInputFileNumber");
2083 if (index>=0) {
2084 index1 = sjdl2.Index("\n", index);
2085 sjdl2.Remove(index, index1-index+1);
2086 }
2087 index = sjdl2.Index("InputDataCollection");
2088 if (index>=0) {
2089 index1 = sjdl2.Index(";", index);
2090 sjdl2.Remove(index, index1-index+1);
2091 }
2092 index = sjdl2.Index("InputDataListFormat");
2093 if (index>=0) {
2094 index1 = sjdl2.Index("\n", index);
2095 sjdl2.Remove(index, index1-index+1);
2096 }
2097 index = sjdl2.Index("InputDataList");
2098 if (index>=0) {
2099 index1 = sjdl2.Index("\n", index);
2100 sjdl2.Remove(index, index1-index+1);
2101 }
f47d5cb4 2102 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2103 // Write jdl to file
a8739e8a 2104 ofstream out;
d2a409b2 2105 out.open(fJDLName.Data(), ios::out);
a8739e8a 2106 if (out.bad()) {
d3b18c4c 2107 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2108 return kFALSE;
2109 }
2110 out << sjdl << endl;
b3e07543 2111 out.close();
0f389141 2112 TString mergeJDLName = fExecutable;
2113 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2114 if (fMergeViaJDL) {
2115 ofstream out1;
2116 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2117 if (out1.bad()) {
d3b18c4c 2118 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2119 return kFALSE;
2120 }
2121 out1 << sjdl1 << endl;
b3e07543 2122 out1.close();
2123 ofstream out2;
2124 TString finalJDL = mergeJDLName;
2125 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2126 out2.open(finalJDL.Data(), ios::out);
2127 if (out2.bad()) {
2128 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2129 return kFALSE;
2130 }
2131 out2 << sjdl2 << endl;
2132 out2.close();
0f389141 2133 }
a8739e8a 2134
2135 // Copy jdl to grid workspace
2136 if (!copy) {
d3b18c4c 2137 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2138 } else {
d2a409b2 2139 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2140 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2141 TString finalJDL = mergeJDLName;
2142 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2143 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2144 if (fProductionMode) {
b5fe9cba 2145 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2146 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2147 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2148 }
d2a409b2 2149 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2150 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2151 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2152 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2153// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2154 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2155 Fatal("","Terminating");
0f389141 2156 if (fMergeViaJDL) {
b3e07543 2157 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2158// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2159// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2160 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2161 Fatal("","Terminating");
2162 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2163 Fatal("","Terminating");
0f389141 2164 }
a8739e8a 2165 }
2166 return kTRUE;
2167}
2168
c57f56b7 2169//______________________________________________________________________________
5513444a 2170Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2171{
2172// Returns true if file exists.
5513444a 2173 if (!gGrid) return kFALSE;
2ea55496 2174 TString slfn = lfn;
2175 slfn.ReplaceAll("alien://","");
2176 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2177 if (!res) return kFALSE;
2178 TMap *map = dynamic_cast<TMap*>(res->At(0));
2179 if (!map) {
2180 delete res;
2181 return kFALSE;
2182 }
2183 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2184 if (!objs || !objs->GetString().Length()) {
2185 delete res;
2186 return kFALSE;
2187 }
2188 delete res;
2189 return kTRUE;
2190}
2191
923e2ca5 2192//______________________________________________________________________________
2193Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2194{
2195// Returns true if directory exists. Can be also a path.
2196 if (!gGrid) return kFALSE;
2197 // Check if dirname is a path
2198 TString dirstripped = dirname;
2199 dirstripped = dirstripped.Strip();
2200 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2201 TString dir = gSystem->BaseName(dirstripped);
2202 dir += "/";
2203 TString path = gSystem->DirName(dirstripped);
2204 TGridResult *res = gGrid->Ls(path, "-F");
2205 if (!res) return kFALSE;
2206 TIter next(res);
2207 TMap *map;
2208 TObject *obj;
2209 while ((map=dynamic_cast<TMap*>(next()))) {
2210 obj = map->GetValue("name");
2211 if (!obj) break;
2212 if (dir == obj->GetName()) {
2213 delete res;
2214 return kTRUE;
2215 }
2216 }
2217 delete res;
2218 return kFALSE;
2219}
2220
c57f56b7 2221//______________________________________________________________________________
a2f5fc01 2222void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2223{
2224// Check input data type.
a2f5fc01 2225 isCollection = kFALSE;
2226 isXml = kFALSE;
2227 useTags = kFALSE;
c57f56b7 2228 if (!gGrid) {
2229 Error("CheckDataType", "No connection to grid");
2230 return;
2231 }
a2f5fc01 2232 isCollection = IsCollection(lfn);
c57f56b7 2233 TString msg = "\n##### file: ";
2234 msg += lfn;
a2f5fc01 2235 if (isCollection) {
c57f56b7 2236 msg += " type: raw_collection;";
2237 // special treatment for collections
a2f5fc01 2238 isXml = kFALSE;
c57f56b7 2239 // check for tag files in the collection
2240 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2241 if (!res) {
2242 msg += " using_tags: No (unknown)";
23329835 2243 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2244 return;
2245 }
2246 const char* typeStr = res->GetKey(0, "origLFN");
2247 if (!typeStr || !strlen(typeStr)) {
2248 msg += " using_tags: No (unknown)";
23329835 2249 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2250 return;
2251 }
2252 TString file = typeStr;
a2f5fc01 2253 useTags = file.Contains(".tag");
2254 if (useTags) msg += " using_tags: Yes";
c57f56b7 2255 else msg += " using_tags: No";
23329835 2256 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2257 return;
2258 }
2259 TString slfn(lfn);
2260 slfn.ToLower();
a2f5fc01 2261 isXml = slfn.Contains(".xml");
2262 if (isXml) {
c57f56b7 2263 // Open xml collection and check if there are tag files inside
2264 msg += " type: xml_collection;";
2265 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2266 if (!coll) {
2267 msg += " using_tags: No (unknown)";
23329835 2268 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2269 return;
2270 }
2271 TMap *map = coll->Next();
2272 if (!map) {
2273 msg += " using_tags: No (unknown)";
23329835 2274 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2275 return;
2276 }
2277 map = (TMap*)map->GetValue("");
2278 TString file;
2279 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2280 useTags = file.Contains(".tag");
c57f56b7 2281 delete coll;
a2f5fc01 2282 if (useTags) msg += " using_tags: Yes";
c57f56b7 2283 else msg += " using_tags: No";
23329835 2284 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2285 return;
2286 }
a2f5fc01 2287 useTags = slfn.Contains(".tag");
c57f56b7 2288 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2289 else msg += " type: unknown file;";
a2f5fc01 2290 if (useTags) msg += " using_tags: Yes";
c57f56b7 2291 else msg += " using_tags: No";
23329835 2292 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2293}
2294
4e5c5506 2295//______________________________________________________________________________
2296void AliAnalysisAlien::EnablePackage(const char *package)
2297{
2298// Enables a par file supposed to exist in the current directory.
2299 TString pkg(package);
2300 pkg.ReplaceAll(".par", "");
2301 pkg += ".par";
2302 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2303 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2304 return;
2305 }
2306 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2307 Info("EnablePackage", "AliEn plugin will use .par packages");
2308 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2309 if (!fPackages) {
2310 fPackages = new TObjArray();
2311 fPackages->SetOwner();
2312 }
2313 fPackages->Add(new TObjString(pkg));
2314}
2315
3bdcb562 2316//______________________________________________________________________________
2317TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2318{
2319// Make a tree from files having the location specified in fFileForTestMode.
2320// Inspired from JF's CreateESDChain.
2321 if (fFileForTestMode.IsNull()) {
2322 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2323 return NULL;
2324 }
2325 if (gSystem->AccessPathName(fFileForTestMode)) {
2326 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2327 return NULL;
2328 }
2329 // Open the file
2330 ifstream in;
2331 in.open(fFileForTestMode);
2332 Int_t count = 0;
2333 // Read the input list of files and add them to the chain
b505f1f8 2334 TString line;
d6c60197 2335 TString streeName(treeName);
2336 if (IsUseMCchain()) streeName = "TE";
2337 TChain *chain = new TChain(streeName);
06f67114 2338 TList *friends = new TList();
2339 TChain *cfriend = 0;
2340 if (!fFriendChainName.IsNull()) {
2341 TObjArray *list = fFriendChainName.Tokenize(" ");
2342 TIter next(list);
2343 TObjString *str;
2344 while((str=(TObjString*)next())) {
2345 cfriend = new TChain(streeName, str->GetName());
2346 friends->Add(cfriend);
2347 chain->AddFriend(cfriend);
2348 }
2349 delete list;
2350 }
2351 TString bpath;
2352 TIter nextfriend(friends);
b505f1f8 2353 while (in.good())
2354 {
3bdcb562 2355 in >> line;
ffbe06b9 2356 if (line.IsNull() || line.BeginsWith("#")) continue;
3bdcb562 2357 if (count++ == fNtestFiles) break;
2358 TString esdFile(line);
2359 TFile *file = TFile::Open(esdFile);
b505f1f8 2360 if (file && !file->IsZombie()) {
2361 chain->Add(esdFile);
3bdcb562 2362 file->Close();
b505f1f8 2363 if (!fFriendChainName.IsNull()) {
4b5d9e21 2364 if (esdFile.Index("#") > -1)
2365 esdFile.Remove(esdFile.Index("#"));
06f67114 2366 bpath = gSystem->DirName(esdFile);
2367 bpath += "/";
2368 TString fileFriend;
2369 nextfriend.Reset();
2370 while ((cfriend=(TChain*)nextfriend())) {
2371 fileFriend = bpath;
2372 fileFriend += cfriend->GetTitle();
2373 file = TFile::Open(fileFriend);
2374 if (file && !file->IsZombie()) {
2375 file->Close();
2376 cfriend->Add(fileFriend);
2377 } else {
2378 Fatal("GetChainForTestMode", "Cannot open friend file: %s", fileFriend.Data());
2379 return 0;
2380 }
2381 }
4b5d9e21 2382 }
3bdcb562 2383 } else {
2384 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2385 }
b505f1f8 2386 }
2387 in.close();
2388 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2389 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2390 delete chain;
06f67114 2391 friends->Delete();
2392 delete friends;
3bdcb562 2393 return NULL;
b505f1f8 2394 }
b505f1f8 2395 return chain;
3bdcb562 2396}
2397
d2a409b2 2398//______________________________________________________________________________
2399const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2400{
2401// Get job status for all jobs with jobid>jobidstart.
2402 static char mstatus[20];
2403 mstatus[0] = '\0';
2404 nrunning = 0;
2405 nwaiting = 0;
2406 nerror = 0;
2407 ndone = 0;
2408 TGridJobStatusList *list = gGrid->Ps("");
2409 if (!list) return mstatus;
2410 Int_t nentries = list->GetSize();
2411 TGridJobStatus *status;
2412 Int_t pid;
2413 for (Int_t ijob=0; ijob<nentries; ijob++) {
2414 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2415 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2416 if (pid<jobidstart) continue;
2417 if (pid == lastid) {
8ddad121 2418 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2419 }
2420 switch (status->GetStatus()) {
2421 case TGridJobStatus::kWAITING:
2422 nwaiting++; break;
2423 case TGridJobStatus::kRUNNING:
2424 nrunning++; break;
2425 case TGridJobStatus::kABORTED:
2426 case TGridJobStatus::kFAIL:
2427 case TGridJobStatus::kUNKNOWN:
2428 nerror++; break;
2429 case TGridJobStatus::kDONE:
2430 ndone++;
2431 }
2432 }
2433 list->Delete();
2434 delete list;
2435 return mstatus;
2436}
2437
c57f56b7 2438//______________________________________________________________________________
2439Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2440{
2441// Returns true if file is a collection. Functionality duplicated from
2442// TAlien::Type() because we don't want to directly depend on TAlien.
2443 if (!gGrid) {
2444 Error("IsCollection", "No connection to grid");
2445 return kFALSE;
2446 }
2447 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2448 if (!res) return kFALSE;
2449 const char* typeStr = res->GetKey(0, "type");
2450 if (!typeStr || !strlen(typeStr)) return kFALSE;
2451 if (!strcmp(typeStr, "collection")) return kTRUE;
2452 delete res;
2453 return kFALSE;
2454}
2455
fe2d7fc2 2456//______________________________________________________________________________
2457Bool_t AliAnalysisAlien::IsSingleOutput() const
2458{
2459// Check if single-ouput option is on.
2460 return (!fOutputSingle.IsNull());
2461}
715da79c 2462
2463//______________________________________________________________________________
2464Long64_t AliAnalysisAlien::RunMacroAndExtractLibs(const char* macro, const char *args, TString &libs)
2465{
2466// Tries to run the specified macro and return the libraries that it loads.
2467 TString expname;
2468 if (strlen(macro)) expname = gSystem->ExpandPathName(macro);
2469 if (expname.IsNull() || gSystem->AccessPathName(expname)) {
2470 ::Error("RunMacroAndExtractLibs","Cannot find macro %s in current directory", macro);
2471 return -1;
2472 }
2473 TString oldlibs = gSystem->GetLibraries();
2474 TMacro m(expname);
2475 Int_t error = 0;
2476 Long64_t retval = m.Exec(args, &error);
2477 if (error != TInterpreter::kNoError)
2478 {
2479 ::Error("RunMacroAndExtractLibs", "Macro interpretation %s failed", macro);
2480 return -1;
2481 }
2482 libs = gSystem->GetLibraries();
2483 libs.ReplaceAll(oldlibs, "");
2484 libs.Strip(TString::kLeading);
2485 TObjArray *libTokens = libs.Tokenize(" ");
2486 libs = "";
2487 for (Int_t i=0; i<libTokens->GetEntries(); i++) {
2488 if (!libs.IsNull()) libs += " ";
2489 libs += gSystem->BaseName(libTokens->At(i)->GetName());
2490 }
2491 delete libTokens;
2492 return retval;
2493}
2494
16a4353c 2495//______________________________________________________________________________
2496void AliAnalysisAlien::Print(Option_t *) const
2497{
2498// Print current plugin settings.
84fcd93f 2499 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2500 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2501 if (mgr && mgr->IsProofMode()) {
2502 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2503 if (TestBit(AliAnalysisGrid::kTest))
2504 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2505 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2506 if (!fProofDataSet.IsNull())
2507 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2508 if (fProofReset==1)
2509 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2510 if (fProofReset>1)
2511 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
874de7ea 2512 if (!fROOTVersion.IsNull())
2513 printf("= ROOT version requested________________________ %s\n", fROOTVersion.Data());
d3b18c4c 2514 else
2515 printf("= ROOT version requested________________________ default\n");
2516 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2517 if (!fAliRootMode.IsNull())
2518 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2519 if (fNproofWorkers)
2520 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2521 if (fNproofWorkersPerSlave)
2522 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2523 if (TestSpecialBit(kClearPackages))
2524 printf("= ClearPackages requested...\n");
2525 if (fIncludePath.Data())
2526 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2527 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2528 if (fPackages && fPackages->GetEntries()) {
2529 TIter next(fPackages);
2530 TObject *obj;
2531 TString list;
2532 while ((obj=next())) list += obj->GetName();
2533 printf("= Par files to be used: ________________________ %s\n", list.Data());
2534 }
2535 if (TestSpecialBit(kProofConnectGrid))
2536 printf("= Requested PROOF connection to grid\n");
2537 return;
2538 }
e1c22e21 2539 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2540 if (fOverwriteMode) {
2541 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2542 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2543 }
348be253 2544 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
780ec36d 2545 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO:Print");
84fcd93f 2546 printf("= Production mode:______________________________ %d\n", fProductionMode);
2547 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2548 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2549 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2550 if (fUser.Length())
84fcd93f 2551 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2552 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2553 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
780ec36d 2554 TString basedatadir = fGridDataDir;
2555 TString pattern = fDataPattern;
2556 pattern.Strip();
2557 Int_t ind = pattern.Index(" ");
2558 if (ind>=0) {
2559 basedatadir += "/%run%/";
2560 basedatadir += pattern(0, ind);
2561 pattern = pattern(ind+1, pattern.Length());
2562 }
2563 printf("= Data base directory path requested: __________ %s\n", basedatadir.Data());
2564 printf("= Data search pattern: _________________________ %s\n", pattern.Data());
84fcd93f 2565 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2566 if (fRunNumbers.Length())
84fcd93f 2567 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2568 if (fRunRange[0])
74013535 2569 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2570 if (!fRunRange[0] && !fRunNumbers.Length()) {
2571 TIter next(fInputFiles);
2572 TObject *obj;
2573 TString list;
2574 while ((obj=next())) list += obj->GetName();
84fcd93f 2575 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2576 }
2577 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2578 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2579 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2580 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2581 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2582 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2583 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2584 printf("=====================================================================\n");
2585 printf("= Job price: ___________________________________ %d\n", fPrice);
2586 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2587 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2588 if (fMaxInitFailed>0)
84fcd93f 2589 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2590 if (fMasterResubmitThreshold>0)
84fcd93f 2591 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2592 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2593 if (fNrunsPerMaster>0)
84fcd93f 2594 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2595 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2596 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2597 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2598 if (fArguments.Length())
84fcd93f 2599 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2600 if (fExecutableArgs.Length())
2601 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2602 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2603 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2604 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2605 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2606 if (fDatasetName)
84fcd93f 2607 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2608 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2609 if (fIncludePath.Data())
84fcd93f 2610 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2611 if (fCloseSE.Length())
84fcd93f 2612 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2613 if (fFriendChainName.Length())
84fcd93f 2614 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2615 if (fPackages && fPackages->GetEntries()) {
16a4353c 2616 TIter next(fPackages);
2617 TObject *obj;
2618 TString list;
2619 while ((obj=next())) list += obj->GetName();
84fcd93f 2620 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2621 }
2622}
2623
c57f56b7 2624//______________________________________________________________________________
2625void AliAnalysisAlien::SetDefaults()
2626{
2627// Set default values for everything. What cannot be filled will be left empty.
2628 if (fGridJDL) delete fGridJDL;
2629 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2630 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2631 fPrice = 1;
2632 fTTL = 30000;
2633 fSplitMaxInputFileNumber = 100;
2634 fMaxInitFailed = 0;
2635 fMasterResubmitThreshold = 0;
bb885a9e 2636 fNtestFiles = 10;
149d288c 2637 fNreplicas = 2;
a8739e8a 2638 fRunRange[0] = 0;
2639 fRunRange[1] = 0;
8e3bd06a 2640 fRunPrefix = "%d";
319593fb 2641 fNrunsPerMaster = 1;
16a4353c 2642 fMaxMergeFiles = 100;
c57f56b7 2643 fRunNumbers = "";
2644 fExecutable = "analysis.sh";
5aa94dc3 2645 fExecutableCommand = "root -b -q -x";
c57f56b7 2646 fArguments = "";
631c0b05 2647 fExecutableArgs = "";
c57f56b7 2648 fAnalysisMacro = "myAnalysis.C";
2649 fAnalysisSource = "";
2650 fAdditionalLibs = "";
2651 fSplitMode = "se";
2652 fAPIVersion = "";
2653 fROOTVersion = "";
2654 fAliROOTVersion = "";
2655 fUser = ""; // Your alien user name
2656 fGridWorkingDir = "";
2657 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2658 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2659 fFriendChainName = "";
c57f56b7 2660 fGridOutputDir = "output";
b83f84f0 2661 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2662 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2663 fInputFormat = "xml-single";
2664 fJDLName = "analysis.jdl";
c6cb3634 2665 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2666 fMergeExcludes = "";
0f389141 2667 fMergeViaJDL = 0;
348be253 2668 SetUseCopy(kTRUE);
2669 SetCheckCopy(kTRUE);
149d288c 2670 SetDefaultOutputs(kTRUE);
e1c22e21 2671 fOverwriteMode = 1;
c57f56b7 2672}
2673
a41b2882 2674//______________________________________________________________________________
2675void AliAnalysisAlien::SetFriendChainName(const char *name, const char *libnames)
2676{
2677 // Set file name for the chain of friends and optionally additional libs to be loaded.
2678 // Libs should be separated by blancs.
2679 fFriendChainName = name;
06f67114 2680 fFriendChainName.ReplaceAll(",", " ");
2681 fFriendChainName.Strip();
2682 fFriendChainName.ReplaceAll(" ", " ");
2683
a41b2882 2684 fFriendLibs = libnames;
1a3f255c 2685 if (fFriendLibs.Length()) {
2686 if(!fFriendLibs.Contains(".so"))
2687 Fatal("SetFriendChainName()", "You should provide explicit library names (with extension)");
2688 fFriendLibs.ReplaceAll(",", " ");
2689 fFriendLibs.Strip();
2690 fFriendLibs.ReplaceAll(" ", " ");
a41b2882 2691 }
2692}
2693
874de7ea 2694//______________________________________________________________________________
2695void AliAnalysisAlien::SetRootVersionForProof(const char *version)
2696{
2697// Obsolete method. Use SetROOTVersion instead
2698 Warning("SetRootVersionForProof", "Obsolete. Use SetROOTVersion instead");
95e16f01 2699 if (fROOTVersion.IsNull()) SetROOTVersion(version);
2700 else Error("SetRootVersionForProof", "ROOT version already set to %s", fROOTVersion.Data());
874de7ea 2701}
2702
0f389141 2703//______________________________________________________________________________
b3e07543 2704Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2705{
b3e07543 2706// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2707 // First check if the result is already in the output directory.
2708 if (FileExists(Form("%s/%s",aliendir,filename))) {
2709 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2710 return kFALSE;
2711 }
b3e07543 2712 // Now check the last stage done.
2713 Int_t stage = 0;
2714 while (1) {
2715 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2716 stage++;
2717 }
2718 // Next stage of merging
2719 stage++;
2720 TString pattern = "*root_archive.zip";
2721 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2722 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2723 if (res) delete res;
2724 // Write standard output to file
2725 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2726 // Count the number of files inside
2727 ifstream ifile;
2728 ifile.open(Form("Stage_%d.xml",stage));
2729 if (!ifile.good()) {
2730 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2731 return kFALSE;
b3e07543 2732 }
2733 TString line;
2734 Int_t nfiles = 0;
2735 while (!ifile.eof()) {
2736 ifile >> line;
2737 if (line.Contains("/event")) nfiles++;
7c2cd90a 2738 }
b3e07543 2739 ifile.close();
2740 if (!nfiles) {
2741 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2742 return kFALSE;
2743 } else {
2744 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2745 }
b3e07543 2746 // Copy the file in the output directory
2747 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2748// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2749 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2750 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2751 // Check if this is the last stage to be done.
2752 Bool_t laststage = (nfiles<nperchunk);
2753 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
0ad3ea1f 2754 Int_t jobId = 0;
b3e07543 2755 if (laststage) {
2756 printf("### Submiting final merging stage %d\n", stage);
2757 TString finalJDL = jdl;
2758 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2759 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
0ad3ea1f 2760 jobId = SubmitSingleJob(query);
b3e07543 2761 } else {
2762 printf("### Submiting merging stage %d\n", stage);
03335324 2763 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
0ad3ea1f 2764 jobId = SubmitSingleJob(query);
7c2cd90a 2765 }
0ad3ea1f 2766 if (!jobId) return kFALSE;
2767
2768 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
2769 fGridJobIDs.Append(Form("%d", jobId));
2770 if (!fGridStages.IsNull()) fGridStages.Append(" ");
2771 fGridStages.Append(Form("%s_merge_stage%d",
2772 laststage ? "final" : "partial", stage));
2773
b3e07543 2774 return kTRUE;
2775}
7c2cd90a 2776
105631b5 2777//______________________________________________________________________________
2778AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2779{
2780// Loat the analysis manager from a file.
2781 TFile *file = TFile::Open(fname);
2782 if (!file) {
2783 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2784 return 0;
2785 }
2786 TIter nextkey(file->GetListOfKeys());
2787 AliAnalysisManager *mgr = 0;
2788 TKey *key;
2789 while ((key=(TKey*)nextkey())) {
2790 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2791 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2792 }
2793 if (!mgr)
2794 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2795 return mgr;
2796}
2797
7c2cd90a 2798//______________________________________________________________________________
2799Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2800{
2801// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2802 if (!gGrid) return 0;
2803 printf("=> %s ------> ",query);
2804 TGridResult *res = gGrid->Command(query);
2805 if (!res) return 0;
2806 TString jobId = res->GetKey(0,"jobId");
2807 delete res;
2808 if (jobId.IsNull()) {
2809 printf("submission failed. Reason:\n");
2810 gGrid->Stdout();
2811 gGrid->Stderr();
2812 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2813 return 0;
2814 }
0ad3ea1f 2815 Int_t ijobId = jobId.Atoi();
2816 printf(" Job id: '%s' (%d)\n", jobId.Data(), ijobId);
2817 return ijobId;
7c2cd90a 2818}
2819
ec5acdce 2820//______________________________________________________________________________
2821Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2822{
2823// Merges a collection of output files using concatenation.
d66aa2a9 2824 TString scoll(collection);
2825 if (!scoll.Contains(".xml")) return kFALSE;
ec5acdce 2826 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2827 if (!coll) {
2828 ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2829 return kFALSE;
2830 }
2831 // Iterate grid collection
2832 TString outtmp;
2833 Bool_t merged = kFALSE;
2834 Int_t ifile = 0;
2835 while (coll->Next()) {
2836 TString fname = gSystem->DirName(coll->GetTURL());
2837 fname += "/";
2838 fname += output;
2839 outtmp = Form("%d_%s", ifile, output);
2840 if (!TFile::Cp(fname, outtmp)) {
2841 ::Error("MergeInfo", "Could not copy %s", fname.Data());
2842 continue;
2843 }
2844 ifile++;
2845 if (ifile<2) {
2846 gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2847 continue;
2848 }
2849 gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2850 gSystem->Exec("cp tempmerged lastmerged");
2851 }
2852 if (ifile) {
2853 gSystem->Exec(Form("cp lastmerged %s", output));
2854 gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2855 merged = kTRUE;
2856 }
2857 return merged;
2858}
2859
7c2cd90a 2860//______________________________________________________________________________
b3e07543 2861Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2862{
b3e07543 2863// Merge given output files from basedir. Basedir can be an alien output directory
2864// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2865// files in a group (ignored for xml input). Merging can be done in stages:
2866// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2867// stage=1 : works with an xml of all root_archive.zip in the output directory
2868// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2869 TString outputFile = output;
0f389141 2870 TString command;
a2f5fc01 2871 TString outputChunk;
2872 TString previousChunk = "";
b3e07543 2873 TObjArray *listoffiles = new TObjArray();
2874// listoffiles->SetOwner();
a2f5fc01 2875 Int_t countChunk = 0;
2876 Int_t countZero = nmaxmerge;
0f389141 2877 Bool_t merged = kTRUE;
d66aa2a9 2878 Bool_t isGrid = kTRUE;
a2f5fc01 2879 Int_t index = outputFile.Index("@");
2880 if (index > 0) outputFile.Remove(index);
7c2cd90a 2881 TString inputFile = outputFile;
b3e07543 2882 TString sbasedir = basedir;
2883 if (sbasedir.Contains(".xml")) {
2884 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2885 nmaxmerge = 9999999;
2886 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2887 if (!coll) {
2888 ::Error("MergeOutput", "Input XML collection empty.");
2889 return kFALSE;
2890 }
2891 // Iterate grid collection
2892 while (coll->Next()) {
2893 TString fname = gSystem->DirName(coll->GetTURL());
2894 fname += "/";
2895 fname += inputFile;
2896 listoffiles->Add(new TNamed(fname.Data(),""));
2897 }
ffbe06b9 2898 } else if (sbasedir.Contains(".txt")) {
d66aa2a9 2899 // The file having the .txt extension is expected to contain a list of
2900 // folders where the output files will be looked. For alien folders,
2901 // the full folder LFN is expected (starting with alien://)
ffbe06b9 2902 // Assume lfn's on each line
2903 TString line;
2904 ifstream in;
2905 in.open(sbasedir);
d66aa2a9 2906 if (in.fail()) {
2907 ::Error("MergeOutput", "File %s cannot be opened. Merging stopped." ,sbasedir.Data());
2908 return kTRUE;
2909 }
ffbe06b9 2910 Int_t nfiles = 0;
2911 while (in.good()) {
2912 in >> line;
2913 if (line.IsNull() || line.BeginsWith("#")) continue;
d66aa2a9 2914 line.Strip();
2915 if (!line.Contains("alien:")) isGrid = kFALSE;
2916 line += "/";
2917 line += outputFile;
ffbe06b9 2918 nfiles++;
2919 listoffiles->Add(new TNamed(line.Data(),""));
2920 }
d66aa2a9 2921 in.close();
ffbe06b9 2922 if (!nfiles) {
2923 ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data());
2924 delete listoffiles;
2925 return kFALSE;
2926 }
b3e07543 2927 } else {
2928 command = Form("find %s/ *%s", basedir, inputFile.Data());
2929 printf("command: %s\n", command.Data());
2930 TGridResult *res = gGrid->Command(command);
2931 if (!res) {
2932 ::Error("MergeOutput","No result for the find command\n");
2933 delete listoffiles;
2934 return kFALSE;
2935 }
2936 TIter nextmap(res);
2937 TMap *map = 0;
2938 while ((map=(TMap*)nextmap())) {
2939 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2940 if (!objs || !objs->GetString().Length()) {
2941 // Nothing found - skip this output
2942 delete res;
2943 delete listoffiles;
2944 return kFALSE;
2945 }
2946 listoffiles->Add(new TNamed(objs->GetName(),""));
2947 }
2948 delete res;
2949 }
2950 if (!listoffiles->GetEntries()) {
7c2cd90a 2951 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2952 delete listoffiles;
0f389141 2953 return kFALSE;
2954 }
2955
2956 TFileMerger *fm = 0;
b3e07543 2957 TIter next0(listoffiles);
2958 TObjArray *listoffilestmp = new TObjArray();
2959 listoffilestmp->SetOwner();
2960 TObject *nextfile;
2961 TString snextfile;
2962 // Keep only the files at upper level
2963 Int_t countChar = 0;
2964 while ((nextfile=next0())) {
2965 snextfile = nextfile->GetName();
2966 Int_t crtCount = snextfile.CountChar('/');
2967 if (nextfile == listoffiles->First()) countChar = crtCount;
2968 if (crtCount < countChar) countChar = crtCount;
2969 }
2970 next0.Reset();
2971 while ((nextfile=next0())) {
2972 snextfile = nextfile->GetName();
2973 Int_t crtCount = snextfile.CountChar('/');
2974 if (crtCount > countChar) {
2975 delete nextfile;
2976 continue;
2977 }
2978 listoffilestmp->Add(nextfile);
2979 }
2980 delete listoffiles;
2981 listoffiles = listoffilestmp; // Now contains 'good' files
2982 listoffiles->Print();
2983 TIter next(listoffiles);
7c2cd90a 2984 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2985 outputChunk = outputFile;
2986 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2987 // Check for existent temporary merge files
e1c22e21 2988 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2989 // Preserve old merging functionality for stage 0.
2990 if (stage==0) {
2991 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2992 while (1) {
2993 // Skip as many input files as in a chunk
2ea55496 2994 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2995 nextfile = next();
2996 if (!nextfile) {
2ea55496 2997 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2998 delete listoffiles;
2ea55496 2999 return kFALSE;
3000 }
b3e07543 3001 snextfile = nextfile->GetName();
7c2cd90a 3002 }
3003 outputChunk = outputFile;
3004 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
3005 countChunk++;
3006 if (gSystem->AccessPathName(outputChunk)) continue;
3007 // Merged file with chunks up to <countChunk> found
3008 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
3009 previousChunk = outputChunk;
3010 break;
3011 }
3012 }
3013 countZero = nmaxmerge;
3014
b3e07543 3015 while ((nextfile=next())) {
3016 snextfile = nextfile->GetName();
2ea55496 3017 // Loop 'find' results and get next LFN
7c2cd90a 3018 if (countZero == nmaxmerge) {
3019 // First file in chunk - create file merger and add previous chunk if any.
d66aa2a9 3020 fm = new TFileMerger(isGrid);
7c2cd90a 3021 fm->SetFastMethod(kTRUE);
3022 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
3023 outputChunk = outputFile;
3024 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
3025 }
3026 // If last file found, put merged results in the output file
b3e07543 3027 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 3028 // Add file to be merged and decrement chunk counter.
b3e07543 3029 fm->AddFile(snextfile);
7c2cd90a 3030 countZero--;
b3e07543 3031 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 3032 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3033 // Nothing found - skip this output
3034 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 3035 merged = kFALSE;
3036 break;
7c2cd90a 3037 }
3038 fm->OutputFile(outputChunk);
3039 // Merge the outputs, then go to next chunk
3040 if (!fm->Merge()) {
3041 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 3042 merged = kFALSE;
3043 break;
7c2cd90a 3044 } else {
3045 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
3046 gSystem->Unlink(previousChunk);
3047 }
b3e07543 3048 if (nextfile == listoffiles->Last()) break;
7c2cd90a 3049 countChunk++;
3050 countZero = nmaxmerge;
3051 previousChunk = outputChunk;
0f389141 3052 }
0f389141 3053 }
b3e07543 3054 delete listoffiles;
3055 delete fm;
7c2cd90a 3056 return merged;
3057 }
3058 // Merging stage different than 0.
3059 // Move to the begining of the requested chunk.
d66aa2a9 3060 fm = new TFileMerger(isGrid);
7c2cd90a 3061 fm->SetFastMethod(kTRUE);
b3e07543 3062 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
3063 delete listoffiles;
7c2cd90a 3064 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3065 // Nothing found - skip this output
3066 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
3067 delete fm;
3068 return kFALSE;
3069 }
b3e07543 3070 fm->OutputFile(outputFile);
7c2cd90a 3071 // Merge the outputs
3072 if (!fm->Merge()) {
3073 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
3074 delete fm;
3075 return kFALSE;
3076 } else {
b3e07543 3077 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 3078 }
7c2cd90a 3079 delete fm;
3080 return kTRUE;
0f389141 3081}
3082
c57f56b7 3083//______________________________________________________________________________
3084Bool_t AliAnalysisAlien::MergeOutputs()
3085{
3086// Merge analysis outputs existing in the AliEn space.
3087 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
3088 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3089 if (!Connect()) {
3090 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
3091 return kFALSE;
0f389141 3092 }
c9e8f7fd 3093 if (fMergeViaJDL) {
3094 if (!TestBit(AliAnalysisGrid::kMerge)) {
3095 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
3096 return kFALSE;
3097 }
3098 if (fProductionMode) {
3099 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
3100 return kFALSE;
3101 }
0f389141 3102 Info("MergeOutputs", "Submitting merging JDL");
a03be957 3103 if (!SubmitMerging()) return kFALSE;
0f389141 3104 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 3105 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 3106 return kFALSE;
c57f56b7 3107 }
3108 // Get the output path
dc030a24 3109 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 3110 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 3111 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
3112 return kFALSE;
3113 }
3114 if (!fOutputFiles.Length()) {
3115 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3116 return kFALSE;
0f389141 3117 }
3118 // Check if fast read option was requested
e1c22e21 3119 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
3120 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 3121 if (fFastReadOption) {
d3339be3 3122 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
3123 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 3124 gEnv->SetValue("XNet.ConnectTimeout",50);
3125 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 3126 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 3127 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 3128 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 3129 }
e8b839ab 3130 // Make sure we change the temporary directory
3131 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 3132 // Set temporary compilation directory to current one
3133 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 3134 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3135 TIter next(list);
3136 TObjString *str;
a2f5fc01 3137 TString outputFile;
c57f56b7 3138 Bool_t merged = kTRUE;
3139 while((str=(TObjString*)next())) {
a2f5fc01 3140 outputFile = str->GetString();
3141 Int_t index = outputFile.Index("@");
3142 if (index > 0) outputFile.Remove(index);
3143 TString outputChunk = outputFile;
3144 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 3145 // Skip already merged outputs
a2f5fc01 3146 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 3147 if (fOverwriteMode) {
a2f5fc01 3148 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
3149 gSystem->Unlink(outputFile);
3150 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3151 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3152 outputChunk.Data());
3153 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3154 }
3155 } else {
a2f5fc01 3156 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 3157 continue;
3158 }
3159 } else {
a2f5fc01 3160 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3161 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3162 outputChunk.Data());
3163 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3164 }
3165 }
37d21c01 3166 if (fMergeExcludes.Contains(outputFile.Data()) ||
3167 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 3168 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 3169 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 3170 if (!merged) {
3171 Error("MergeOutputs", "Terminate() will NOT be executed");
09d5920f 3172 delete list;
3173 return kFALSE;
ff07ec61 3174 }
3175 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
3176 if (fileOpened) fileOpened->Close();
c57f56b7 3177 }
09d5920f 3178 delete list;
0f389141 3179 return kTRUE;
c57f56b7 3180}
3181
bb885a9e 3182//______________________________________________________________________________
3183void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
3184{
3185// Use the output files connected to output containers from the analysis manager
3186// rather than the files defined by SetOutputFiles
3187 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 3188 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 3189 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
3190}
3191
149d288c 3192//______________________________________________________________________________
3193void AliAnalysisAlien::SetOutputFiles(const char *list)
3194{
3195// Manually set the output files list.
3196// Removes duplicates. Not allowed if default outputs are not disabled.
3197 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3198 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
3199 return;
3200 }
3201 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
3202 fOutputFiles = "";
3203 TString slist = list;
3204 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
3205 TObjArray *arr = slist.Tokenize(" ");
3206 TObjString *os;
3207 TIter next(arr);
3208 TString sout;
3209 while ((os=(TObjString*)next())) {
3210 sout = os->GetString();
3211 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
3212 if (fOutputFiles.Contains(sout)) continue;
3213 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
3214 fOutputFiles += sout;
3215 }
3216 delete arr;
f790bc1b 3217}
149d288c 3218
3219//______________________________________________________________________________
3220void AliAnalysisAlien::SetOutputArchive(const char *list)
3221{
3222// Manually set the output archive list. Free text - you are on your own...
3223// Not allowed if default outputs are not disabled.
3224 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3225 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
3226 return;
3227 }
3228 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
3229 fOutputArchive = list;
3230}
3231
3232//______________________________________________________________________________
3233void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
3234{
3235// Setting a prefered output SE is not allowed anymore.
3236 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
3237}
3238
28ce6c9c 3239//______________________________________________________________________________
3240void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
3241{
3242// Set some PROOF special parameter.
3243 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3244 if (pair) {
3245 TObject *old = pair->Key();
3246 TObject *val = pair->Value();
3247 fProofParam.Remove(old);
3248 delete old;
3249 delete val;
3250 }
3251 fProofParam.Add(new TObjString(pname), new TObjString(value));
3252}
3253
3254//______________________________________________________________________________
3255const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
3256{
3257// Returns a special PROOF parameter.
3258 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3259 if (!pair) return 0;
3260 return pair->Value()->GetName();
3261}
3262
5513444a 3263//______________________________________________________________________________
3264Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 3265{
3266// Start remote grid analysis.
3bdcb562 3267 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3268 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
3269 if (!mgr || !mgr->IsInitialized()) {
3270 Error("StartAnalysis", "You need an initialized analysis manager for this");
3271 return kFALSE;
3272 }
3273 // Are we in PROOF mode ?
3274 if (mgr->IsProofMode()) {
baa34407 3275 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3276 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3277 if (fProofCluster.IsNull()) {
3278 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3279 return kFALSE;
3280 }
3281 if (fProofDataSet.IsNull() && !testMode) {
3282 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3283 return kFALSE;
3284 }
3285 // Set the needed environment
3286 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3287 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3288 if (fProofReset && !testMode) {
3289 if (fProofReset==1) {
3290 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3291 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3292 } else {
3293 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3294 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3295 }
3296 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3297 return kFALSE;
3298 }
baa34407 3299
3300 if (!testMode) {
3301 // Check if there is an old active session
3302 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3303 if (nsessions) {
3304 Error("StartAnalysis","You have to reset your old session first\n");
3305 return kFALSE;
3306 }
53911ca6 3307 }
3bdcb562 3308 // Do we need to change the ROOT version ? The success of this cannot be checked.
874de7ea 3309 if (!fROOTVersion.IsNull() && !testMode) {
cbccef47 3310 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"VO_ALICE@ROOT::%s\");",
874de7ea 3311 fProofCluster.Data(), fROOTVersion.Data()));
3bdcb562 3312 }
3313 // Connect to PROOF and check the status
3314 Long_t proof = 0;
27734f0e 3315 TString sworkers;
3316 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3317 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3318 if (!testMode) {
27734f0e 3319 if (!sworkers.IsNull())
3320 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3321 else
3322 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3323 } else {
3324 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3325 if (!proof) {
3326 Error("StartAnalysis", "Could not start PROOF in test mode");
3327 return kFALSE;
3328 }
3329 }
3330 if (!proof) {
3331 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3332 return kFALSE;
3333 }
27734f0e 3334 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3335 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3336 // Set proof special parameters if any
3337 TIter nextpp(&fProofParam);
3338 TObject *proofparam;
3339 while ((proofparam=nextpp())) {
3340 TString svalue = GetProofParameter(proofparam->GetName());
3341 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3342 }
3bdcb562 3343 // Is dataset existing ?
3344 if (!testMode) {
3345 TString dataset = fProofDataSet;
3346 Int_t index = dataset.Index("#");
3347 if (index>=0) dataset.Remove(index);
830acc4c 3348// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3349// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3350// return kFALSE;
3351// }
3352// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3353 }
3354 // Is ClearPackages() needed ?
3355 if (TestSpecialBit(kClearPackages)) {
3356 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3357 gROOT->ProcessLine("gProof->ClearPackages();");
3358 }
3359 // Is a given aliroot mode requested ?
3360 TList optionsList;
1ac846e9 3361 TString parLibs;
830acc4c 3362 if (!fAliRootMode.IsNull()) {
3bdcb562 3363 TString alirootMode = fAliRootMode;
3364 if (alirootMode == "default") alirootMode = "";
3365 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3366 optionsList.SetOwner();
3367 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3368 // Check the additional libs to be loaded
3369 TString extraLibs;
1ac846e9 3370 Bool_t parMode = kFALSE;
dc1ffbcb 3371 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3372 // Parse the extra libs for .so
3373 if (fAdditionalLibs.Length()) {
a41b2882 3374 TString additionalLibs = fAdditionalLibs;
3375 additionalLibs.Strip();
3376 if (additionalLibs.Length() && fFriendLibs.Length())
3377 additionalLibs += " ";
3378 additionalLibs += fFriendLibs;
3379 TObjArray *list = additionalLibs.Tokenize(" ");
3bdcb562 3380 TIter next(list);
3381 TObjString *str;
1ac846e9 3382 while((str=(TObjString*)next())) {
3383 if (str->GetString().Contains(".so")) {
3384 if (parMode) {
3385 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3386 break;
3387 }
3388 TString stmp = str->GetName();
3389 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3390 stmp.ReplaceAll(".so","");
3391 if (!extraLibs.IsNull()) extraLibs += ":";
3392 extraLibs += stmp;
3393 continue;
3394 }
3395 if (str->GetString().Contains(".par")) {
3396 // The first par file found in the list will not allow any further .so
3397 parMode = kTRUE;
3398 if (!parLibs.IsNull()) parLibs += ":";
3399 parLibs += str->GetName();
3400 continue;
3401 }
3bdcb562 3402 }
3403 if (list) delete list;
3404 }
a41b2882 3405 if (!extraLibs.IsNull()) {
3406 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3407 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3408 }
830acc4c 3409 // Check extra includes
3410 if (!fIncludePath.IsNull()) {
3411 TString includePath = fIncludePath;
3412 includePath.ReplaceAll(" ",":");
dc030a24 3413 includePath.ReplaceAll("$ALICE_ROOT/","");
3414 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3415 includePath.ReplaceAll("-I","");
dc030a24 3416 includePath.Remove(TString::kTrailing, ':');
830acc4c 3417 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3418 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3419 }
27734f0e 3420 // Check if connection to grid is requested
3421 if (TestSpecialBit(kProofConnectGrid))
3422 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3423 // Enable AliRoot par
3424 if (testMode) {
3425 // Enable proof lite package
3426 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3427 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3428 TNamed *obj = (TNamed*)optionsList.At(i);
3429 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3430 }
3431 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3432 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3433 Info("StartAnalysis", "AliRootProofLite enabled");
3434 } else {
3435 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3436 return kFALSE;
3437 }
3438 } else {
dc030a24 3439 if ( ! fAliROOTVersion.IsNull() ) {
3440 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3441 fAliROOTVersion.Data(), &optionsList))) {
3442 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3443 return kFALSE;
3444 }
3445 }
27734f0e 3446 }
1ac846e9 3447 // Enable first par files from fAdditionalLibs
3448 if (!parLibs.IsNull()) {
3449 TObjArray *list = parLibs.Tokenize(":");
3450 TIter next(list);
3451 TObjString *package;
3452 while((package=(TObjString*)next())) {
40501bb7 3453 TString spkg = package->GetName();
3454 spkg.ReplaceAll(".par", "");
3455 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3456 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3457 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3458 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3459 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3460 return kFALSE;
3461 }
3462 } else {
3463 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3464 return kFALSE;
3465 }
3466 }
3467 if (list) delete list;
3468 }
3bdcb562 3469 } else {
3470 if (fAdditionalLibs.Contains(".so") && !testMode) {
3471 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3472 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3473 return kFALSE;
3474 }
3475 }
3476 // Enable par files if requested
3477 if (fPackages && fPackages->GetEntries()) {
3478 TIter next(fPackages);
3479 TObject *package;
3480 while ((package=next())) {
1ac846e9 3481 // Skip packages already enabled
3482 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3483 TString spkg = package->GetName();
3484 spkg.ReplaceAll(".par", "");
3485 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3486 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3487 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3488 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3489 return kFALSE;
3490 }
3491 } else {
3492 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3493 return kFALSE;
3494 }
3495 }
3496 }
3497 // Do we need to load analysis source files ?
3498 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3499 if (fAnalysisSource.Length()) {
3500 TObjArray *list = fAnalysisSource.Tokenize(" ");
3501 TIter next(list);
3502 TObjString *str;
3503 while((str=(TObjString*)next())) {
3504 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3505 }
3bdcb562 3506 if (list) delete list;
3507 }
830acc4c 3508 if (testMode) {
3509 // Register dataset to proof lite.
3510 if (fFileForTestMode.IsNull()) {
3511 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3512 return kFALSE;
3513 }
3514 if (gSystem->AccessPathName(fFileForTestMode)) {
3515 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3516 return kFALSE;
3517 }
3518 TFileCollection *coll = new TFileCollection();
3519 coll->AddFromFile(fFileForTestMode);
8ddad121 3520 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3521 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3522 }
3bdcb562 3523 return kTRUE;
3524 }
c57f56b7 3525
43da816a 3526 // Check if output files have to be taken from the analysis manager
3527 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3528 // Add output files and AOD files
3529 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3530 // Add extra files registered to the analysis manager
f790bc1b 3531 TString extra = GetListOfFiles("ext");
3532 if (!extra.IsNull()) {
149d288c 3533 extra.ReplaceAll(".root", "*.root");
f790bc1b 3534 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3535 fOutputFiles += extra;
84fcd93f 3536 }
149d288c 3537 // Compose the output archive.
b83f84f0 3538 fOutputArchive = "log_archive.zip:std*@disk=1 ";
ec5acdce 3539 if (mgr->IsCollectThroughput())
95e16f01 3540 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",fOutputFiles.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 3541 else
3542 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3543 }
f7b1cbc2 3544// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3545 if (TestBit(AliAnalysisGrid::kOffline)) {
3546 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3547 \n there nor any job run. You can revise the JDL and analysis \
3548 \n macro then run the same in \"submit\" mode.");
3549 } else if (TestBit(AliAnalysisGrid::kTest)) {
3550 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3551 \n dataset.");
3552 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3553 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3554 \n space and job submitted.");
3555 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3556 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3557 if (fMergeViaJDL) CheckInputData();
5513444a 3558 return kTRUE;
c57f56b7 3559 } else {
3560 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3561 }
3562
348be253 3563 Print();
c57f56b7 3564 if (!Connect()) {
3565 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3566 return kFALSE;
16a4353c 3567 }
d3b18c4c 3568 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3569 if (!CheckInputData()) {
3570 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3571 return kFALSE;
c57f56b7 3572 }
d3339be3 3573 if (!CreateDataset(fDataPattern)) {
3574 TString serror;
3575 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3576 if (fRunNumbers.Length()) serror = "run numbers";
3577 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3578 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3579 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3580 return kFALSE;
3581 }
21ca8e59 3582 WriteAnalysisFile();
c57f56b7 3583 WriteAnalysisMacro();
3584 WriteExecutable();
3585 WriteValidationScript();
0f389141 3586 if (fMergeViaJDL) {
3587 WriteMergingMacro();
3588 WriteMergeExecutable();
3589 WriteValidationScript(kTRUE);
3590 }
5513444a 3591 if (!CreateJDL()) return kFALSE;
3592 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3593 if (testMode) {
c57f56b7 3594 // Locally testing the analysis
3595 Info("StartAnalysis", "\n_______________________________________________________________________ \
3596 \n Running analysis script in a daughter shell as on a worker node \
3597 \n_______________________________________________________________________");
149d288c 3598 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3599 TIter next(list);
3600 TObjString *str;
a2f5fc01 3601 TString outputFile;
c57f56b7 3602 while((str=(TObjString*)next())) {
a2f5fc01 3603 outputFile = str->GetString();
3604 Int_t index = outputFile.Index("@");
3605 if (index > 0) outputFile.Remove(index);
3606 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3607 }
3608 delete list;
3609 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3610 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3611// gSystem->Exec("cat stdout");
5513444a 3612 return kFALSE;
c57f56b7 3613 }
5513444a 3614 // Check if submitting is managed by LPM manager
a3e84053 3615 if (fProductionMode) {
7ac61826 3616 //TString prodfile = fJDLName;
3617 //prodfile.ReplaceAll(".jdl", ".prod");
3618 //WriteProductionFile(prodfile);
5513444a 3619 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3620 return kFALSE;
3621 }
a8739e8a 3622 // Submit AliEn job(s)
d2a409b2 3623 gGrid->Cd(fGridOutputDir);
a8739e8a 3624 TGridResult *res;
c57f56b7 3625 TString jobID = "";
0ad3ea1f 3626 fGridJobIDs = "";
3627 fGridStages = "";
d2a409b2 3628 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3629 // Submit a given xml or a set of runs
a8739e8a 3630 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3631 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3632 if (res) {
3633 const char *cjobId = res->GetKey(0,"jobId");
3634 if (!cjobId) {
a03be957 3635 gGrid->Stdout();
3636 gGrid->Stderr();
a8739e8a 3637 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3638 return kFALSE;
a8739e8a 3639 } else {
3640 Info("StartAnalysis", "\n_______________________________________________________________________ \
3641 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3642 \n_______________________________________________________________________",
3643 fJDLName.Data(), cjobId);
3644 jobID = cjobId;
0ad3ea1f 3645 if (jobID.Atoi()) {
3646 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3647 fGridJobIDs.Append(jobID);
3648 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3649 fGridStages.Append("full");
3650 }
a8739e8a 3651 }
3652 delete res;
a03be957 3653 } else {
3654 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3655 return kFALSE;
a8739e8a 3656 }
3657 } else {
d2a409b2 3658 // Submit for a range of enumeration of runs.
a03be957 3659 if (!Submit()) return kFALSE;
0ad3ea1f 3660 jobID = fGridJobIDs;
c57f56b7 3661 }
a8739e8a 3662
d88e4159 3663 if (fDropToShell) {
3664 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3665 \n You may exit at any time and terminate the job later using the option <terminate> \
3666 \n ##################################################################################", jobID.Data());
3667 gSystem->Exec("aliensh");
3668 } else {
3669 Info("StartAnalysis", "\n#### SUBMITTED JOB %s TO ALIEN QUEUE #### \
3670 \n Remember to terminate the job later using the option <terminate> \
3671 \n ##################################################################################", jobID.Data());
3672 }
5513444a 3673 return kTRUE;
c57f56b7 3674}
3675
f790bc1b 3676//______________________________________________________________________________
3677const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3678{
3679// Get a comma-separated list of output files of the requested type.
3680// Type can be (case unsensitive):
3681// aod - list of aod files (std, extensions and filters)
3682// out - list of output files connected to containers (but not aod's or extras)
3683// ext - list of extra files registered to the manager
3684// ter - list of files produced in terminate
3685 static TString files;
3686 files = "";
3687 TString stype = type;
3688 stype.ToLower();
3689 TString aodfiles, extra;
3690 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3691 if (!mgr) {
3692 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3693 return files.Data();
3694 }
3695 if (mgr->GetOutputEventHandler()) {
3696 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3697 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3698 if (!extraaod.IsNull()) {
3699 aodfiles += ",";
3700 aodfiles += extraaod;
3701 }
3702 }
3703 if (stype.Contains("aod")) {
3704 files = aodfiles;
3705 if (stype == "aod") return files.Data();
3706 }
3707 // Add output files that are not in the list of AOD files
3708 TString outputfiles = "";
3709 TIter next(mgr->GetOutputs());
3710 AliAnalysisDataContainer *output;
3711 const char *filename = 0;
3712 while ((output=(AliAnalysisDataContainer*)next())) {
3713 filename = output->GetFileName();
3714 if (!(strcmp(filename, "default"))) continue;
3715 if (outputfiles.Contains(filename)) continue;
3716 if (aodfiles.Contains(filename)) continue;
3717 if (!outputfiles.IsNull()) outputfiles += ",";
3718 outputfiles += filename;
3719 }
3720 if (stype.Contains("out")) {
3721 if (!files.IsNull()) files += ",";
3722 files += outputfiles;
3723 if (stype == "out") return files.Data();
3724 }
3725 // Add extra files registered to the analysis manager
3726 TString sextra;
3727 extra = mgr->GetExtraFiles();
3728 if (!extra.IsNull()) {
3729 extra.Strip();
3730 extra.ReplaceAll(" ", ",");
3731 TObjArray *fextra = extra.Tokenize(",");
3732 TIter nextx(fextra);
3733 TObject *obj;
3734 while ((obj=nextx())) {
3735 if (aodfiles.Contains(obj->GetName())) continue;
3736 if (outputfiles.Contains(obj->GetName())) continue;
3737 if (sextra.Contains(obj->GetName())) continue;
3738 if (!sextra.IsNull()) sextra += ",";
3739 sextra += obj->GetName();
3740 }
3741 delete fextra;
3742 if (stype.Contains("ext")) {
3743 if (!files.IsNull()) files += ",";
3744 files += sextra;
3745 }
3746 }
3747 if (stype == "ext") return files.Data();
3748 TString termfiles;
3749 if (!fTerminateFiles.IsNull()) {
3750 fTerminateFiles.Strip();
3751 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3752 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3753 TIter nextx(fextra);
3754 TObject *obj;
3755 while ((obj=nextx())) {
3756 if (aodfiles.Contains(obj->GetName())) continue;
3757 if (outputfiles.Contains(obj->GetName())) continue;
3758 if (termfiles.Contains(obj->GetName())) continue;
3759 if (sextra.Contains(obj->GetName())) continue;
3760 if (!termfiles.IsNull()) termfiles += ",";
3761 termfiles += obj->GetName();
3762 }
3763 delete fextra;
3764 }
3765 if (stype.Contains("ter")) {
3766 if (!files.IsNull() && !termfiles.IsNull()) {
3767 files += ",";
3768 files += termfiles;
3769 }
3770 }
3771 return files.Data();
3772}
3773
d2a409b2 3774//______________________________________________________________________________
a03be957 3775Bool_t AliAnalysisAlien::Submit()
d2a409b2 3776{
3777// Submit all master jobs.
3778 Int_t nmasterjobs = fInputFiles->GetEntries();
3779 Long_t tshoot = gSystem->Now();
a03be957 3780 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3781 while (fNsubmitted < nmasterjobs) {
3782 Long_t now = gSystem->Now();
3783 if ((now-tshoot)>30000) {
3784 tshoot = now;
a03be957 3785 if (!SubmitNext()) return kFALSE;
d2a409b2 3786 }
3787 }
a03be957 3788 return kTRUE;
d2a409b2 3789}
3790
0f389141 3791//______________________________________________________________________________
a03be957 3792Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3793{
3794// Submit all merging jobs.
dc030a24 3795 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3796 gGrid->Cd(fGridOutputDir);
3797 TString mergeJDLName = fExecutable;
3798 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3799 if (!fInputFiles) {
3800 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3801 return kFALSE;
3802 }
0f389141 3803 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3804 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3805 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3806 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3807 if (fOutputToRunNo) {
3808 // The output directory is the run number
3809 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3810 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3811 } else {
17b4e9a4 3812 if (!fRunNumbers.Length() && !fRunRange[0]) {
3813 // The output directory is the grid outdir
3814 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3815 runOutDir = fGridOutputDir;
3816 } else {
3817 // The output directory is the master number in 3 digits format
3818 printf("### Submitting merging job for master <%03d>\n", i);
3819 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3820 }
7c2cd90a 3821 }
3822 // Check now the number of merging stages.
2b222f7c 3823 TObjArray *list = fOutputFiles.Tokenize(",");
3824 TIter next(list);
3825 TObjString *str;
3826 TString outputFile;
3827 while((str=(TObjString*)next())) {
3828 outputFile = str->GetString();
3829 Int_t index = outputFile.Index("@");
3830 if (index > 0) outputFile.Remove(index);
37d21c01 3831 if (!fMergeExcludes.Contains(outputFile) &&
3832 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3833 }
3834 delete list;
b3e07543 3835 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3836 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3837 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3838 }
a03be957 3839 if (!ntosubmit) return kTRUE;
d88e4159 3840 if (fDropToShell) {
3841 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3842 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3843 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3844 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3845 \n ################################################################################################################");
3846 gSystem->Exec("aliensh");
3847 } else {
3848 Info("StartAnalysis", "\n #### STARTED MERGING JOBS FOR YOU #### \
3849 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL') \
3850 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3851 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3852 \n ################################################################################################################");
3853 }
a03be957 3854 return kTRUE;
0f389141 3855}
3856
d2a409b2 3857//______________________________________________________________________________
a03be957 3858Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3859{
08d5b699 3860// Submit next bunch of master jobs if the queue is free. The first master job is
3861// submitted right away, while the next will not be unless the previous was split.
3862// The plugin will not submit new master jobs if there are more that 500 jobs in
3863// waiting phase.
d2a409b2 3864 static Bool_t iscalled = kFALSE;
3865 static Int_t firstmaster = 0;
3866 static Int_t lastmaster = 0;
3867 static Int_t npermaster = 0;
a03be957 3868 if (iscalled) return kTRUE;
d2a409b2 3869 iscalled = kTRUE;
3870 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3871 Int_t ntosubmit = 0;
3872 TGridResult *res;
3873 TString jobID = "";
3bdcb562 3874 Int_t nmasterjobs = fInputFiles->GetEntries();
3875 if (!fNsubmitted) {
3876 ntosubmit = 1;
3877 if (!IsUseSubmitPolicy()) {
90d50a8c 3878 if (nmasterjobs>5)
3bdcb562 3879 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3880 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3881 ntosubmit = nmasterjobs;
3882 }
3883 } else {
d2a409b2 3884 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3885 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3886 // If last master not split, just return
a03be957 3887 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3888 // No more than 100 waiting jobs
08d5b699 3889 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3890 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3891 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3892 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3893 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3894 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3895 }
d2a409b2 3896 for (Int_t i=0; i<ntosubmit; i++) {
3897 // Submit for a range of enumeration of runs.
a03be957 3898 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3899 TString query;
cd11251e 3900 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3901 runOutDir.ReplaceAll(".xml", "");
3902 if (fOutputToRunNo)
3903 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3904 else
3905 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3906 printf("********* %s\n",query.Data());
d2a409b2 3907 res = gGrid->Command(query);
3908 if (res) {
98ca124f 3909 TString cjobId1 = res->GetKey(0,"jobId");
3910 if (!cjobId1.Length()) {
d2a409b2 3911 iscalled = kFALSE;
a03be957 3912 gGrid->Stdout();
3913 gGrid->Stderr();
3914 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3915 return kFALSE;
d2a409b2 3916 } else {
3917 Info("StartAnalysis", "\n_______________________________________________________________________ \
3918 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3919 \n_______________________________________________________________________",
98ca124f 3920 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
0ad3ea1f 3921 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3922 fGridJobIDs.Append(cjobId1);
3923 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3924 fGridStages.Append("full");
d2a409b2 3925 jobID += cjobId1;
3926 jobID += " ";
98ca124f 3927 lastmaster = cjobId1.Atoi();
d2a409b2 3928 if (!firstmaster) firstmaster = lastmaster;
3929 fNsubmitted++;
3930 }
3931 delete res;
a03be957 3932 } else {
3933 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3934 return kFALSE;
d2a409b2 3935 }
3936 }
3937 iscalled = kFALSE;
a03be957 3938 return kTRUE;
d2a409b2 3939}
3940
c57f56b7 3941//______________________________________________________________________________
3942void AliAnalysisAlien::WriteAnalysisFile()
3943{
f10e8481 3944// Write current analysis manager into the file <analysisFile>
3945 TString analysisFile = fExecutable;
3946 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3947 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3948 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3949 if (!mgr || !mgr->IsInitialized()) {
3950 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3951 return;
3952 }
3953 // Check analysis type
3954 TObject *handler;
3955 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3956 handler = (TObject*)mgr->GetInputEventHandler();
3957 if (handler) {
65eb22f8 3958 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3959 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3960 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3961 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3962 } else {
3963 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3964 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3965 }
c57f56b7 3966 }
3967 TDirectory *cdir = gDirectory;
f10e8481 3968 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3969 if (file) {
e16a394c 3970 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3971 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3972 // Unless merging makes no sense
3973 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3974 mgr->Write();
3975 delete file;
fe2d7fc2 3976 // Enable termination for local jobs
3977 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3978 }
3979 if (cdir) cdir->cd();
f10e8481 3980 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3981 }
3982 Bool_t copy = kTRUE;
d3b18c4c 3983 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3984 if (copy) {
3985 CdWork();
3986 TString workdir = gGrid->GetHomeDirectory();
3987 workdir += fGridWorkingDir;
d3b18c4c 3988 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3989 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3990 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3991 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3992 }
3993}
3994
3995//______________________________________________________________________________
3996void AliAnalysisAlien::WriteAnalysisMacro()
3997{
3998// Write the analysis macro that will steer the analysis in grid mode.
3999 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4000 ofstream out;
4001 out.open(fAnalysisMacro.Data(), ios::out);
4002 if (!out.good()) {
4003 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4004 return;
4005 }
5b273635 4006 Bool_t hasSTEERBase = kFALSE;
4007 Bool_t hasESD = kFALSE;
4008 Bool_t hasAOD = kFALSE;
4009 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4010 Bool_t hasOADB = kFALSE;
5b273635 4011 Bool_t hasANALYSISalice = kFALSE;
4012 Bool_t hasCORRFW = kFALSE;
c57f56b7 4013 TString func = fAnalysisMacro;
4014 TString type = "ESD";
4015 TString comment = "// Analysis using ";
715da79c 4016 if (fMCLoop) {
4017 type = "MCGEN";
4018 comment += "MCGEN";
4019 } else {
4020 if (IsUseMCchain()) {
4021 type = "MC";
4022 comment += "MC";
4023 } else {
4024 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
4025 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
4026 type = "AOD";
4027 comment += "AOD";
4028 }
4029 }
4030 }
0df6ccf2 4031 if (type!="AOD" && fFriendChainName!="") {
4032 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
4033 return;
4034 }
c57f56b7 4035 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
4036 else comment += " data";
4037 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
4038 func.ReplaceAll(".C", "");
4039 out << "void " << func.Data() << "()" << endl;
4040 out << "{" << endl;
4041 out << comment.Data() << endl;
4042 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 4043 out << " TStopwatch timer;" << endl;
4044 out << " timer.Start();" << endl << endl;
13f0c00b 4045 // Change temp directory to current one
fdee57f1 4046 if (!IsLocalTest()) {
4047 out << "// connect to AliEn and make the chain" << endl;
4048 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
4049 }
13f0c00b 4050 out << "// Set temporary merging directory to current one" << endl;
4051 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4052 out << "// Set temporary compilation directory to current one" << endl;
4053 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 4054 // Reset existing include path
4055 out << "// Reset existing include path and add current directory first in the search" << endl;
4056 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 4057 if (!fExecutableCommand.Contains("aliroot")) {
4058 out << "// load base root libraries" << endl;
4059 out << " gSystem->Load(\"libTree\");" << endl;
4060 out << " gSystem->Load(\"libGeom\");" << endl;
4061 out << " gSystem->Load(\"libVMC\");" << endl;
4062 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4063 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4064 }
d5c6455a 4065 if (fAdditionalRootLibs.Length()) {
47a4137d 4066 // in principle libtree /lib geom libvmc etc. can go into this list, too
4067 out << "// Add aditional libraries" << endl;
4068 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4069 TIter next(list);
4070 TObjString *str;
4071 while((str=(TObjString*)next())) {
4072 if (str->GetString().Contains(".so"))
4073 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 4074 }
47a4137d 4075 if (list) delete list;
d5c6455a 4076 }
57377eb5 4077 out << "// Load analysis framework libraries" << endl;
652561d5 4078 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 4079 if (!fPackages) {
f790bc1b 4080 if (!fExecutableCommand.Contains("aliroot")) {
4081 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4082 out << " gSystem->Load(\"libESD\");" << endl;
4083 out << " gSystem->Load(\"libAOD\");" << endl;
4084 }
4e5c5506 4085 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4086 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 4087 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4088 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 4089 } else {
4e5c5506 4090 TIter next(fPackages);
4091 TObject *obj;
57377eb5 4092 TString pkgname;
57377eb5 4093 while ((obj=next())) {
4094 pkgname = obj->GetName();
4478e6f1 4095 if (pkgname == "STEERBase" ||
4096 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4097 if (pkgname == "ESD" ||
4098 pkgname == "ESD.par") hasESD = kTRUE;
4099 if (pkgname == "AOD" ||
4100 pkgname == "AOD.par") hasAOD = kTRUE;
4101 if (pkgname == "ANALYSIS" ||
4102 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4103 if (pkgname == "OADB" ||
4104 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 4105 if (pkgname == "ANALYSISalice" ||
4106 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4107 if (pkgname == "CORRFW" ||
4108 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 4109 }
4110 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 4111 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4112 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 4113 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4114 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 4115 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4116 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 4117 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4118 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4119 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4120 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 4121 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4122 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 4123 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4124 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 4125 out << "// Compile other par packages" << endl;
4126 next.Reset();
fcc9bb6f 4127 while ((obj=next())) {
4128 pkgname = obj->GetName();
4478e6f1 4129 if (pkgname == "STEERBase" ||
4130 pkgname == "STEERBase.par" ||
4131 pkgname == "ESD" ||
4132 pkgname == "ESD.par" ||
4133 pkgname == "AOD" ||
4134 pkgname == "AOD.par" ||
4135 pkgname == "ANALYSIS" ||
4136 pkgname == "ANALYSIS.par" ||
7ea40ac8 4137 pkgname == "OADB" ||
4138 pkgname == "OADB.par" ||
4478e6f1 4139 pkgname == "ANALYSISalice" ||
4140 pkgname == "ANALYSISalice.par" ||
4141 pkgname == "CORRFW" ||
4142 pkgname == "CORRFW.par") continue;
5b273635 4143 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 4144 }
4e5c5506 4145 }
b3e07543 4146 out << "// include path" << endl;
4147 // Get the include path from the interpreter and remove entries pointing to AliRoot
4148 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4149 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4150 out << " TIter nextpath(listpaths);" << endl;
4151 out << " TObjString *pname;" << endl;
4152 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4153 out << " TString current = pname->GetName();" << endl;
4154 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4155 out << " gSystem->AddIncludePath(current);" << endl;
4156 out << " }" << endl;
4157 out << " if (listpaths) delete listpaths;" << endl;
4158 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4159 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4160 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
715da79c 4161 if (fMCLoop && !fGeneratorLibs.IsNull()) {
4162 out << "// MC generator libraries" << endl;
4163 TObjArray *list = fGeneratorLibs.Tokenize(" ");
4164 TIter next(list);
4165 TObjString *str;
4166 while((str=(TObjString*)next())) {
4167 out << " gSystem->Load(\"" << str->GetName() << "\");" << endl;
4168 }
4169 delete list;
4170 }
6da75e0b 4171 if (fAdditionalLibs.Length()) {
4172 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4173 TString additionalLibs = fAdditionalLibs;
4174 additionalLibs.Strip();
4175 if (additionalLibs.Length() && fFriendLibs.Length())
4176 additionalLibs += " ";
4177 additionalLibs += fFriendLibs;
4178 TObjArray *list = additionalLibs.Tokenize(" ");
6da75e0b 4179 TIter next(list);
4180 TObjString *str;
4181 while((str=(TObjString*)next())) {
4182 if (str->GetString().Contains(".so"))
4183 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 4184 if (str->GetString().Contains(".par"))
4185 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 4186 }
715da79c 4187 delete list;
6da75e0b 4188 }
4189 out << endl;
c57f56b7 4190 out << "// analysis source to be compiled at runtime (if any)" << endl;
4191 if (fAnalysisSource.Length()) {
4192 TObjArray *list = fAnalysisSource.Tokenize(" ");
4193 TIter next(list);
4194 TObjString *str;
4195 while((str=(TObjString*)next())) {
4196 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4197 }
4198 if (list) delete list;
4199 }
4200 out << endl;
b3e07543 4201// out << " printf(\"Currently load libraries:\\n\");" << endl;
4202// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 4203 if (fFastReadOption) {
d3339be3 4204 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
4205 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 4206 out << "// fast xrootd reading enabled" << endl;
4207 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4208 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4209 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4210 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4211 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4212 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 4213 }
c57f56b7 4214 out << "// read the analysis manager from file" << endl;
f10e8481 4215 TString analysisFile = fExecutable;
4216 analysisFile.ReplaceAll(".sh", ".root");
105631b5 4217 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4218 << analysisFile << "\");" << endl;
4219 out << " if (!mgr) return;" << endl;
21ca8e59 4220 if (IsLocalTest()) {
4221 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
4222 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 4223 if (fFileForTestMode.IsNull())
4224 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
4225 else
4226 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 4227 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4b5d9e21 4228 if (!fFriendChainName.IsNull())
b9f1efa5 4229 out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\",\"" << fFriendLibs << "\");" << endl;
024dfa52 4230 if (IsUseMCchain())
4231 out << " plugin->SetUseMCchain();" << endl;
715da79c 4232 if (fMCLoop)
4233 out << " plugin->SetMCLoop(kTRUE);" << endl;
21ca8e59 4234 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 4235 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 4236 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 4237 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
4238 } else {
7a366e0b 4239 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 4240 out << " mgr->SetNSysInfo(100);" << endl;
4241 }
21ca8e59 4242 }
c57f56b7 4243 out << " mgr->PrintStatus();" << endl;
52b6a92b 4244 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 4245 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 4246 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 4247 } else {
9a8288ff 4248 if (TestBit(AliAnalysisGrid::kTest))
4249 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4250 else
4251 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 4252 }
4253 }
21ca8e59 4254 if (!IsLocalTest()) {
715da79c 4255 if (fMCLoop) {
4256 out << " mgr->SetCacheSize(0);" << endl;
4257 out << " mgr->EventLoop(" << fNMCevents << ");" << endl;
4258 } else {
4259 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
4260 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
4261 }
21ca8e59 4262 } else {
715da79c 4263 if (fMCLoop) {
4264 out << " mgr->SetCacheSize(0);" << endl;
4265 out << " mgr->EventLoop(" << fNMCevents << ");" << endl;
4266 }
21ca8e59 4267 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 4268 }
f7498086 4269 out << " timer.Stop();" << endl;
4270 out << " timer.Print();" << endl;
c57f56b7 4271 out << "}" << endl << endl;
715da79c 4272 if (!IsLocalTest() && !fMCLoop) {
fcc9bb6f 4273 out <<"//________________________________________________________________________________" << endl;
c57f56b7 4274 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
4275 out << "{" << endl;
4276 out << "// Create a chain using url's from xml file" << endl;
242accb2 4277 out << " TString filename;" << endl;
4278 out << " Int_t run = 0;" << endl;
c85cfc0f 4279 if (IsUseMCchain()) {
4280 out << " TString treename = \"TE\";" << endl;
4281 } else {
780ec36d 4282 if (!fTreeName.IsNull()) {
4283 out << " TString treename = \"" << fTreeName << "\";" << endl;
4284 } else {
4285 out << " TString treename = type;" << endl;
4286 out << " treename.ToLower();" << endl;
4287 out << " treename += \"Tree\";" << endl;
4288 }
c85cfc0f 4289 }
e02fee64 4290 out << " printf(\"***************************************\\n\");" << endl;
4291 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
4292 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 4293 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
4294 out << " if (!coll) {" << endl;
4295 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
4296 out << " return NULL;" << endl;
4297 out << " }" << endl;
242accb2 4298 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 4299 out << " TChain *chain = new TChain(treename);" << endl;
06f67114 4300 if(!fFriendChainName.IsNull()) {
4301 out << " TList *friends = new TList();" << endl;
4302 out << " TIter nextfriend(friends);" << endl;
4303 out << " TChain *cfriend = 0;" << endl;
4304 TObjArray *list = fFriendChainName.Tokenize(" ");
4305 TIter next(list);
4306 TObjString *str;
4307 while((str=(TObjString*)next())) {
4308 out << " cfriend = new TChain(treename, \"" << str->GetName() << "\");" << endl;
4309 out << " friends->Add(cfriend);" << endl;
4310 out << " chain->AddFriend(cfriend);" << endl;
4311 }
4312 delete list;
4313// out << " TChain *chainFriend = new TChain(treename);" << endl;
0df6ccf2 4314 }
c57f56b7 4315 out << " coll->Reset();" << endl;
0df6ccf2 4316 out << " while (coll->Next()) {" << endl;
242accb2 4317 out << " filename = coll->GetTURL("");" << endl;
4318 out << " if (mgr) {" << endl;
4319 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
4320 out << " if (nrun && nrun != run) {" << endl;
4321 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
4322 out << " mgr->SetRunFromPath(nrun);" << endl;
4323 out << " run = nrun;" << endl;
4324 out << " }" << endl;
4325 out << " }" << endl;
4326 out << " chain->Add(filename);" << endl;
06f67114 4327 if(!fFriendChainName.IsNull()) {
4328 out << " TString bpath=coll->GetTURL(\"\");" << endl;
4329 out << " if (bpath.Index(\"#\") > -1) bpath.Remove(bpath.Index(\"#\"));" << endl;
4330 out << " bpath = gSystem->DirName(bpath);" << endl;
4331 out << " bpath += \"/\";" << endl;
4332 out << " TString fileFriend;" << endl;
4333 out << " nextfriend.Reset();" << endl;
4334 out << " while ((cfriend=(TChain*)nextfriend())) {" << endl;
4335 out << " fileFriend = bpath;" << endl;
4336 out << " fileFriend += cfriend->GetTitle();" << endl;
4337 out << " TFile *file = TFile::Open(fileFriend);" << endl;
4338 out << " if (file) {" << endl;
4339 out << " file->Close();" << endl;
4340 out << " cfriend->Add(fileFriend.Data());" << endl;
4341 out << " } else {" << endl;
4342 out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
4343 out << " return 0;" << endl;
4344 out << " }" << endl;
4b5d9e21 4345 out << " }" << endl;
0df6ccf2 4346 }
4347 out << " }" << endl;
c57f56b7 4348 out << " if (!chain->GetNtrees()) {" << endl;
4349 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
4350 out << " return NULL;" << endl;
4351 out << " }" << endl;
4352 out << " return chain;" << endl;
fcc9bb6f 4353 out << "}" << endl << endl;
c57f56b7 4354 }
5b273635 4355 if (hasANALYSISalice) {
4356 out <<"//________________________________________________________________________________" << endl;
4357 out << "Bool_t SetupPar(const char *package) {" << endl;
4358 out << "// Compile the package and set it up." << endl;
4359 out << " TString pkgdir = package;" << endl;
4360 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4361 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4362 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4363 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4364 out << " // Check for BUILD.sh and execute" << endl;
4365 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4366 out << " printf(\"*******************************\\n\");" << endl;
4367 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4368 out << " printf(\"*******************************\\n\");" << endl;
4369 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4370 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4371 out << " gSystem->ChangeDirectory(cdir);" << endl;
4372 out << " return kFALSE;" << endl;
4373 out << " }" << endl;
4374 out << " } else {" << endl;
4375 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4376 out << " gSystem->ChangeDirectory(cdir);" << endl;
4377 out << " return kFALSE;" << endl;
4378 out << " }" << endl;
4379 out << " // Check for SETUP.C and execute" << endl;
4380 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4381 out << " printf(\"*******************************\\n\");" << endl;
4382 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4383 out << " printf(\"*******************************\\n\");" << endl;
4384 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4385 out << " } else {" << endl;
4386 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4387 out << " gSystem->ChangeDirectory(cdir);" << endl;
4388 out << " return kFALSE;" << endl;
4389 out << " }" << endl;
4390 out << " // Restore original workdir" << endl;
4391 out << " gSystem->ChangeDirectory(cdir);" << endl;
4392 out << " return kTRUE;" << endl;
4393 out << "}" << endl;
4394 }
c57f56b7 4395 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4396 }
4397 Bool_t copy = kTRUE;
d3b18c4c 4398 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4399 if (copy) {
4400 CdWork();
4401 TString workdir = gGrid->GetHomeDirectory();
4402 workdir += fGridWorkingDir;
4403 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4404 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4405// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4406 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4407 Form("alien://%s/%s", workdir.Data(),
4408 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4409 }
4410}
4411
0f389141 4412//______________________________________________________________________________
4413void AliAnalysisAlien::WriteMergingMacro()
4414{
4415// Write a macro to merge the outputs per master job.
4416 if (!fMergeViaJDL) return;
4417 if (!fOutputFiles.Length()) {
4418 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4419 return;
4420 }
ec5acdce 4421 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
0f389141 4422 TString mergingMacro = fExecutable;
4423 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4424 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4425 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4426 ofstream out;
4427 out.open(mergingMacro.Data(), ios::out);
4428 if (!out.good()) {
4429 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4430 return;
4431 }
5b273635 4432 Bool_t hasSTEERBase = kFALSE;
4433 Bool_t hasESD = kFALSE;
4434 Bool_t hasAOD = kFALSE;
4435 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4436 Bool_t hasOADB = kFALSE;
5b273635 4437 Bool_t hasANALYSISalice = kFALSE;
4438 Bool_t hasCORRFW = kFALSE;
0f389141 4439 TString func = mergingMacro;
4440 TString comment;
4441 func.ReplaceAll(".C", "");
f47d5cb4 4442 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4443 out << "{" << endl;
4444 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4445 out << " TStopwatch timer;" << endl;
4446 out << " timer.Start();" << endl << endl;
b3e07543 4447 // Reset existing include path
4448 out << "// Reset existing include path and add current directory first in the search" << endl;
4449 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4450 if (!fExecutableCommand.Contains("aliroot")) {
4451 out << "// load base root libraries" << endl;
4452 out << " gSystem->Load(\"libTree\");" << endl;
4453 out << " gSystem->Load(\"libGeom\");" << endl;
4454 out << " gSystem->Load(\"libVMC\");" << endl;
4455 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4456 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4457 }
0f389141 4458 if (fAdditionalRootLibs.Length()) {
4459 // in principle libtree /lib geom libvmc etc. can go into this list, too
4460 out << "// Add aditional libraries" << endl;
4461 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4462 TIter next(list);
4463 TObjString *str;
4464 while((str=(TObjString*)next())) {
4465 if (str->GetString().Contains(".so"))
4466 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4467 }
4468 if (list) delete list;
4469 }
0f389141 4470 out << "// Load analysis framework libraries" << endl;
4471 if (!fPackages) {
7c2cd90a 4472 if (!fExecutableCommand.Contains("aliroot")) {
4473 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4474 out << " gSystem->Load(\"libESD\");" << endl;
4475 out << " gSystem->Load(\"libAOD\");" << endl;
4476 }
0f389141 4477 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4478 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4479 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4480 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4481 } else {
4482 TIter next(fPackages);
4483 TObject *obj;
4484 TString pkgname;
5b273635 4485 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4486 while ((obj=next())) {
4487 pkgname = obj->GetName();
4488 if (pkgname == "STEERBase" ||
4489 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4490 if (pkgname == "ESD" ||
4491 pkgname == "ESD.par") hasESD = kTRUE;
4492 if (pkgname == "AOD" ||
4493 pkgname == "AOD.par") hasAOD = kTRUE;
4494 if (pkgname == "ANALYSIS" ||
4495 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4496 if (pkgname == "OADB" ||
4497 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4498 if (pkgname == "ANALYSISalice" ||
4499 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4500 if (pkgname == "CORRFW" ||
4501 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4502 }
5b273635 4503 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4504 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4505 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4506 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4507 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4508 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4509 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4510 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4511 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4512 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4513 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4514 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4515 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4516 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4517 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4518 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4519 out << "// Compile other par packages" << endl;
4520 next.Reset();
4521 while ((obj=next())) {
4522 pkgname = obj->GetName();
4523 if (pkgname == "STEERBase" ||
4524 pkgname == "STEERBase.par" ||
4525 pkgname == "ESD" ||
4526 pkgname == "ESD.par" ||
4527 pkgname == "AOD" ||
4528 pkgname == "AOD.par" ||
4529 pkgname == "ANALYSIS" ||
4530 pkgname == "ANALYSIS.par" ||
7ea40ac8 4531 pkgname == "OADB" ||
4532 pkgname == "OADB.par" ||
0f389141 4533 pkgname == "ANALYSISalice" ||
4534 pkgname == "ANALYSISalice.par" ||
4535 pkgname == "CORRFW" ||
4536 pkgname == "CORRFW.par") continue;
5b273635 4537 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4538 }
4539 }
b3e07543 4540 out << "// include path" << endl;
4541 // Get the include path from the interpreter and remove entries pointing to AliRoot
4542 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4543 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4544 out << " TIter nextpath(listpaths);" << endl;
4545 out << " TObjString *pname;" << endl;
4546 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4547 out << " TString current = pname->GetName();" << endl;
4548 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4549 out << " gSystem->AddIncludePath(current);" << endl;
4550 out << " }" << endl;
4551 out << " if (listpaths) delete listpaths;" << endl;
4552 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4553 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4554 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4555 if (fAdditionalLibs.Length()) {
4556 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4557 TString additionalLibs = fAdditionalLibs;
4558 additionalLibs.Strip();
4559 if (additionalLibs.Length() && fFriendLibs.Length())
4560 additionalLibs += " ";
4561 additionalLibs += fFriendLibs;
4562 TObjArray *list = additionalLibs.Tokenize(" ");
0f389141 4563 TIter next(list);
4564 TObjString *str;
4565 while((str=(TObjString*)next())) {
4566 if (str->GetString().Contains(".so"))
4567 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4568 }
4569 if (list) delete list;
4570 }
4571 out << endl;
4572 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4573 if (fAnalysisSource.Length()) {
4574 TObjArray *list = fAnalysisSource.Tokenize(" ");
4575 TIter next(list);
4576 TObjString *str;
4577 while((str=(TObjString*)next())) {
4578 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4579 }
4580 if (list) delete list;
4581 }
149d288c 4582 out << endl;
4583
0f389141 4584 if (fFastReadOption) {
4585 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4586 out << "// fast xrootd reading enabled" << endl;
4587 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4588 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4589 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4590 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4591 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4592 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4593 }
4594 // Change temp directory to current one
fdee57f1 4595 out << "// Connect to AliEn" << endl;
4596 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
e8b839ab 4597 out << "// Set temporary merging directory to current one" << endl;
4598 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4599 out << "// Set temporary compilation directory to current one" << endl;
4600 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
7c2cd90a 4601 out << " TString outputDir = dir;" << endl;
f790bc1b 4602 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4603 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4604 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4605 out << " TIter *iter = new TIter(list);" << endl;
4606 out << " TObjString *str;" << endl;
a2f5fc01 4607 out << " TString outputFile;" << endl;
0f389141 4608 out << " Bool_t merged = kTRUE;" << endl;
ec5acdce 4609 TString analysisFile = fExecutable;
4610 analysisFile.ReplaceAll(".sh", ".root");
4611 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4612 << analysisFile << "\");" << endl;
4613 out << " if (!mgr) {" << endl;
4614 out << " printf(\"ERROR: Analysis manager could not be extracted from file \");" << endl;
4615 out << " return;" << endl;
4616 out << " }" << endl;
d66aa2a9 4617 if (IsLocalTest()) {
4618 out << " printf(\"===================================\n\");" << endl;
4619 out << " printf(\"Testing merging...\\n\");" << endl;
4620 out << " printf(\"===================================\n\");" << endl;
4621 }
0f389141 4622 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4623 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4624 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4625 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4626 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4627 out << " // Skip already merged outputs" << endl;
a2f5fc01 4628 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
d66aa2a9 4629 out << " printf(\"Output file <%s> found. Not merging again.\\n\",outputFile.Data());" << endl;
0f389141 4630 out << " continue;" << endl;
4631 out << " }" << endl;
a2f5fc01 4632 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4633 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4634 out << " if (!merged) {" << endl;
a2f5fc01 4635 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4636 out << " return;" << endl;
0f389141 4637 out << " }" << endl;
4638 out << " }" << endl;
d66aa2a9 4639 if (mgr && mgr->IsCollectThroughput() && !IsLocalTest()) {
ec5acdce 4640 out << " TString infolog = \"" << mgr->GetFileInfoLog() << "\";" << endl;
4641 out << " AliAnalysisAlien::MergeInfo(infolog, outputDir);" << endl;
4642 }
7c2cd90a 4643 out << " // all outputs merged, validate" << endl;
4644 out << " ofstream out;" << endl;
4645 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4646 out << " out.close();" << endl;
4647 out << " // read the analysis manager from file" << endl;
d66aa2a9 4648 if (IsLocalTest()) {
4649 out << " printf(\"===================================\n\");" << endl;
4650 out << " printf(\"Testing Terminate()...\\n\");" << endl;
4651 out << " printf(\"===================================\n\");" << endl;
4652 } else {
4653 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
4654 }
5c4250fc 4655 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4656 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4657 out << " mgr->PrintStatus();" << endl;
ec5acdce 4658 if (mgr) {
4659 if (mgr->GetDebugLevel()>3) {
f866cba5 4660 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4661 } else {
9a8288ff 4662 if (TestBit(AliAnalysisGrid::kTest))
4663 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4664 else
4665 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4666 }
4667 }
9a8288ff 4668 out << " TTree *tree = NULL;" << endl;
4669 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4670 out << "}" << endl << endl;
5b273635 4671 if (hasANALYSISalice) {
4672 out <<"//________________________________________________________________________________" << endl;
4673 out << "Bool_t SetupPar(const char *package) {" << endl;
4674 out << "// Compile the package and set it up." << endl;
4675 out << " TString pkgdir = package;" << endl;
4676 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4677 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4678 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4679 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4680 out << " // Check for BUILD.sh and execute" << endl;
4681 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4682 out << " printf(\"*******************************\\n\");" << endl;
4683 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4684 out << " printf(\"*******************************\\n\");" << endl;
4685 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4686 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4687 out << " gSystem->ChangeDirectory(cdir);" << endl;
4688 out << " return kFALSE;" << endl;
4689 out << " }" << endl;
4690 out << " } else {" << endl;
4691 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4692 out << " gSystem->ChangeDirectory(cdir);" << endl;
4693 out << " return kFALSE;" << endl;
4694 out << " }" << endl;
4695 out << " // Check for SETUP.C and execute" << endl;
4696 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4697 out << " printf(\"*******************************\\n\");" << endl;
4698 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4699 out << " printf(\"*******************************\\n\");" << endl;
4700 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4701 out << " } else {" << endl;
4702 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4703 out << " gSystem->ChangeDirectory(cdir);" << endl;
4704 out << " return kFALSE;" << endl;
4705 out << " }" << endl;
4706 out << " // Restore original workdir" << endl;
4707 out << " gSystem->ChangeDirectory(cdir);" << endl;
4708 out << " return kTRUE;" << endl;
4709 out << "}" << endl;
4710 }
0f389141 4711 }
4712 Bool_t copy = kTRUE;
d3b18c4c 4713 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4714 if (copy) {
4715 CdWork();
4716 TString workdir = gGrid->GetHomeDirectory();
4717 workdir += fGridWorkingDir;
4718 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4719 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4720// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4721 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4722 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4723 }
4724}
4725
4726//______________________________________________________________________________
4727Bool_t AliAnalysisAlien::SetupPar(const char *package)
4728{
205b201f 4729// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4730// Note that for loading the compiled library. The current directory should have precedence in
4731// LD_LIBRARY_PATH
4732 TString pkgdir = package;
4733 pkgdir.ReplaceAll(".par","");
b50a41c3 4734 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4735 TString cdir = gSystem->WorkingDirectory();
4736 gSystem->ChangeDirectory(pkgdir);
4737 // Check for BUILD.sh and execute
4738 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4739 printf("**************************************************\n");
4740 printf("*** Building PAR archive %s\n", package);
4741 printf("**************************************************\n");
4742 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4743 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4744 gSystem->ChangeDirectory(cdir);
4745 return kFALSE;
4746 }
4747 } else {
4748 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4749 gSystem->ChangeDirectory(cdir);
4750 return kFALSE;
4751 }
4752 // Check for SETUP.C and execute
4753 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4754 printf("**************************************************\n");
4755 printf("*** Setup PAR archive %s\n", package);
4756 printf("**************************************************\n");
4757 gROOT->Macro("PROOF-INF/SETUP.C");
4758 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4759 } else {
4760 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4761 gSystem->ChangeDirectory(cdir);
4762 return kFALSE;
4763 }
4764 // Restore original workdir
4765 gSystem->ChangeDirectory(cdir);
4766 return kTRUE;
4767}
4768
c57f56b7 4769//______________________________________________________________________________
4770void AliAnalysisAlien::WriteExecutable()
4771{
4772// Generate the alien executable script.
5aa94dc3 4773 // Patch executable with -x to catch error code
4774 if (fExecutableCommand.Contains("root") &&
4775 fExecutableCommand.Contains("-q") &&
4776 !fExecutableCommand.Contains("-x")) fExecutableCommand += " -x";
c57f56b7 4777 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4778 ofstream out;
4779 out.open(fExecutable.Data(), ios::out);
4780 if (out.bad()) {
5513444a 4781 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4782 return;
4783 }
4784 out << "#!/bin/bash" << endl;
2a4d5166 4785 // Make sure we can properly compile par files
4786 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4787 out << "echo \"=========================================\"" << endl;
4788 out << "echo \"############## PATH : ##############\"" << endl;
4789 out << "echo $PATH" << endl;
4790 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4791 out << "echo $LD_LIBRARY_PATH" << endl;
4792 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4793 out << "echo $ROOTSYS" << endl;
4794 out << "echo \"############## which root : ##############\"" << endl;
4795 out << "which root" << endl;
4796 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4797 out << "echo $ALICE_ROOT" << endl;
4798 out << "echo \"############## which aliroot : ##############\"" << endl;
4799 out << "which aliroot" << endl;
9c5ddadc 4800 out << "echo \"############## system limits : ##############\"" << endl;
4801 out << "ulimit -a" << endl;
4802 out << "echo \"############## memory : ##############\"" << endl;
4803 out << "free -m" << endl;
c57f56b7 4804 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4805 out << fExecutableCommand << " ";
370c9880 4806 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4807 out << "RET=$?" << endl;
4808 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4809 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4810 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4811 out << " let sig=\"$RET - 128\""<<endl;
4812 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4813 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4814 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4815 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4816 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4817 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4818 out << " fi"<<endl;
4819 out << " exit $RET"<< endl;
4820 out << "fi" << endl << endl ;
4821 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4822 out << "echo \"############## memory after: ##############\"" << endl;
4823 out << "free -m" << endl;
c57f56b7 4824 }
4825 Bool_t copy = kTRUE;
d3b18c4c 4826 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4827 if (copy) {
4828 CdWork();
4829 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4830 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4831 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4832 workdir += fGridWorkingDir;
4833 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4834 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4835 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4836// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4837 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4838 executable.Data())) Fatal("","Terminating");
c57f56b7 4839 }
4840}
4841
0f389141 4842//______________________________________________________________________________
4843void AliAnalysisAlien::WriteMergeExecutable()
4844{
4845// Generate the alien executable script for the merging job.
4846 if (!fMergeViaJDL) return;
4847 TString mergeExec = fExecutable;
4848 mergeExec.ReplaceAll(".sh", "_merge.sh");
4849 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4850 ofstream out;
4851 out.open(mergeExec.Data(), ios::out);
4852 if (out.bad()) {
4853 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4854 return;
4855 }
4856 out << "#!/bin/bash" << endl;
2a4d5166 4857 // Make sure we can properly compile par files
4858 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4859 out << "echo \"=========================================\"" << endl;
4860 out << "echo \"############## PATH : ##############\"" << endl;
4861 out << "echo $PATH" << endl;
4862 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4863 out << "echo $LD_LIBRARY_PATH" << endl;
4864 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4865 out << "echo $ROOTSYS" << endl;
4866 out << "echo \"############## which root : ##############\"" << endl;
4867 out << "which root" << endl;
4868 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4869 out << "echo $ALICE_ROOT" << endl;
4870 out << "echo \"############## which aliroot : ##############\"" << endl;
4871 out << "which aliroot" << endl;
4872 out << "echo \"############## system limits : ##############\"" << endl;
4873 out << "ulimit -a" << endl;
4874 out << "echo \"############## memory : ##############\"" << endl;
4875 out << "free -m" << endl;
4876 out << "echo \"=========================================\"" << endl << endl;
0f389141 4877 TString mergeMacro = fExecutable;
4878 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4879 if (IsOneStageMerging())
4880 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4881 else
f47d5cb4 4882 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4883 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4884 out << "RET=$?" << endl;
4885 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4886 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4887 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4888 out << " let sig=\"$RET - 128\""<<endl;
4889 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4890 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4891 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4892 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4893 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4894 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4895 out << " fi"<<endl;
4896 out << " exit $RET"<< endl;
4897 out << "fi" << endl << endl ;
0f389141 4898 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4899 out << "echo \"############## memory after: ##############\"" << endl;
4900 out << "free -m" << endl;
0f389141 4901 }
4902 Bool_t copy = kTRUE;
d3b18c4c 4903 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4904 if (copy) {
4905 CdWork();
4906 TString workdir = gGrid->GetHomeDirectory();
4907 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4908 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4909 workdir += fGridWorkingDir;
4910 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4911 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4912 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4913// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4914 if (!copyLocal2Alien("WriteMergeExecutable",
4915 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4916 }
4917}
4918
c57f56b7 4919//______________________________________________________________________________
5513444a 4920void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4921{
4922// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4923// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4924// Next lines: full_path_to_dataset XXX (XXX is a string)
4925// To submit, one has to: submit jdl XXX for all lines
4926 ofstream out;
4927 out.open(filename, ios::out);
4928 if (out.bad()) {
4929 Error("WriteProductionFile", "Bad file name: %s", filename);
4930 return;
4931 }
d3b18c4c 4932 TString workdir;
4933 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4934 workdir = gGrid->GetHomeDirectory();
5513444a 4935 workdir += fGridWorkingDir;
f5e8c702 4936 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4937 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4938 out << locjdl << " " << njobspermaster << endl;
5513444a 4939 Int_t nmasterjobs = fInputFiles->GetEntries();
4940 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4941 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4942 runOutDir.ReplaceAll(".xml", "");
4943 if (fOutputToRunNo)
4944 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4945 else
4946 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4947 }
d3b18c4c 4948 if (gGrid) {
4949 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4950 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4951// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4952 if (!copyLocal2Alien("WriteProductionFile", filename,
4953 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4954 }
5513444a 4955}
4956
4957//______________________________________________________________________________
0f389141 4958void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4959{
4960// Generate the alien validation script.
4961 // Generate the validation script
4962 TObjString *os;
d3b18c4c 4963 if (fValidationScript.IsNull()) {
4964 fValidationScript = fExecutable;
4965 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4966 }
4967 TString validationScript = fValidationScript;
4968 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4969 if (!Connect()) {
4970 Error("WriteValidationScript", "Alien connection required");
4971 return;
4972 }
d3b18c4c 4973 if (!fTerminateFiles.IsNull()) {
4974 fTerminateFiles.Strip();
4975 fTerminateFiles.ReplaceAll(" ",",");
4976 }
a2f5fc01 4977 TString outStream = "";
4978 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4979 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4980 ofstream out;
0d5d317c 4981 out.open(validationScript, ios::out);
c57f56b7 4982 out << "#!/bin/bash" << endl;
4983 out << "##################################################" << endl;
4984 out << "validateout=`dirname $0`" << endl;
4985 out << "validatetime=`date`" << endl;
4986 out << "validated=\"0\";" << endl;
4987 out << "error=0" << endl;
4988 out << "if [ -z $validateout ]" << endl;
4989 out << "then" << endl;
4990 out << " validateout=\".\"" << endl;
4991 out << "fi" << endl << endl;
4992 out << "cd $validateout;" << endl;
4993 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4994 out << "echo \"*******************************************************\"" << outStream << endl;
4995 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4996 out << "" << endl;
a2f5fc01 4997 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4998 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4999 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
5000 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5001 out << "ls -la ./" << outStream << endl;
5002 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 5003 out << "##################################################" << endl;
ebec370a 5004 out << "" << endl;
5005
5006 out << "if [ ! -f stderr ] ; then" << endl;
5007 out << " error=1" << endl;
a2f5fc01 5008 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
5009 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5010 out << "fi" << endl;
5011
b34c9f51 5012 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
5013 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
5014 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
5015 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
5016 out << "" << endl;
5017
ebec370a 5018 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
5019 out << " error=1" << endl;
a2f5fc01 5020 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
5021 out << " echo \"$parArch\" " << outStream << endl;
5022 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5023 out << "fi" << endl;
5024
5025 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
5026 out << " error=1" << endl;
a2f5fc01 5027 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
5028 out << " echo \"$segViol\" " << outStream << endl;
5029 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5030 out << "fi" << endl;
5031
5032 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
5033 out << " error=1" << endl;
a2f5fc01 5034 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
5035 out << " echo \"$segFault\" " << outStream << endl;
5036 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5037 out << "fi" << endl;
5038
b34c9f51 5039 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
5040 out << " error=1" << endl;
a2f5fc01 5041 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
5042 out << " echo \"$glibcErr\" " << outStream << endl;
5043 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 5044 out << "fi" << endl;
5045
ebec370a 5046 // Part dedicated to the specific analyses running into the train
5047
d3b18c4c 5048 TString outputFiles = fOutputFiles;
5049 if (merge && !fTerminateFiles.IsNull()) {
5050 outputFiles += ",";
5051 outputFiles += fTerminateFiles;
5052 }
5053 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 5054 TIter next1(arr);
a2f5fc01 5055 TString outputFile;
bbd88db8 5056 while (!merge && (os=(TObjString*)next1())) {
5057 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 5058 outputFile = os->GetString();
5059 Int_t index = outputFile.Index("@");
5060 if (index > 0) outputFile.Remove(index);
bbd88db8 5061 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 5062 if (outputFile.Contains("*")) continue;
5063 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 5064 out << " error=1" << endl;
7c2cd90a 5065 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
5066 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 5067 out << "fi" << endl;
5068 }
5069 delete arr;
7c2cd90a 5070 out << "if ! [ -f outputs_valid ] ; then" << endl;
5071 out << " error=1" << endl;
5072 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
5073 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
5074 out << "fi" << endl;
923e2ca5 5075
c57f56b7 5076 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 5077 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 5078 if (!IsKeepLogs()) {
5079 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 5080 outStream = "";
149d288c 5081 out << " rm -f std*" << endl;
5082 }
c57f56b7 5083 out << "fi" << endl;
5084
a2f5fc01 5085 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5086 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 5087 out << "cd -" << endl;
5088 out << "exit $error" << endl;
5089 }
5090 Bool_t copy = kTRUE;
d3b18c4c 5091 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 5092 if (copy) {
5093 CdWork();
5094 TString workdir = gGrid->GetHomeDirectory();
5095 workdir += fGridWorkingDir;
d3b18c4c 5096 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 5097 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 5098// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
5099 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
5100 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 5101 }
5102}