]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
protection added
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
715da79c 32#include "TInterpreter.h"
c57f56b7 33#include "TFile.h"
830acc4c 34#include "TFileCollection.h"
3bdcb562 35#include "TChain.h"
c57f56b7 36#include "TObjString.h"
37#include "TObjArray.h"
77f34eae 38#include "TMacro.h"
c57f56b7 39#include "TGrid.h"
40#include "TGridResult.h"
41#include "TGridCollection.h"
42#include "TGridJDL.h"
d2a409b2 43#include "TGridJobStatusList.h"
44#include "TGridJobStatus.h"
c57f56b7 45#include "TFileMerger.h"
46#include "AliAnalysisManager.h"
77f34eae 47#include "AliAnalysisTaskCfg.h"
bb885a9e 48#include "AliVEventHandler.h"
49#include "AliAnalysisDataContainer.h"
65eb22f8 50#include "AliMultiInputEventHandler.h"
c57f56b7 51
c82bb898 52using std::ofstream;
53using std::ifstream;
54using std::ios;
55using std::endl;
c57f56b7 56ClassImp(AliAnalysisAlien)
6c395669 57#if 0
58;
59#endif
c57f56b7 60
6c395669 61namespace {
62 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
63 {
64 TString sl(Form("file:%s", loc));
65 TString sr(Form("alien://%s", rem));
66 Bool_t ret = TFile::Cp(sl, sr);
67 if (!ret) {
68 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
69 }
70 return ret;
71 }
72}
73
c57f56b7 74//______________________________________________________________________________
75AliAnalysisAlien::AliAnalysisAlien()
76 :AliAnalysisGrid(),
77 fGridJDL(NULL),
0f389141 78 fMergingJDL(NULL),
c57f56b7 79 fPrice(0),
80 fTTL(0),
81 fSplitMaxInputFileNumber(0),
82 fMaxInitFailed(0),
83 fMasterResubmitThreshold(0),
bb885a9e 84 fNtestFiles(0),
319593fb 85 fNrunsPerMaster(0),
16a4353c 86 fMaxMergeFiles(0),
b3e07543 87 fMaxMergeStages(0),
d2a409b2 88 fNsubmitted(0),
a3e84053 89 fProductionMode(0),
cd11251e 90 fOutputToRunNo(0),
0f389141 91 fMergeViaJDL(0),
92 fFastReadOption(0),
e1c22e21 93 fOverwriteMode(1),
149d288c 94 fNreplicas(2),
3bdcb562 95 fNproofWorkers(0),
27734f0e 96 fNproofWorkersPerSlave(0),
3bdcb562 97 fProofReset(0),
715da79c 98 fNMCevents(0),
99 fNMCjobs(0),
c57f56b7 100 fRunNumbers(),
101 fExecutable(),
0a1c1f7f 102 fExecutableCommand(),
c57f56b7 103 fArguments(),
631c0b05 104 fExecutableArgs(),
c57f56b7 105 fAnalysisMacro(),
106 fAnalysisSource(),
d3b18c4c 107 fValidationScript(),
d5c6455a 108 fAdditionalRootLibs(),
c57f56b7 109 fAdditionalLibs(),
715da79c 110 fGeneratorLibs(),
c57f56b7 111 fSplitMode(),
112 fAPIVersion(),
113 fROOTVersion(),
114 fAliROOTVersion(),
648174cf 115 fExternalPackages(),
c57f56b7 116 fUser(),
117 fGridWorkingDir(),
118 fGridDataDir(),
119 fDataPattern(),
120 fGridOutputDir(),
121 fOutputArchive(),
122 fOutputFiles(),
123 fInputFormat(),
e7c71df0 124 fDatasetName(),
c57f56b7 125 fJDLName(),
d3b18c4c 126 fTerminateFiles(),
bb885a9e 127 fMergeExcludes(),
37d21c01 128 fRegisterExcludes(),
f965131e 129 fIncludePath(),
bb885a9e 130 fCloseSE(),
0df6ccf2 131 fFriendChainName(),
c6cb3634 132 fJobTag(),
648174cf 133 fOutputSingle(),
5fce53f4 134 fRunPrefix(),
3bdcb562 135 fProofCluster(),
136 fProofDataSet(),
137 fFileForTestMode(),
3bdcb562 138 fAliRootMode(),
d88e4159 139 fProofProcessOpt(),
f47d5cb4 140 fMergeDirName(),
4e5c5506 141 fInputFiles(0),
28ce6c9c 142 fPackages(0),
77f34eae 143 fModules(0),
d88e4159 144 fProofParam(),
a41b2882 145 fDropToShell(true),
715da79c 146 fMCLoop(false),
a41b2882 147 fGridJobIDs(""),
148 fGridStages(""),
780ec36d 149 fFriendLibs(""),
150 fTreeName()
c57f56b7 151{
152// Dummy ctor.
153 SetDefaults();
154}
155
156//______________________________________________________________________________
157AliAnalysisAlien::AliAnalysisAlien(const char *name)
158 :AliAnalysisGrid(name),
159 fGridJDL(NULL),
0f389141 160 fMergingJDL(NULL),
c57f56b7 161 fPrice(0),
162 fTTL(0),
163 fSplitMaxInputFileNumber(0),
164 fMaxInitFailed(0),
165 fMasterResubmitThreshold(0),
bb885a9e 166 fNtestFiles(0),
319593fb 167 fNrunsPerMaster(0),
16a4353c 168 fMaxMergeFiles(0),
b3e07543 169 fMaxMergeStages(0),
d2a409b2 170 fNsubmitted(0),
a3e84053 171 fProductionMode(0),
cd11251e 172 fOutputToRunNo(0),
0f389141 173 fMergeViaJDL(0),
174 fFastReadOption(0),
e1c22e21 175 fOverwriteMode(1),
149d288c 176 fNreplicas(2),
3bdcb562 177 fNproofWorkers(0),
27734f0e 178 fNproofWorkersPerSlave(0),
3bdcb562 179 fProofReset(0),
715da79c 180 fNMCevents(0),
181 fNMCjobs(0),
c57f56b7 182 fRunNumbers(),
183 fExecutable(),
0a1c1f7f 184 fExecutableCommand(),
c57f56b7 185 fArguments(),
631c0b05 186 fExecutableArgs(),
c57f56b7 187 fAnalysisMacro(),
188 fAnalysisSource(),
d3b18c4c 189 fValidationScript(),
d5c6455a 190 fAdditionalRootLibs(),
c57f56b7 191 fAdditionalLibs(),
715da79c 192 fGeneratorLibs(),
c57f56b7 193 fSplitMode(),
194 fAPIVersion(),
195 fROOTVersion(),
196 fAliROOTVersion(),
648174cf 197 fExternalPackages(),
c57f56b7 198 fUser(),
199 fGridWorkingDir(),
200 fGridDataDir(),
201 fDataPattern(),
202 fGridOutputDir(),
203 fOutputArchive(),
204 fOutputFiles(),
205 fInputFormat(),
e7c71df0 206 fDatasetName(),
c57f56b7 207 fJDLName(),
d3b18c4c 208 fTerminateFiles(),
bb885a9e 209 fMergeExcludes(),
37d21c01 210 fRegisterExcludes(),
f965131e 211 fIncludePath(),
bb885a9e 212 fCloseSE(),
0df6ccf2 213 fFriendChainName(),
c6cb3634 214 fJobTag(),
648174cf 215 fOutputSingle(),
5fce53f4 216 fRunPrefix(),
3bdcb562 217 fProofCluster(),
218 fProofDataSet(),
219 fFileForTestMode(),
3bdcb562 220 fAliRootMode(),
d88e4159 221 fProofProcessOpt(),
f47d5cb4 222 fMergeDirName(),
4e5c5506 223 fInputFiles(0),
28ce6c9c 224 fPackages(0),
77f34eae 225 fModules(0),
d88e4159 226 fProofParam(),
0ad3ea1f 227 fDropToShell(true),
715da79c 228 fMCLoop(false),
0ad3ea1f 229 fGridJobIDs(""),
a41b2882 230 fGridStages(""),
780ec36d 231 fFriendLibs(""),
232 fTreeName()
c57f56b7 233{
234// Default ctor.
235 SetDefaults();
236}
237
238//______________________________________________________________________________
239AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
240 :AliAnalysisGrid(other),
241 fGridJDL(NULL),
0f389141 242 fMergingJDL(NULL),
c57f56b7 243 fPrice(other.fPrice),
244 fTTL(other.fTTL),
245 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
246 fMaxInitFailed(other.fMaxInitFailed),
247 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 248 fNtestFiles(other.fNtestFiles),
319593fb 249 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 250 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 251 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 252 fNsubmitted(other.fNsubmitted),
a3e84053 253 fProductionMode(other.fProductionMode),
cd11251e 254 fOutputToRunNo(other.fOutputToRunNo),
0f389141 255 fMergeViaJDL(other.fMergeViaJDL),
256 fFastReadOption(other.fFastReadOption),
257 fOverwriteMode(other.fOverwriteMode),
149d288c 258 fNreplicas(other.fNreplicas),
3bdcb562 259 fNproofWorkers(other.fNproofWorkers),
27734f0e 260 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 261 fProofReset(other.fProofReset),
715da79c 262 fNMCevents(other.fNMCevents),
263 fNMCjobs(other.fNMCjobs),
c57f56b7 264 fRunNumbers(other.fRunNumbers),
265 fExecutable(other.fExecutable),
0a1c1f7f 266 fExecutableCommand(other.fExecutableCommand),
c57f56b7 267 fArguments(other.fArguments),
631c0b05 268 fExecutableArgs(other.fExecutableArgs),
c57f56b7 269 fAnalysisMacro(other.fAnalysisMacro),
270 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 271 fValidationScript(other.fValidationScript),
d5c6455a 272 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 273 fAdditionalLibs(other.fAdditionalLibs),
715da79c 274 fGeneratorLibs(other.fGeneratorLibs),
c57f56b7 275 fSplitMode(other.fSplitMode),
276 fAPIVersion(other.fAPIVersion),
277 fROOTVersion(other.fROOTVersion),
278 fAliROOTVersion(other.fAliROOTVersion),
648174cf 279 fExternalPackages(other.fExternalPackages),
c57f56b7 280 fUser(other.fUser),
281 fGridWorkingDir(other.fGridWorkingDir),
282 fGridDataDir(other.fGridDataDir),
283 fDataPattern(other.fDataPattern),
284 fGridOutputDir(other.fGridOutputDir),
285 fOutputArchive(other.fOutputArchive),
286 fOutputFiles(other.fOutputFiles),
287 fInputFormat(other.fInputFormat),
e7c71df0 288 fDatasetName(other.fDatasetName),
c57f56b7 289 fJDLName(other.fJDLName),
d3b18c4c 290 fTerminateFiles(other.fTerminateFiles),
bb885a9e 291 fMergeExcludes(other.fMergeExcludes),
37d21c01 292 fRegisterExcludes(other.fRegisterExcludes),
f965131e 293 fIncludePath(other.fIncludePath),
bb885a9e 294 fCloseSE(other.fCloseSE),
0df6ccf2 295 fFriendChainName(other.fFriendChainName),
c6cb3634 296 fJobTag(other.fJobTag),
648174cf 297 fOutputSingle(other.fOutputSingle),
5fce53f4 298 fRunPrefix(other.fRunPrefix),
3bdcb562 299 fProofCluster(other.fProofCluster),
300 fProofDataSet(other.fProofDataSet),
301 fFileForTestMode(other.fFileForTestMode),
3bdcb562 302 fAliRootMode(other.fAliRootMode),
d88e4159 303 fProofProcessOpt(other.fProofProcessOpt),
f47d5cb4 304 fMergeDirName(other.fMergeDirName),
4e5c5506 305 fInputFiles(0),
28ce6c9c 306 fPackages(0),
77f34eae 307 fModules(0),
d88e4159 308 fProofParam(),
0ad3ea1f 309 fDropToShell(other.fDropToShell),
715da79c 310 fMCLoop(other.fMCLoop),
0ad3ea1f 311 fGridJobIDs(other.fGridJobIDs),
a41b2882 312 fGridStages(other.fGridStages),
780ec36d 313 fFriendLibs(other.fFriendLibs),
314 fTreeName(other.fTreeName)
c57f56b7 315{
316// Copy ctor.
317 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 318 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 319 fRunRange[0] = other.fRunRange[0];
320 fRunRange[1] = other.fRunRange[1];
c57f56b7 321 if (other.fInputFiles) {
322 fInputFiles = new TObjArray();
323 TIter next(other.fInputFiles);
324 TObject *obj;
325 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
326 fInputFiles->SetOwner();
327 }
4e5c5506 328 if (other.fPackages) {
329 fPackages = new TObjArray();
330 TIter next(other.fPackages);
331 TObject *obj;
332 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
333 fPackages->SetOwner();
334 }
77f34eae 335 if (other.fModules) {
336 fModules = new TObjArray();
337 fModules->SetOwner();
338 TIter next(other.fModules);
339 AliAnalysisTaskCfg *mod, *crt;
340 while ((crt=(AliAnalysisTaskCfg*)next())) {
341 mod = new AliAnalysisTaskCfg(*crt);
342 fModules->Add(mod);
343 }
344 }
c57f56b7 345}
346
347//______________________________________________________________________________
348AliAnalysisAlien::~AliAnalysisAlien()
349{
350// Destructor.
77f34eae 351 delete fGridJDL;
352 delete fMergingJDL;
353 delete fInputFiles;
354 delete fPackages;
355 delete fModules;
28ce6c9c 356 fProofParam.DeleteAll();
c57f56b7 357}
358
359//______________________________________________________________________________
360AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
361{
362// Assignment.
363 if (this != &other) {
364 AliAnalysisGrid::operator=(other);
365 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 366 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 367 fPrice = other.fPrice;
368 fTTL = other.fTTL;
369 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
370 fMaxInitFailed = other.fMaxInitFailed;
371 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 372 fNtestFiles = other.fNtestFiles;
a3e84053 373 fNrunsPerMaster = other.fNrunsPerMaster;
374 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 375 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 376 fNsubmitted = other.fNsubmitted;
377 fProductionMode = other.fProductionMode;
cd11251e 378 fOutputToRunNo = other.fOutputToRunNo;
0f389141 379 fMergeViaJDL = other.fMergeViaJDL;
380 fFastReadOption = other.fFastReadOption;
381 fOverwriteMode = other.fOverwriteMode;
149d288c 382 fNreplicas = other.fNreplicas;
3bdcb562 383 fNproofWorkers = other.fNproofWorkers;
27734f0e 384 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 385 fProofReset = other.fProofReset;
715da79c 386 fNMCevents = other.fNMCevents;
387 fNMCjobs = other.fNMCjobs;
c57f56b7 388 fRunNumbers = other.fRunNumbers;
389 fExecutable = other.fExecutable;
0a1c1f7f 390 fExecutableCommand = other.fExecutableCommand;
c57f56b7 391 fArguments = other.fArguments;
631c0b05 392 fExecutableArgs = other.fExecutableArgs;
c57f56b7 393 fAnalysisMacro = other.fAnalysisMacro;
394 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 395 fValidationScript = other.fValidationScript;
d5c6455a 396 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 397 fAdditionalLibs = other.fAdditionalLibs;
715da79c 398 fGeneratorLibs = other.fGeneratorLibs;
c57f56b7 399 fSplitMode = other.fSplitMode;
400 fAPIVersion = other.fAPIVersion;
401 fROOTVersion = other.fROOTVersion;
402 fAliROOTVersion = other.fAliROOTVersion;
648174cf 403 fExternalPackages = other.fExternalPackages;
c57f56b7 404 fUser = other.fUser;
405 fGridWorkingDir = other.fGridWorkingDir;
406 fGridDataDir = other.fGridDataDir;
407 fDataPattern = other.fDataPattern;
408 fGridOutputDir = other.fGridOutputDir;
409 fOutputArchive = other.fOutputArchive;
410 fOutputFiles = other.fOutputFiles;
411 fInputFormat = other.fInputFormat;
e7c71df0 412 fDatasetName = other.fDatasetName;
c57f56b7 413 fJDLName = other.fJDLName;
d3b18c4c 414 fTerminateFiles = other.fTerminateFiles;
bb885a9e 415 fMergeExcludes = other.fMergeExcludes;
37d21c01 416 fRegisterExcludes = other.fRegisterExcludes;
f965131e 417 fIncludePath = other.fIncludePath;
bb885a9e 418 fCloseSE = other.fCloseSE;
0df6ccf2 419 fFriendChainName = other.fFriendChainName;
c6cb3634 420 fJobTag = other.fJobTag;
648174cf 421 fOutputSingle = other.fOutputSingle;
5fce53f4 422 fRunPrefix = other.fRunPrefix;
3bdcb562 423 fProofCluster = other.fProofCluster;
424 fProofDataSet = other.fProofDataSet;
425 fFileForTestMode = other.fFileForTestMode;
3bdcb562 426 fAliRootMode = other.fAliRootMode;
d88e4159 427 fProofProcessOpt = other.fProofProcessOpt;
f47d5cb4 428 fMergeDirName = other.fMergeDirName;
d88e4159 429 fDropToShell = other.fDropToShell;
715da79c 430 fMCLoop = other.fMCLoop;
0ad3ea1f 431 fGridJobIDs = other.fGridJobIDs;
432 fGridStages = other.fGridStages;
a41b2882 433 fFriendLibs = other.fFriendLibs;
780ec36d 434 fTreeName = other.fTreeName;
c57f56b7 435 if (other.fInputFiles) {
436 fInputFiles = new TObjArray();
437 TIter next(other.fInputFiles);
438 TObject *obj;
439 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
440 fInputFiles->SetOwner();
441 }
4e5c5506 442 if (other.fPackages) {
443 fPackages = new TObjArray();
444 TIter next(other.fPackages);
445 TObject *obj;
446 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
447 fPackages->SetOwner();
448 }
77f34eae 449 if (other.fModules) {
450 fModules = new TObjArray();
451 fModules->SetOwner();
452 TIter next(other.fModules);
453 AliAnalysisTaskCfg *mod, *crt;
454 while ((crt=(AliAnalysisTaskCfg*)next())) {
455 mod = new AliAnalysisTaskCfg(*crt);
456 fModules->Add(mod);
457 }
458 }
c57f56b7 459 }
460 return *this;
461}
462
37d21c01 463//______________________________________________________________________________
464void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
465{
466// Add a single additional library to be loaded. Extension must be present.
467 TString lib(name);
468 if (!lib.Contains(".")) {
469 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
470 return;
471 }
472 if (fAdditionalLibs.Contains(name)) {
473 Warning("AddAdditionalLibrary", "Library %s already added.", name);
474 return;
475 }
476 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
477 fAdditionalLibs += lib;
478}
479
77f34eae 480//______________________________________________________________________________
481void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
482{
483// Adding a module. Checks if already existing. Becomes owned by this.
484 if (!module) return;
485 if (GetModule(module->GetName())) {
486 Error("AddModule", "A module having the same name %s already added", module->GetName());
487 return;
488 }
489 if (!fModules) {
490 fModules = new TObjArray();
491 fModules->SetOwner();
492 }
493 fModules->Add(module);
494}
495
496//______________________________________________________________________________
497void AliAnalysisAlien::AddModules(TObjArray *list)
498{
499// Adding a list of modules. Checks if already existing. Becomes owned by this.
500 TIter next(list);
501 AliAnalysisTaskCfg *module;
502 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
503}
504
505//______________________________________________________________________________
506Bool_t AliAnalysisAlien::CheckDependencies()
507{
508// Check if all dependencies are satisfied. Reorder modules if needed.
509 Int_t nmodules = GetNmodules();
510 if (!nmodules) {
511 Warning("CheckDependencies", "No modules added yet to check their dependencies");
512 return kTRUE;
513 }
514 AliAnalysisTaskCfg *mod = 0;
515 AliAnalysisTaskCfg *dep = 0;
516 TString depname;
517 Int_t i, j, k;
518 for (i=0; i<nmodules; i++) {
519 mod = (AliAnalysisTaskCfg*) fModules->At(i);
520 Int_t ndeps = mod->GetNdeps();
521 Int_t istart = i;
522 for (j=0; j<ndeps; j++) {
523 depname = mod->GetDependency(j);
524 dep = GetModule(depname);
525 if (!dep) {
526 Error("CheckDependencies","Dependency %s not added for module %s",
527 depname.Data(), mod->GetName());
528 return kFALSE;
529 }
530 if (dep->NeedsDependency(mod->GetName())) {
531 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
532 mod->GetName(), dep->GetName());
533 return kFALSE;
534 }
535 Int_t idep = fModules->IndexOf(dep);
536 // The dependency task must come first
537 if (idep>i) {
538 // Remove at idep and move all objects below up one slot
539 // down to index i included.
540 fModules->RemoveAt(idep);
3e40fd4c 541 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 542 fModules->AddAt(dep, i++);
543 }
544 //Redo from istart if dependencies were inserted
545 if (i>istart) i=istart-1;
546 }
547 }
548 return kTRUE;
549}
550
3e40fd4c 551//______________________________________________________________________________
552AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
553{
554// Create the analysis manager and optionally execute the macro in filename.
555 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
715da79c 556 if (mgr) {
557 mgr->SetMCLoop(fMCLoop);
558 return mgr;
559 }
3e40fd4c 560 mgr = new AliAnalysisManager(name);
561 mgr->SetGridHandler((AliAnalysisGrid*)this);
715da79c 562 mgr->SetMCLoop(fMCLoop);
3e40fd4c 563 if (strlen(filename)) {
564 TString line = gSystem->ExpandPathName(filename);
565 line.Prepend(".x ");
566 gROOT->ProcessLine(line.Data());
567 }
568 return mgr;
569}
570
77f34eae 571//______________________________________________________________________________
572Int_t AliAnalysisAlien::GetNmodules() const
573{
574// Get number of modules.
575 if (!fModules) return 0;
576 return fModules->GetEntries();
577}
578
579//______________________________________________________________________________
580AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
581{
582// Get a module by name.
583 if (!fModules) return 0;
584 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
585}
586
587//______________________________________________________________________________
588Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
589{
590// Load a given module.
591 if (mod->IsLoaded()) return kTRUE;
192ddca2 592 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
593 if (!mgr) {
594 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
595 return kFALSE;
596 }
77f34eae 597 Int_t ndeps = mod->GetNdeps();
598 TString depname;
599 for (Int_t j=0; j<ndeps; j++) {
600 depname = mod->GetDependency(j);
601 AliAnalysisTaskCfg *dep = GetModule(depname);
602 if (!dep) {
603 Error("LoadModule","Dependency %s not existing for module %s",
604 depname.Data(), mod->GetName());
605 return kFALSE;
606 }
607 if (!LoadModule(dep)) {
608 Error("LoadModule","Dependency %s for module %s could not be loaded",
609 depname.Data(), mod->GetName());
610 return kFALSE;
611 }
612 }
613 // Load libraries for the module
614 if (!mod->CheckLoadLibraries()) {
615 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
616 return kFALSE;
617 }
192ddca2 618 // Check if a custom file name was requested
619 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
620
621 // Check if a custom terminate file name was requested
622 if (strlen(mod->GetTerminateFileName())) {
623 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
624 fTerminateFiles += mod->GetTerminateFileName();
625 }
626
77f34eae 627 // Execute the macro
628 if (mod->ExecuteMacro()<0) {
629 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
630 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
631 return kFALSE;
632 }
633 // Configure dependencies
634 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
635 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
636 mod->GetConfigMacro()->GetTitle(), mod->GetName());
637 return kFALSE;
638 }
5e8c7c22 639 // Adjust extra libraries
640 Int_t nlibs = mod->GetNlibs();
641 TString lib;
642 for (Int_t i=0; i<nlibs; i++) {
643 lib = mod->GetLibrary(i);
5e8c7c22 644 lib = Form("lib%s.so", lib.Data());
c9d6bd12 645 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 646 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
647 fAdditionalLibs += lib;
648 }
77f34eae 649 return kTRUE;
650}
651
139fe6ba 652//______________________________________________________________________________
653Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
654{
655// Generate the full train.
656 fAdditionalLibs = "";
657 if (!LoadModules()) return kFALSE;
658 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
659 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 660 mgr->RunLocalInit();
139fe6ba 661 mgr->PrintStatus();
662 Int_t productionMode = fProductionMode;
663 SetProductionMode();
664 TString macro = fAnalysisMacro;
665 TString executable = fExecutable;
666 TString validation = fValidationScript;
667 TString execCommand = fExecutableCommand;
668 SetAnalysisMacro(Form("%s.C", name));
669 SetExecutable(Form("%s.sh", name));
a8e31fb0 670// SetExecutableCommand("aliroot -b -q ");
139fe6ba 671 SetValidationScript(Form("%s_validation.sh", name));
672 StartAnalysis();
673 SetProductionMode(productionMode);
674 fAnalysisMacro = macro;
675 fExecutable = executable;
676 fExecutableCommand = execCommand;
677 fValidationScript = validation;
678 return kTRUE;
679}
680
77f34eae 681//______________________________________________________________________________
21ca8e59 682Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 683{
684// Generate test macros for a single module or for the full train.
5e8c7c22 685 fAdditionalLibs = "";
77f34eae 686 if (strlen(modname)) {
21ca8e59 687 if (!CheckDependencies()) return kFALSE;
688 AliAnalysisTaskCfg *mod = GetModule(modname);
689 if (!mod) {
690 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
691 return kFALSE;
692 }
693 if (!LoadModule(mod)) return kFALSE;
a41b2882 694 if (!LoadFriendLibs()) return kFALSE;
21ca8e59 695 } else if (!LoadModules()) return kFALSE;
696 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
697 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 698 mgr->RunLocalInit();
21ca8e59 699 mgr->PrintStatus();
700 SetLocalTest(kTRUE);
701 Int_t productionMode = fProductionMode;
702 SetProductionMode();
703 TString macro = fAnalysisMacro;
704 TString executable = fExecutable;
705 TString validation = fValidationScript;
706 TString execCommand = fExecutableCommand;
707 SetAnalysisMacro(Form("%s.C", name));
708 SetExecutable(Form("%s.sh", name));
d66aa2a9 709 fOutputFiles = GetListOfFiles("outaod");
710 // Add extra files registered to the analysis manager
711 TString extra = GetListOfFiles("ext");
712 if (!extra.IsNull()) {
713 extra.ReplaceAll(".root", "*.root");
714 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
715 fOutputFiles += extra;
716 }
a8e31fb0 717// SetExecutableCommand("aliroot -b -q ");
21ca8e59 718 SetValidationScript(Form("%s_validation.sh", name));
719 WriteAnalysisFile();
720 WriteAnalysisMacro();
721 WriteExecutable();
722 WriteValidationScript();
ffbe06b9 723 WriteMergingMacro();
724 WriteMergeExecutable();
725 WriteValidationScript(kTRUE);
21ca8e59 726 SetLocalTest(kFALSE);
727 SetProductionMode(productionMode);
728 fAnalysisMacro = macro;
729 fExecutable = executable;
730 fExecutableCommand = execCommand;
731 fValidationScript = validation;
77f34eae 732 return kTRUE;
733}
734
735//______________________________________________________________________________
736Bool_t AliAnalysisAlien::LoadModules()
737{
738// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 739 fAdditionalLibs = "";
21ca8e59 740 Int_t nmodules = GetNmodules();
741 if (!nmodules) {
742 Warning("LoadModules", "No module to be loaded");
743 return kTRUE;
744 }
3e40fd4c 745 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
746 if (!mgr) {
747 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
748 return kFALSE;
749 }
77f34eae 750 if (!CheckDependencies()) return kFALSE;
21ca8e59 751 nmodules = GetNmodules();
77f34eae 752 AliAnalysisTaskCfg *mod;
21ca8e59 753 for (Int_t imod=0; imod<nmodules; imod++) {
754 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 755 if (!LoadModule(mod)) return kFALSE;
756 }
a41b2882 757 // Load additional friend libraries
758 return LoadFriendLibs();
77f34eae 759}
760
a41b2882 761//______________________________________________________________________________
762Bool_t AliAnalysisAlien::LoadFriendLibs() const
763{
764// Load libraries required for reading friends.
765 if (fFriendLibs.Length()) {
766 TObjArray *list = 0;
767 TString lib;
768 if (fFriendLibs.Contains(",")) list = fFriendLibs.Tokenize(",");
769 else list = fFriendLibs.Tokenize(" ");
770 for (Int_t ilib=0; ilib<list->GetEntriesFast(); ilib++) {
771 lib = list->At(ilib)->GetName();
772 lib.ReplaceAll(".so","");
773 lib.ReplaceAll(" ","");
774 if (lib.BeginsWith("lib")) lib.Remove(0, 3);
775 lib.Prepend("lib");
776 Int_t loaded = strlen(gSystem->GetLibraries(lib,"",kFALSE));
777 if (!loaded) loaded = gSystem->Load(lib);
778 if (loaded < 0) {
779 Error("LoadModules", "Cannot load library for friends %s", lib.Data());
780 return kFALSE;
781 }
782 }
783 delete list;
784 }
785 return kTRUE;
786}
787
74013535 788//______________________________________________________________________________
789void AliAnalysisAlien::SetRunPrefix(const char *prefix)
790{
791// Set the run number format. Can be a prefix or a format like "%09d"
792 fRunPrefix = prefix;
793 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
794}
795
f965131e 796//______________________________________________________________________________
797void AliAnalysisAlien::AddIncludePath(const char *path)
798{
799// Add include path in the remote analysis macro.
800 TString p(path);
801 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
802 else fIncludePath += Form("-I%s ", path);
803}
804
c57f56b7 805//______________________________________________________________________________
806void AliAnalysisAlien::AddRunNumber(Int_t run)
807{
808// Add a run number to the list of runs to be processed.
809 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 810 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 811}
812
98690895 813//______________________________________________________________________________
814void AliAnalysisAlien::AddRunList(const char* runList)
815{
816// Add several runs into the list of runs; they are expected to be separated by a blank character.
817 TString sList = runList;
818 TObjArray *list = sList.Tokenize(" ");
819 Int_t n = list->GetEntries();
820 for (Int_t i = 0; i < n; i++) {
821 TObjString *os = (TObjString*)list->At(i);
822 AddRunNumber(os->GetString().Atoi());
823 }
824 delete list;
825}
826
ee75cfc3 827//______________________________________________________________________________
828void AliAnalysisAlien::AddRunNumber(const char* run)
829{
830// Add a run number to the list of runs to be processed.
7aa5d539 831 TString runs = run;
832 TObjString *os;
833 TObjArray *arr = runs.Tokenize(" ");
834 TIter next(arr);
835 TString prefix;
836 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
837 while ((os=(TObjString*)next())){
838 if (fRunNumbers.Length()) fRunNumbers += " ";
839 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
840 }
841 delete arr;
ee75cfc3 842}
843
c57f56b7 844//______________________________________________________________________________
845void AliAnalysisAlien::AddDataFile(const char *lfn)
846{
847// Adds a data file to the input to be analysed. The file should be a valid LFN
848// or point to an existing file in the alien workdir.
849 if (!fInputFiles) fInputFiles = new TObjArray();
850 fInputFiles->Add(new TObjString(lfn));
851}
648174cf 852
853//______________________________________________________________________________
854void AliAnalysisAlien::AddExternalPackage(const char *package)
855{
856// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
857 if (fExternalPackages) fExternalPackages += " ";
858 fExternalPackages += package;
859}
860
c57f56b7 861//______________________________________________________________________________
862Bool_t AliAnalysisAlien::Connect()
863{
864// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
865 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 866 if (fProductionMode) return kTRUE;
c57f56b7 867 if (!gGrid) {
868 Info("Connect", "Trying to connect to AliEn ...");
869 TGrid::Connect("alien://");
870 }
871 if (!gGrid || !gGrid->IsConnected()) {
872 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
873 return kFALSE;
874 }
875 fUser = gGrid->GetUser();
876 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
877 return kTRUE;
878}
879
880//______________________________________________________________________________
881void AliAnalysisAlien::CdWork()
882{
883// Check validity of alien workspace. Create directory if possible.
884 if (!Connect()) {
885 Error("CdWork", "Alien connection required");
886 return;
887 }
888 TString homedir = gGrid->GetHomeDirectory();
889 TString workdir = homedir + fGridWorkingDir;
923e2ca5 890 if (DirectoryExists(workdir)) {
891 gGrid->Cd(workdir);
892 return;
893 }
894 // Work directory not existing - create it
895 gGrid->Cd(homedir);
b93f8109 896 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 897 gGrid->Cd(fGridWorkingDir);
d3b18c4c 898 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 899 } else {
d3b18c4c 900 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 901 workdir.Data(), homedir.Data());
902 fGridWorkingDir = "";
903 }
c57f56b7 904}
905
348be253 906//______________________________________________________________________________
907Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
908{
909// Check if file copying is possible.
d3b18c4c 910 if (fProductionMode) return kTRUE;
b5ae4fe0 911 TString salienpath(alienpath);
912 if (salienpath.Contains(" ")) {
913 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
914 return kFALSE;
915 }
348be253 916 if (!Connect()) {
917 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
918 return kFALSE;
919 }
d3339be3 920 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
921 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 922 // Check if alien_CLOSE_SE is defined
923 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
924 if (!closeSE.IsNull()) {
925 Info("CheckFileCopy", "Your current close storage is pointing to: \
926 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
927 } else {
928 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
929 }
930 // Check if grid directory exists.
931 if (!DirectoryExists(alienpath)) {
932 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
933 return kFALSE;
934 }
b5ae4fe0 935 TString stest = "plugin_test_copy";
936 TFile f(stest, "RECREATE");
348be253 937 // User may not have write permissions to current directory
938 if (f.IsZombie()) {
939 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
940 gSystem->WorkingDirectory());
941 return kFALSE;
942 }
943 f.Close();
b5ae4fe0 944 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
945 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 946 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 947 \n# 1. Make sure you have write permissions there. If this is the case: \
948 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
949 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
950 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
951 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 952 gSystem->Unlink(stest.Data());
348be253 953 return kFALSE;
954 }
b5ae4fe0 955 gSystem->Unlink(stest.Data());
956 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 957 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 958 return kTRUE;
959}
960
c57f56b7 961//______________________________________________________________________________
962Bool_t AliAnalysisAlien::CheckInputData()
963{
964// Check validity of input data. If necessary, create xml files.
d3b18c4c 965 if (fProductionMode) return kTRUE;
a8739e8a 966 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 967 if (!fGridDataDir.Length()) {
968 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
969 return kFALSE;
970 }
9f020952 971 if (fMergeViaJDL) {
972 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
973 return kFALSE;
974 }
d2a409b2 975 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 976 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
977 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 978 return kTRUE;
c57f56b7 979 }
980 // Process declared files
a2f5fc01 981 Bool_t isCollection = kFALSE;
982 Bool_t isXml = kFALSE;
983 Bool_t useTags = kFALSE;
c57f56b7 984 Bool_t checked = kFALSE;
d3b18c4c 985 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 986 TString file;
987 TString workdir = gGrid->GetHomeDirectory();
988 workdir += fGridWorkingDir;
989 if (fInputFiles) {
990 TObjString *objstr;
991 TIter next(fInputFiles);
992 while ((objstr=(TObjString*)next())) {
993 file = workdir;
994 file += "/";
995 file += objstr->GetString();
996 // Store full lfn path
997 if (FileExists(file)) objstr->SetString(file);
998 else {
999 file = objstr->GetName();
1000 if (!FileExists(objstr->GetName())) {
1001 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
1002 objstr->GetName(), workdir.Data());
1003 return kFALSE;
1004 }
1005 }
1006 Bool_t iscoll, isxml, usetags;
1007 CheckDataType(file, iscoll, isxml, usetags);
1008 if (!checked) {
1009 checked = kTRUE;
a2f5fc01 1010 isCollection = iscoll;
1011 isXml = isxml;
1012 useTags = usetags;
1013 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1014 } else {
a2f5fc01 1015 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 1016 Error("CheckInputData", "Some conflict was found in the types of inputs");
1017 return kFALSE;
1018 }
1019 }
1020 }
1021 }
1022 // Process requested run numbers
a8739e8a 1023 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 1024 // Check validity of alien data directory
1025 if (!fGridDataDir.Length()) {
1026 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
1027 return kFALSE;
1028 }
923e2ca5 1029 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 1030 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
1031 return kFALSE;
1032 }
a2f5fc01 1033 if (isCollection) {
c57f56b7 1034 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
1035 return kFALSE;
1036 }
1037
a2f5fc01 1038 if (checked && !isXml) {
c57f56b7 1039 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
1040 return kFALSE;
1041 }
1042 // Check validity of run number(s)
1043 TObjArray *arr;
1044 TObjString *os;
74013535 1045 TString format;
319593fb 1046 Int_t nruns = 0;
904f9f5f 1047 TString schunk, schunk2;
c57f56b7 1048 TString path;
1049 if (!checked) {
1050 checked = kTRUE;
a2f5fc01 1051 useTags = fDataPattern.Contains("tag");
1052 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1053 }
a2f5fc01 1054 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 1055 Error("CheckInputData", "Cannot mix input files using/not using tags");
1056 return kFALSE;
1057 }
1058 if (fRunNumbers.Length()) {
a8739e8a 1059 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 1060 arr = fRunNumbers.Tokenize(" ");
1061 TIter next(arr);
1062 while ((os=(TObjString*)next())) {
1063 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1064 if (!DirectoryExists(path)) {
1065 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 1066 continue;
c57f56b7 1067 }
1068 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
1069 TString msg = "\n##### file: ";
1070 msg += path;
1071 msg += " type: xml_collection;";
a2f5fc01 1072 if (useTags) msg += " using_tags: Yes";
c57f56b7 1073 else msg += " using_tags: No";
23329835 1074 Info("CheckDataType", "%s", msg.Data());
319593fb 1075 if (fNrunsPerMaster<2) {
d2a409b2 1076 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 1077 } else {
1078 nruns++;
1079 if (((nruns-1)%fNrunsPerMaster) == 0) {
1080 schunk = os->GetString();
1081 }
1082 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1083 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1084 AddDataFile(schunk);
319593fb 1085 }
c57f56b7 1086 }
1087 delete arr;
a8739e8a 1088 } else {
1089 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1090 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1091 format = Form("%%s/%s ", fRunPrefix.Data());
1092 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1093 if (!DirectoryExists(path)) {
a8739e8a 1094 continue;
1095 }
74013535 1096 format = Form("%%s/%s.xml", fRunPrefix.Data());
1097 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1098 TString msg = "\n##### file: ";
1099 msg += path;
1100 msg += " type: xml_collection;";
a2f5fc01 1101 if (useTags) msg += " using_tags: Yes";
a8739e8a 1102 else msg += " using_tags: No";
23329835 1103 Info("CheckDataType", "%s", msg.Data());
319593fb 1104 if (fNrunsPerMaster<2) {
74013535 1105 format = Form("%s.xml", fRunPrefix.Data());
1106 AddDataFile(Form(format.Data(),irun));
319593fb 1107 } else {
1108 nruns++;
1109 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1110 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1111 }
74013535 1112 format = Form("_%s.xml", fRunPrefix.Data());
1113 schunk2 = Form(format.Data(), irun);
319593fb 1114 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1115 schunk += schunk2;
d2a409b2 1116 AddDataFile(schunk);
319593fb 1117 }
a8739e8a 1118 }
904f9f5f 1119 if (!fInputFiles) {
1120 schunk += schunk2;
1121 AddDataFile(schunk);
1122 }
c57f56b7 1123 }
1124 return kTRUE;
1125}
1126
21ca8e59 1127//______________________________________________________________________________
4a8b667c 1128Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1129{
1130// Copy data from the given grid directory according a pattern and make a local
1131// dataset.
2ba4c287 1132// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1133 if (!Connect()) {
1134 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
4a8b667c 1135 return 0;
21ca8e59 1136 }
1137 if (!DirectoryExists(griddir)) {
1138 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
4a8b667c 1139 return 0;
21ca8e59 1140 }
1141 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1142 printf("Running command: %s\n", command.Data());
1143 TGridResult *res = gGrid->Command(command);
1144 Int_t nfound = res->GetEntries();
1145 if (!nfound) {
1146 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
4a8b667c 1147 return 0;
21ca8e59 1148 }
1149 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1150
1151 // archives
1152 TObjArray* additionalArchives = 0;
1153 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1154 additionalArchives = TString(archivefile).Tokenize(";");
1155 archivefile = additionalArchives->At(0)->GetName();
1156 additionalArchives->RemoveAt(0);
1157 additionalArchives->Compress();
1158 }
1159
21ca8e59 1160 // Copy files locally
1161 ofstream out;
1162 out.open(output, ios::out);
1163 TMap *map;
1164 TString turl, dirname, filename, temp;
1165 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1166 gSystem->MakeDirectory(outputdir);
1167 gSystem->ChangeDirectory(outputdir);
4a8b667c 1168 Int_t ncopied = 0;
21ca8e59 1169 for (Int_t i=0; i<nfound; i++) {
1170 map = (TMap*)res->At(i);
1171 turl = map->GetValue("turl")->GetName();
1172 filename = gSystem->BaseName(turl.Data());
1173 dirname = gSystem->DirName(turl.Data());
1174 dirname = gSystem->BaseName(dirname.Data());
1175 gSystem->MakeDirectory(dirname);
43b807f6 1176
1177 TString source(turl);
1178 TString targetFileName(filename);
1179
1180 if (strlen(archivefile) > 0) {
4a8b667c 1181// TODO here the archive in which the file resides should be determined
1182// however whereis returns only a guid, and guid2lfn does not work
1183// Therefore we use the one provided as argument for now
1184 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1185 targetFileName = archivefile;
43b807f6 1186 }
1187 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
4a8b667c 1188 Bool_t success = kTRUE;
1189 if (additionalArchives) {
1190 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1191 TString target;
1192 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1193 gSystem->MakeDirectory(gSystem->DirName(target));
1194 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1195 }
1196 }
2ba4c287 1197
4a8b667c 1198 if (success) {
1199 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1200 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1201 ncopied++;
1202 }
21ca8e59 1203 }
1204 }
1205 gSystem->ChangeDirectory(cdir);
1206 delete res;
4a8b667c 1207 delete additionalArchives;
1208 return ncopied;
21ca8e59 1209}
1210
c57f56b7 1211//______________________________________________________________________________
1212Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1213{
1214// Create dataset for the grid data directory + run number.
3af861ff 1215 const Int_t gMaxEntries = 15000;
d3b18c4c 1216 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1217 if (!Connect()) {
1218 Error("CreateDataset", "Cannot create dataset with no grid connection");
1219 return kFALSE;
1220 }
1221
1222 // Cd workspace
d3b18c4c 1223 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1224 TString workdir = gGrid->GetHomeDirectory();
1225 workdir += fGridWorkingDir;
1226
1227 // Compose the 'find' command arguments
74013535 1228 TString format;
c57f56b7 1229 TString command;
780ec36d 1230 TString delimiter = pattern;
1231 delimiter.Strip();
1232 if (delimiter.Contains(" ")) delimiter = "";
1233 else delimiter = " ";
c57f56b7 1234 TString options = "-x collection ";
bb885a9e 1235 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1236 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1237 TString conditions = "";
3af861ff 1238 Int_t nstart = 0;
1239 Int_t ncount = 0;
1240 Int_t stage = 0;
c57f56b7 1241 TString file;
1242 TString path;
319593fb 1243 Int_t nruns = 0;
904f9f5f 1244 TString schunk, schunk2;
ab254fd1 1245 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1246 if (!fRunNumbers.Length() && !fRunRange[0]) {
1247 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1248 // Make a single data collection from data directory.
1249 path = fGridDataDir;
923e2ca5 1250 if (!DirectoryExists(path)) {
d2a409b2 1251 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1252 return kFALSE;
780ec36d 1253 }
923e2ca5 1254// CdWork();
d2a409b2 1255 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1256 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1257 while (1) {
1258 ncount = 0;
1259 stage++;
1260 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1261 command = "find ";
1262 command += Form("%s -o %d ",options.Data(), nstart);
1263 command += path;
780ec36d 1264 command += delimiter;
3af861ff 1265 command += pattern;
1266 command += conditions;
1267 printf("command: %s\n", command.Data());
1268 TGridResult *res = gGrid->Command(command);
1269 if (res) delete res;
1270 // Write standard output to file
1271 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1272 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1273 Bool_t nullFile = kFALSE;
1274 if (!hasGrep) {
1275 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1276 } else {
1277 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1278 if (nullFile) {
1279 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1280 gSystem->Exec("rm -f __tmp*");
1281 return kFALSE;
1282 }
1283 TString line;
1284 ifstream in;
1285 in.open("__tmp__");
1286 in >> line;
1287 in.close();
1288 gSystem->Exec("rm -f __tmp__");
1289 ncount = line.Atoi();
1290 }
1291 }
1292 if (ncount == gMaxEntries) {
1293 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1294 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1295 if (!cbase) cbase = cadd;
1296 else {
1297 cbase->Add(cadd);
1298 delete cadd;
defd7a3a 1299 }
3af861ff 1300 nstart += ncount;
1301 } else {
1302 if (cbase) {
1303 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1304 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1305 cbase->Add(cadd);
1306 delete cadd;
1307 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1308 delete cbase; cbase = 0;
1309 } else {
1310 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1311 }
1312 gSystem->Exec("rm -f __tmp*");
1313 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1314 break;
1315 }
0f389141 1316 }
1317 Bool_t fileExists = FileExists(file);
1318 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1319 // Copy xml file to alien space
0f389141 1320 if (fileExists) gGrid->Rm(file);
d2a409b2 1321 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1322 if (!FileExists(file)) {
1323 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1324 return kFALSE;
1325 }
1326 // Update list of files to be processed.
1327 }
1328 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1329 return kTRUE;
1330 }
c57f56b7 1331 // Several runs
a2f5fc01 1332 Bool_t nullResult = kTRUE;
a8739e8a 1333 if (fRunNumbers.Length()) {
1334 TObjArray *arr = fRunNumbers.Tokenize(" ");
1335 TObjString *os;
1336 TIter next(arr);
1337 while ((os=(TObjString*)next())) {
3af861ff 1338 nstart = 0;
1339 stage = 0;
0dab9d6d 1340 path = Form("%s/%s", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1341 if (!DirectoryExists(path)) continue;
1342// CdWork();
a8739e8a 1343 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1344 else file = Form("%s.xml", os->GetString().Data());
319593fb 1345 // If local collection file does not exist, create it via 'find' command.
3af861ff 1346 while (1) {
1347 ncount = 0;
1348 stage++;
1349 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1350 command = "find ";
1351 command += Form("%s -o %d ",options.Data(), nstart);
1352 command += path;
780ec36d 1353 command += delimiter;
3af861ff 1354 command += pattern;
1355 command += conditions;
1356 TGridResult *res = gGrid->Command(command);
1357 if (res) delete res;
1358 // Write standard output to file
1359 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1360 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1361 Bool_t nullFile = kFALSE;
1362 if (!hasGrep) {
1363 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1364 } else {
1365 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1366 if (nullFile) {
1367 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1368 gSystem->Exec("rm -f __tmp*");
1369 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1370 break;
3af861ff 1371 }
1372 TString line;
1373 ifstream in;
1374 in.open("__tmp__");
1375 in >> line;
1376 in.close();
1377 gSystem->Exec("rm -f __tmp__");
1378 ncount = line.Atoi();
1379 }
1380 nullResult = kFALSE;
1381 }
1382 if (ncount == gMaxEntries) {
1383 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1384 if (fNrunsPerMaster > 1) {
1385 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1386 file.Data(),gMaxEntries);
1387 return kFALSE;
1388 }
1389 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1390 if (!cbase) cbase = cadd;
1391 else {
1392 cbase->Add(cadd);
1393 delete cadd;
defd7a3a 1394 }
3af861ff 1395 nstart += ncount;
1396 } else {
c89a84bc 1397 if (cbase && fNrunsPerMaster<2) {
3af861ff 1398 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1399 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1400 cbase->Add(cadd);
1401 delete cadd;
1402 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1403 delete cbase; cbase = 0;
1404 } else {
1405 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1406 }
1407 gSystem->Exec("rm -f __tmp*");
1408 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1409 break;
d3339be3 1410 }
3af861ff 1411 }
a8739e8a 1412 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1413 // Check if there is one run per master job.
1414 if (fNrunsPerMaster<2) {
1415 if (FileExists(file)) {
0f389141 1416 if (fOverwriteMode) gGrid->Rm(file);
1417 else {
1418 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1419 continue;
1420 }
319593fb 1421 }
1422 // Copy xml file to alien space
1423 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1424 if (!FileExists(file)) {
1425 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1426 delete arr;
1427 return kFALSE;
1428 }
1429 } else {
1430 nruns++;
1431 if (((nruns-1)%fNrunsPerMaster) == 0) {
1432 schunk = os->GetString();
1433 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1434 } else {
1435 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1436 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1437 cbase->Add(cadd);
1438 delete cadd;
1439 }
1440 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1441 continue;
1442 }
1443 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1444 if (FileExists(schunk)) {
1445 if (fOverwriteMode) gGrid->Rm(file);
1446 else {
1447 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1448 continue;
1449 }
319593fb 1450 }
84fcd93f 1451 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1452 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1453 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1454 if (!FileExists(schunk)) {
1455 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1456 delete arr;
1457 return kFALSE;
1458 }
d3339be3 1459 }
a8739e8a 1460 }
1461 delete arr;
a2f5fc01 1462 if (nullResult) {
d3339be3 1463 Error("CreateDataset", "No valid dataset corresponding to the query!");
1464 return kFALSE;
1465 }
a8739e8a 1466 } else {
1467 // Process a full run range.
1468 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
0dab9d6d 1469 format = Form("%%s/%s", fRunPrefix.Data());
3af861ff 1470 nstart = 0;
1471 stage = 0;
74013535 1472 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1473 if (!DirectoryExists(path)) continue;
1474// CdWork();
74013535 1475 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1476 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1477 else file = Form(format.Data(), irun);
0f389141 1478 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1479 if (fOverwriteMode) gGrid->Rm(file);
1480 else {
1481 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1482 continue;
1483 }
a8739e8a 1484 }
319593fb 1485 // If local collection file does not exist, create it via 'find' command.
3af861ff 1486 while (1) {
1487 ncount = 0;
1488 stage++;
1489 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1490 command = "find ";
1491 command += Form("%s -o %d ",options.Data(), nstart);
1492 command += path;
780ec36d 1493 command += delimiter;
3af861ff 1494 command += pattern;
1495 command += conditions;
1496 TGridResult *res = gGrid->Command(command);
1497 if (res) delete res;
1498 // Write standard output to file
1499 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1500 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1501 Bool_t nullFile = kFALSE;
1502 if (!hasGrep) {
1503 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1504 } else {
1505 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1506 if (nullFile) {
1507 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1508 gSystem->Exec("rm -f __tmp*");
1604c9ed 1509 break;
3af861ff 1510 }
1511 TString line;
1512 ifstream in;
1513 in.open("__tmp__");
1514 in >> line;
1515 in.close();
1516 gSystem->Exec("rm -f __tmp__");
1517 ncount = line.Atoi();
1518 }
1519 nullResult = kFALSE;
1520 }
1521 if (ncount == gMaxEntries) {
1522 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1523 if (fNrunsPerMaster > 1) {
1524 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1525 file.Data(),gMaxEntries);
1526 return kFALSE;
1527 }
1528 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1529 if (!cbase) cbase = cadd;
1530 else {
1531 cbase->Add(cadd);
1532 delete cadd;
defd7a3a 1533 }
3af861ff 1534 nstart += ncount;
1535 } else {
c89a84bc 1536 if (cbase && fNrunsPerMaster<2) {
3af861ff 1537 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1538 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1539 cbase->Add(cadd);
1540 delete cadd;
1541 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1542 delete cbase; cbase = 0;
1543 } else {
1544 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1545 }
1546 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1547 break;
d3339be3 1548 }
319593fb 1549 }
a8739e8a 1550 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1551 // Check if there is one run per master job.
1552 if (fNrunsPerMaster<2) {
1553 if (FileExists(file)) {
0f389141 1554 if (fOverwriteMode) gGrid->Rm(file);
1555 else {
1556 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1557 continue;
1558 }
319593fb 1559 }
1560 // Copy xml file to alien space
1561 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1562 if (!FileExists(file)) {
1563 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1564 return kFALSE;
1565 }
1566 } else {
1567 nruns++;
95e5b448 1568 // Check if the collection for the chunk exist locally.
1569 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1570 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1571 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1572 else continue;
1573 }
84fcd93f 1574 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1575 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1576 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1577 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1578 } else {
1579 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1580 cbase->Add(cadd);
1581 delete cadd;
1582 }
74013535 1583 format = Form("%%s_%s.xml", fRunPrefix.Data());
1584 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1585 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1586 continue;
1587 }
904f9f5f 1588 schunk = schunk2;
319593fb 1589 if (FileExists(schunk)) {
0f389141 1590 if (fOverwriteMode) gGrid->Rm(schunk);
1591 else {
1592 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1593 continue;
1594 }
319593fb 1595 }
84fcd93f 1596 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1597 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1598 if (FileExists(schunk)) {
0f389141 1599 if (fOverwriteMode) gGrid->Rm(schunk);
1600 else {
1601 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1602 continue;
1603 }
95e5b448 1604 }
319593fb 1605 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1606 if (!FileExists(schunk)) {
1607 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1608 return kFALSE;
1609 }
1610 }
c57f56b7 1611 }
a2f5fc01 1612 if (nullResult) {
d3339be3 1613 Error("CreateDataset", "No valid dataset corresponding to the query!");
1614 return kFALSE;
1615 }
a8739e8a 1616 }
c57f56b7 1617 return kTRUE;
1618}
1619
1620//______________________________________________________________________________
1621Bool_t AliAnalysisAlien::CreateJDL()
1622{
1623// Generate a JDL file according to current settings. The name of the file is
1624// specified by fJDLName.
ec5acdce 1625 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
c57f56b7 1626 Bool_t error = kFALSE;
1627 TObjArray *arr = 0;
1628 Bool_t copy = kTRUE;
d3b18c4c 1629 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1630 Bool_t generate = kTRUE;
1631 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1632 if (!Connect()) {
1633 Error("CreateJDL", "Alien connection required");
1634 return kFALSE;
1635 }
1636 // Check validity of alien workspace
d3b18c4c 1637 TString workdir;
1638 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1639 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1640 workdir += fGridWorkingDir;
1641 if (generate) {
1642 TObjString *os;
1e466628 1643 if (!fInputFiles && !fMCLoop) {
c57f56b7 1644 Error("CreateJDL()", "Define some input files for your analysis.");
1645 error = kTRUE;
1646 }
1647 // Compose list of input files
1648 // Check if output files were defined
1649 if (!fOutputFiles.Length()) {
1650 Error("CreateJDL", "You must define at least one output file");
1651 error = kTRUE;
1652 }
1653 // Check if an output directory was defined and valid
1654 if (!fGridOutputDir.Length()) {
1655 Error("CreateJDL", "You must define AliEn output directory");
1656 error = kTRUE;
1657 } else {
d3b18c4c 1658 if (!fProductionMode) {
1659 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1660 if (!DirectoryExists(fGridOutputDir)) {
1661 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1662 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1663 } else {
1664 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1665 // error = kTRUE;
1666 }
67a3b254 1667 } else {
1668 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1669 }
1670 gGrid->Cd(workdir);
1671 }
c57f56b7 1672 }
1673 // Exit if any error up to now
1674 if (error) return kFALSE;
1675 // Set JDL fields
0f389141 1676 if (!fUser.IsNull()) {
1677 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1678 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
6da4839e
AG		 1679 }
1680 TString executable = fExecutable;
1681 if (!executable.BeginsWith("/"))
1682 executable.Prepend(Form("%s/", workdir.Data()));
1683 fGridJDL->SetExecutable(executable, "This is the startup script");
1684 TString mergeExec = executable;
0f389141 1685 mergeExec.ReplaceAll(".sh", "_merge.sh");
1686 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1687 mergeExec.ReplaceAll(".sh", ".C");
9c5cde18 1688 fMergingJDL->AddToInputSandbox(Form("LF:%s", mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1689 if (!fArguments.IsNull())
1690 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1691 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1692 else {
1693 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1694 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1695 }
b3e07543 1696
1f0d1ca2 1697 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1698 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1699 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1700 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1701
0f389141 1702 if (fMaxInitFailed > 0) {
c57f56b7 1703 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1704 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1705 }
6da4839e 1706 if (fSplitMaxInputFileNumber > 0 && !fMCLoop) {
c57f56b7 1707 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1708 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1709 }
d7eb1f39 1710 if (!IsOneStageMerging()) {
f47d5cb4 1711 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1712 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1713 }
1714 if (fSplitMode.Length()) {
c57f56b7 1715 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1716 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1717 }
1718 fMergingJDL->SetValue("Split", "\"se\"");
1719 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1720 if (!fAliROOTVersion.IsNull()) {
1721 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1722 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1723 }
1724 if (!fROOTVersion.IsNull()) {
c57f56b7 1725 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1726 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1727 }
1728 if (!fAPIVersion.IsNull()) {
c57f56b7 1729 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1730 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1731 }
648174cf 1732 if (!fExternalPackages.IsNull()) {
1733 arr = fExternalPackages.Tokenize(" ");
1734 TIter next(arr);
1735 while ((os=(TObjString*)next())) {
1736 TString pkgname = os->GetString();
1737 Int_t index = pkgname.Index("::");
1738 TString pkgversion = pkgname(index+2, pkgname.Length());
1739 pkgname.Remove(index);
1740 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1741 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1742 }
1743 delete arr;
1744 }
6da4839e
AG		 1745 if (!fMCLoop) {
1746 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1747 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
6da4839e 1748 }
d7eb1f39
AG		 1749 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1750 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1751 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1752 TString analysisFile = fExecutable;
1753 analysisFile.ReplaceAll(".sh", ".root");
1754 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1755 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1756 if (fAdditionalLibs.Length()) {
1757 arr = fAdditionalLibs.Tokenize(" ");
1758 TIter next(arr);
1759 while ((os=(TObjString*)next())) {
1760 if (os->GetString().Contains(".so")) continue;
1761 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1762 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1763 }
1764 delete arr;
1765 }
4e5c5506 1766 if (fPackages) {
1767 TIter next(fPackages);
1768 TObject *obj;
0f389141 1769 while ((obj=next())) {
4e5c5506 1770 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1771 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1772 }
4e5c5506 1773 }
95e16f01 1774 const char *comment = "List of output files and archives";
c57f56b7 1775 if (fOutputArchive.Length()) {
e4d9a069 1776 TString outputArchive = fOutputArchive;
1777 if (!fRegisterExcludes.IsNull()) {
1778 arr = fRegisterExcludes.Tokenize(" ");
1779 TIter next1(arr);
1780 while ((os=(TObjString*)next1())) {
1781 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1782 outputArchive.ReplaceAll(os->GetString(),"");
1783 }
1784 delete arr;
1785 }
1786 arr = outputArchive.Tokenize(" ");
c57f56b7 1787 TIter next(arr);
0f389141 1788 Bool_t first = kTRUE;
0f389141 1789 while ((os=(TObjString*)next())) {
0f389141 1790 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1791 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
0f389141 1792 else
95e16f01 1793 fGridJDL->AddToSet("Output", os->GetString());
1794 if (first) fGridJDL->AddToSetDescription("Output", comment);
0f389141 1795 first = kFALSE;
1796 }
c57f56b7 1797 delete arr;
f790bc1b 1798 // Output archive for the merging jdl
f790bc1b 1799 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1800 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1801 // Add normal output files, extra files + terminate files
0c0825e4 1802 TString files;
1803 if (IsMergeAOD()) files = GetListOfFiles("outaodextter");
1804 else files = GetListOfFiles("outextter");
37d21c01 1805 // Do not register files in fRegisterExcludes
1806 if (!fRegisterExcludes.IsNull()) {
1807 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1808 TIter next1(arr);
1809 while ((os=(TObjString*)next1())) {
1810 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1811 files.ReplaceAll(os->GetString(),"");
1812 }
66d71516 1813 delete arr;
0f389141 1814 }
f790bc1b 1815 files.ReplaceAll(".root", "*.root");
ec5acdce 1816
1817 if (mgr->IsCollectThroughput())
95e16f01 1818 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",files.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 1819 else
1820 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1821 } else {
95f84779 1822 TString files = fOutputArchive;
1823 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1824 outputArchive = files;
f790bc1b 1825 }
0f389141 1826 arr = outputArchive.Tokenize(" ");
705adb3e 1827 TIter next2(arr);
0f389141 1828 first = kTRUE;
705adb3e 1829 while ((os=(TObjString*)next2())) {
0cdf65a8 1830 TString currentfile = os->GetString();
0cdf65a8 1831 if (!currentfile.Contains("@") && fCloseSE.Length())
95e16f01 1832 fMergingJDL->AddToSet("Output", Form("%s@%s",currentfile.Data(), fCloseSE.Data()));
0f389141 1833 else
95e16f01 1834 fMergingJDL->AddToSet("Output", currentfile);
1835 if (first) fMergingJDL->AddToSetDescription("Output", comment);
0f389141 1836 first = kFALSE;
1837 }
1838 delete arr;
c57f56b7 1839 }
149d288c 1840 arr = fOutputFiles.Tokenize(",");
c57f56b7 1841 TIter next(arr);
0f389141 1842 Bool_t first = kTRUE;
43da816a 1843 while ((os=(TObjString*)next())) {
e1eaf596 1844 // Ignore ouputs in jdl that are also in outputarchive
1845 TString sout = os->GetString();
f790bc1b 1846 sout.ReplaceAll("*", "");
1847 sout.ReplaceAll(".root", "");
e1eaf596 1848 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1849 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1850 // Ignore fRegisterExcludes
1851 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1852 if (!first) comment = NULL;
43da816a 1853 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1854 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
43da816a 1855 else
95e16f01 1856 fGridJDL->AddToSet("Output", os->GetString());
1857 if (first) fGridJDL->AddToSetDescription("Output", comment);
f790bc1b 1858 if (fMergeExcludes.Contains(sout)) continue;
1859 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1860 fMergingJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
f790bc1b 1861 else
95e16f01 1862 fMergingJDL->AddToSet("Output", os->GetString());
1863 if (first) fMergingJDL->AddToSetDescription("Output", comment);
1864 first = kFALSE;
43da816a 1865 }
c57f56b7 1866 delete arr;
0f389141 1867 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1868 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1869 TString validationScript = fValidationScript;
0f389141 1870 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1871 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1872 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1873 if (fMasterResubmitThreshold) {
1874 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1875 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1876 }
d2a409b2 1877 // Write a jdl with 2 input parameters: collection name and output dir name.
1878 WriteJDL(copy);
c57f56b7 1879 }
1880 // Copy jdl to grid workspace
a8739e8a 1881 if (copy) {
b5e4aaa7 1882 // Check if an output directory was defined and valid
1883 if (!fGridOutputDir.Length()) {
1884 Error("CreateJDL", "You must define AliEn output directory");
1885 return kFALSE;
1886 } else {
1887 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1888 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1889 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1890 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1891 } else {
1892 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1893 return kFALSE;
1894 }
1895 }
1896 gGrid->Cd(workdir);
1897 }
648174cf 1898 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1899 TString mergeJDLName = fExecutable;
1900 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1901 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1902 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1903 if (fProductionMode) {
648174cf 1904 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1905 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1906 }
648174cf 1907 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1908 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1909 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1910 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1911 Fatal("","Terminating");
1912// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1913 if (fMergeViaJDL) {
1914 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1915// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1916 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1917 Fatal("","Terminating");
0f389141 1918 }
648174cf 1919 }
c57f56b7 1920 if (fAdditionalLibs.Length()) {
e7c71df0 1921 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1922 TObjString *os;
1923 TIter next(arr);
1924 while ((os=(TObjString*)next())) {
c57f56b7 1925 if (os->GetString().Contains(".so")) continue;
4e5c5506 1926 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1927 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1928// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1929 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1930 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1931 Fatal("","Terminating");
c57f56b7 1932 }
1933 delete arr;
1934 }
4e5c5506 1935 if (fPackages) {
1936 TIter next(fPackages);
1937 TObject *obj;
1938 while ((obj=next())) {
fdbbc7be 1939 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1940 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1941// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1942 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1943 Form("%s/%s", workdir.Data(), obj->GetName())))
1944 Fatal("","Terminating");
4e5c5506 1945 }
1946 }
c57f56b7 1947 }
1948 return kTRUE;
1949}
1950
a8739e8a 1951//______________________________________________________________________________
d2a409b2 1952Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1953{
1954// Writes one or more JDL's corresponding to findex. If findex is negative,
1955// all run numbers are considered in one go (jdl). For non-negative indices
1956// they correspond to the indices in the array fInputFiles.
dec33edc 1957 if (!fInputFiles && !fMCLoop) return kFALSE;
d3b18c4c 1958 TObject *os;
1959 TString workdir;
1960 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1961 workdir += fGridWorkingDir;
f47d5cb4 1962 TString stageName = "$2";
1963 if (fProductionMode) stageName = "$4";
1964 if (!fMergeDirName.IsNull()) {
1965 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1966 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1967 } else {
1968 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1969 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1970 }
d3b18c4c 1971 if (fProductionMode) {
a8739e8a 1972 TIter next(fInputFiles);
b3e07543 1973 while ((os=next())) {
d3b18c4c 1974 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1975 }
56754bf6 1976 if (!fOutputToRunNo)
1977 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1978 else
1979 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1980 } else {
1981 if (!fRunNumbers.Length() && !fRunRange[0]) {
1982 // One jdl with no parameters in case input data is specified by name.
1983 TIter next(fInputFiles);
1984 while ((os=next()))
1985 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1986 if (!fOutputSingle.IsNull())
1987 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1988 else {
1989 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
b1dade98 1990// fMergingJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1991 }
1992 } else {
1993 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1994 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1995 if (!fOutputSingle.IsNull()) {
1996 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1997 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1998 } else {
1999 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 2000 }
2001 }
a8739e8a 2002 }
2003
a8739e8a 2004 // Generate the JDL as a string
2005 TString sjdl = fGridJDL->Generate();
0f389141 2006 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 2007 // Final merge jdl
2008 if (!fMergeDirName.IsNull()) {
2009 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
2010 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
2011 } else {
2012 fMergingJDL->SetOutputDirectory("$1", "Output directory");
2013 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
2014 }
b3e07543 2015 TString sjdl2 = fMergingJDL->Generate();
2016 Int_t index, index1;
95e16f01 2017 sjdl.ReplaceAll("\",\"", "\",\n \"");
a8739e8a 2018 sjdl.ReplaceAll("(member", "\n (member");
2019 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2020 sjdl.ReplaceAll("{", "{\n ");
2021 sjdl.ReplaceAll("};", "\n};");
2022 sjdl.ReplaceAll("{\n \n", "{\n");
2023 sjdl.ReplaceAll("\n\n", "\n");
2024 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2025 sjdl1.ReplaceAll("\",\"", "\",\n \"");
0f389141 2026 sjdl1.ReplaceAll("(member", "\n (member");
2027 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2028 sjdl1.ReplaceAll("{", "{\n ");
2029 sjdl1.ReplaceAll("};", "\n};");
2030 sjdl1.ReplaceAll("{\n \n", "{\n");
2031 sjdl1.ReplaceAll("\n\n", "\n");
2032 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2033 sjdl2.ReplaceAll("\",\"", "\",\n \"");
b3e07543 2034 sjdl2.ReplaceAll("(member", "\n (member");
2035 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2036 sjdl2.ReplaceAll("{", "{\n ");
2037 sjdl2.ReplaceAll("};", "\n};");
2038 sjdl2.ReplaceAll("{\n \n", "{\n");
2039 sjdl2.ReplaceAll("\n\n", "\n");
2040 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 2041 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 2042 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 2043 index = sjdl.Index("JDLVariables");
2044 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 2045 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2046 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 2047 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 2048 index = fJobTag.Index(":");
2049 if (index < 0) index = fJobTag.Length();
2050 TString jobTag = fJobTag;
f47d5cb4 2051 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 2052 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 2053 if (fProductionMode) {
2054 sjdl1.Prepend("# Generated merging jdl (production mode) \
2055 \n# $1 = full alien path to output directory to be merged \
2056 \n# $2 = train number \
2057 \n# $3 = production (like LHC10b) \
2058 \n# $4 = merging stage \
2059 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2060 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2061 sjdl2.Prepend("# Generated merging jdl \
2062 \n# $1 = full alien path to output directory to be merged \
2063 \n# $2 = train number \
2064 \n# $3 = production (like LHC10b) \
2065 \n# $4 = merging stage \
2066 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2067 } else {
2068 sjdl1.Prepend("# Generated merging jdl \
2069 \n# $1 = full alien path to output directory to be merged \
2070 \n# $2 = merging stage \
03335324 2071 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2072 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2073 sjdl2.Prepend("# Generated merging jdl \
2074 \n# $1 = full alien path to output directory to be merged \
2075 \n# $2 = merging stage \
03335324 2076 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2077 }
0f389141 2078 index = sjdl1.Index("JDLVariables");
2079 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 2080 index = sjdl2.Index("JDLVariables");
2081 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 2082 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2083 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
2084 index = sjdl2.Index("Split =");
2085 if (index>=0) {
2086 index1 = sjdl2.Index("\n", index);
2087 sjdl2.Remove(index, index1-index+1);
2088 }
2089 index = sjdl2.Index("SplitMaxInputFileNumber");
2090 if (index>=0) {
2091 index1 = sjdl2.Index("\n", index);
2092 sjdl2.Remove(index, index1-index+1);
2093 }
2094 index = sjdl2.Index("InputDataCollection");
2095 if (index>=0) {
2096 index1 = sjdl2.Index(";", index);
2097 sjdl2.Remove(index, index1-index+1);
2098 }
2099 index = sjdl2.Index("InputDataListFormat");
2100 if (index>=0) {
2101 index1 = sjdl2.Index("\n", index);
2102 sjdl2.Remove(index, index1-index+1);
2103 }
2104 index = sjdl2.Index("InputDataList");
2105 if (index>=0) {
2106 index1 = sjdl2.Index("\n", index);
2107 sjdl2.Remove(index, index1-index+1);
2108 }
f47d5cb4 2109 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2110 // Write jdl to file
a8739e8a 2111 ofstream out;
d2a409b2 2112 out.open(fJDLName.Data(), ios::out);
a8739e8a 2113 if (out.bad()) {
d3b18c4c 2114 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2115 return kFALSE;
2116 }
2117 out << sjdl << endl;
b3e07543 2118 out.close();
0f389141 2119 TString mergeJDLName = fExecutable;
2120 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2121 if (fMergeViaJDL) {
2122 ofstream out1;
2123 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2124 if (out1.bad()) {
d3b18c4c 2125 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2126 return kFALSE;
2127 }
2128 out1 << sjdl1 << endl;
b3e07543 2129 out1.close();
2130 ofstream out2;
2131 TString finalJDL = mergeJDLName;
2132 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2133 out2.open(finalJDL.Data(), ios::out);
2134 if (out2.bad()) {
2135 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2136 return kFALSE;
2137 }
2138 out2 << sjdl2 << endl;
2139 out2.close();
0f389141 2140 }
a8739e8a 2141
2142 // Copy jdl to grid workspace
2143 if (!copy) {
d3b18c4c 2144 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2145 } else {
d2a409b2 2146 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2147 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2148 TString finalJDL = mergeJDLName;
2149 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2150 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2151 if (fProductionMode) {
b5fe9cba 2152 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2153 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2154 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2155 }
d2a409b2 2156 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2157 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2158 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2159 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2160// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2161 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2162 Fatal("","Terminating");
0f389141 2163 if (fMergeViaJDL) {
b3e07543 2164 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2165// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2166// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2167 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2168 Fatal("","Terminating");
2169 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2170 Fatal("","Terminating");
0f389141 2171 }
a8739e8a 2172 }
2173 return kTRUE;
2174}
2175
c57f56b7 2176//______________________________________________________________________________
5513444a 2177Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2178{
2179// Returns true if file exists.
5513444a 2180 if (!gGrid) return kFALSE;
2ea55496 2181 TString slfn = lfn;
2182 slfn.ReplaceAll("alien://","");
2183 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2184 if (!res) return kFALSE;
2185 TMap *map = dynamic_cast<TMap*>(res->At(0));
2186 if (!map) {
2187 delete res;
2188 return kFALSE;
2189 }
2190 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2191 if (!objs || !objs->GetString().Length()) {
2192 delete res;
2193 return kFALSE;
2194 }
2195 delete res;
2196 return kTRUE;
2197}
2198
923e2ca5 2199//______________________________________________________________________________
2200Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2201{
2202// Returns true if directory exists. Can be also a path.
2203 if (!gGrid) return kFALSE;
2204 // Check if dirname is a path
2205 TString dirstripped = dirname;
2206 dirstripped = dirstripped.Strip();
2207 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2208 TString dir = gSystem->BaseName(dirstripped);
2209 dir += "/";
2210 TString path = gSystem->DirName(dirstripped);
2211 TGridResult *res = gGrid->Ls(path, "-F");
2212 if (!res) return kFALSE;
2213 TIter next(res);
2214 TMap *map;
2215 TObject *obj;
2216 while ((map=dynamic_cast<TMap*>(next()))) {
2217 obj = map->GetValue("name");
2218 if (!obj) break;
2219 if (dir == obj->GetName()) {
2220 delete res;
2221 return kTRUE;
2222 }
2223 }
2224 delete res;
2225 return kFALSE;
2226}
2227
c57f56b7 2228//______________________________________________________________________________
a2f5fc01 2229void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2230{
2231// Check input data type.
a2f5fc01 2232 isCollection = kFALSE;
2233 isXml = kFALSE;
2234 useTags = kFALSE;
c57f56b7 2235 if (!gGrid) {
2236 Error("CheckDataType", "No connection to grid");
2237 return;
2238 }
a2f5fc01 2239 isCollection = IsCollection(lfn);
c57f56b7 2240 TString msg = "\n##### file: ";
2241 msg += lfn;
a2f5fc01 2242 if (isCollection) {
c57f56b7 2243 msg += " type: raw_collection;";
2244 // special treatment for collections
a2f5fc01 2245 isXml = kFALSE;
c57f56b7 2246 // check for tag files in the collection
2247 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2248 if (!res) {
2249 msg += " using_tags: No (unknown)";
23329835 2250 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2251 return;
2252 }
2253 const char* typeStr = res->GetKey(0, "origLFN");
2254 if (!typeStr || !strlen(typeStr)) {
2255 msg += " using_tags: No (unknown)";
23329835 2256 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2257 return;
2258 }
2259 TString file = typeStr;
a2f5fc01 2260 useTags = file.Contains(".tag");
2261 if (useTags) msg += " using_tags: Yes";
c57f56b7 2262 else msg += " using_tags: No";
23329835 2263 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2264 return;
2265 }
2266 TString slfn(lfn);
2267 slfn.ToLower();
a2f5fc01 2268 isXml = slfn.Contains(".xml");
2269 if (isXml) {
c57f56b7 2270 // Open xml collection and check if there are tag files inside
2271 msg += " type: xml_collection;";
2272 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2273 if (!coll) {
2274 msg += " using_tags: No (unknown)";
23329835 2275 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2276 return;
2277 }
2278 TMap *map = coll->Next();
2279 if (!map) {
2280 msg += " using_tags: No (unknown)";
23329835 2281 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2282 return;
2283 }
2284 map = (TMap*)map->GetValue("");
2285 TString file;
2286 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2287 useTags = file.Contains(".tag");
c57f56b7 2288 delete coll;
a2f5fc01 2289 if (useTags) msg += " using_tags: Yes";
c57f56b7 2290 else msg += " using_tags: No";
23329835 2291 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2292 return;
2293 }
a2f5fc01 2294 useTags = slfn.Contains(".tag");
c57f56b7 2295 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2296 else msg += " type: unknown file;";
a2f5fc01 2297 if (useTags) msg += " using_tags: Yes";
c57f56b7 2298 else msg += " using_tags: No";
23329835 2299 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2300}
2301
4e5c5506 2302//______________________________________________________________________________
2303void AliAnalysisAlien::EnablePackage(const char *package)
2304{
2305// Enables a par file supposed to exist in the current directory.
2306 TString pkg(package);
2307 pkg.ReplaceAll(".par", "");
2308 pkg += ".par";
2309 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2310 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2311 return;
2312 }
2313 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2314 Info("EnablePackage", "AliEn plugin will use .par packages");
2315 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2316 if (!fPackages) {
2317 fPackages = new TObjArray();
2318 fPackages->SetOwner();
2319 }
2320 fPackages->Add(new TObjString(pkg));
2321}
2322
3bdcb562 2323//______________________________________________________________________________
2324TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2325{
2326// Make a tree from files having the location specified in fFileForTestMode.
2327// Inspired from JF's CreateESDChain.
2328 if (fFileForTestMode.IsNull()) {
2329 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2330 return NULL;
2331 }
2332 if (gSystem->AccessPathName(fFileForTestMode)) {
2333 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2334 return NULL;
2335 }
2336 // Open the file
2337 ifstream in;
2338 in.open(fFileForTestMode);
2339 Int_t count = 0;
2340 // Read the input list of files and add them to the chain
b505f1f8 2341 TString line;
d6c60197 2342 TString streeName(treeName);
2343 if (IsUseMCchain()) streeName = "TE";
2344 TChain *chain = new TChain(streeName);
06f67114 2345 TList *friends = new TList();
2346 TChain *cfriend = 0;
2347 if (!fFriendChainName.IsNull()) {
2348 TObjArray *list = fFriendChainName.Tokenize(" ");
2349 TIter next(list);
2350 TObjString *str;
2351 while((str=(TObjString*)next())) {
2352 cfriend = new TChain(streeName, str->GetName());
2353 friends->Add(cfriend);
2354 chain->AddFriend(cfriend);
2355 }
2356 delete list;
2357 }
2358 TString bpath;
2359 TIter nextfriend(friends);
b505f1f8 2360 while (in.good())
2361 {
3bdcb562 2362 in >> line;
ffbe06b9 2363 if (line.IsNull() || line.BeginsWith("#")) continue;
3bdcb562 2364 if (count++ == fNtestFiles) break;
2365 TString esdFile(line);
2366 TFile *file = TFile::Open(esdFile);
b505f1f8 2367 if (file && !file->IsZombie()) {
2368 chain->Add(esdFile);
3bdcb562 2369 file->Close();
b505f1f8 2370 if (!fFriendChainName.IsNull()) {
4b5d9e21 2371 if (esdFile.Index("#") > -1)
2372 esdFile.Remove(esdFile.Index("#"));
06f67114 2373 bpath = gSystem->DirName(esdFile);
2374 bpath += "/";
2375 TString fileFriend;
2376 nextfriend.Reset();
2377 while ((cfriend=(TChain*)nextfriend())) {
2378 fileFriend = bpath;
2379 fileFriend += cfriend->GetTitle();
2380 file = TFile::Open(fileFriend);
2381 if (file && !file->IsZombie()) {
2382 file->Close();
2383 cfriend->Add(fileFriend);
2384 } else {
2385 Fatal("GetChainForTestMode", "Cannot open friend file: %s", fileFriend.Data());
2386 return 0;
2387 }
2388 }
4b5d9e21 2389 }
3bdcb562 2390 } else {
2391 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2392 }
b505f1f8 2393 }
2394 in.close();
2395 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2396 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2397 delete chain;
06f67114 2398 friends->Delete();
2399 delete friends;
3bdcb562 2400 return NULL;
b505f1f8 2401 }
b505f1f8 2402 return chain;
3bdcb562 2403}
2404
d2a409b2 2405//______________________________________________________________________________
2406const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2407{
2408// Get job status for all jobs with jobid>jobidstart.
2409 static char mstatus[20];
2410 mstatus[0] = '\0';
2411 nrunning = 0;
2412 nwaiting = 0;
2413 nerror = 0;
2414 ndone = 0;
2415 TGridJobStatusList *list = gGrid->Ps("");
2416 if (!list) return mstatus;
2417 Int_t nentries = list->GetSize();
2418 TGridJobStatus *status;
2419 Int_t pid;
2420 for (Int_t ijob=0; ijob<nentries; ijob++) {
2421 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2422 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2423 if (pid<jobidstart) continue;
2424 if (pid == lastid) {
8ddad121 2425 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2426 }
2427 switch (status->GetStatus()) {
2428 case TGridJobStatus::kWAITING:
2429 nwaiting++; break;
2430 case TGridJobStatus::kRUNNING:
2431 nrunning++; break;
2432 case TGridJobStatus::kABORTED:
2433 case TGridJobStatus::kFAIL:
2434 case TGridJobStatus::kUNKNOWN:
2435 nerror++; break;
2436 case TGridJobStatus::kDONE:
2437 ndone++;
2438 }
2439 }
2440 list->Delete();
2441 delete list;
2442 return mstatus;
2443}
2444
c57f56b7 2445//______________________________________________________________________________
2446Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2447{
2448// Returns true if file is a collection. Functionality duplicated from
2449// TAlien::Type() because we don't want to directly depend on TAlien.
2450 if (!gGrid) {
2451 Error("IsCollection", "No connection to grid");
2452 return kFALSE;
2453 }
2454 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2455 if (!res) return kFALSE;
2456 const char* typeStr = res->GetKey(0, "type");
2457 if (!typeStr || !strlen(typeStr)) return kFALSE;
2458 if (!strcmp(typeStr, "collection")) return kTRUE;
2459 delete res;
2460 return kFALSE;
2461}
2462
fe2d7fc2 2463//______________________________________________________________________________
2464Bool_t AliAnalysisAlien::IsSingleOutput() const
2465{
2466// Check if single-ouput option is on.
2467 return (!fOutputSingle.IsNull());
2468}
715da79c 2469
2470//______________________________________________________________________________
2471Long64_t AliAnalysisAlien::RunMacroAndExtractLibs(const char* macro, const char *args, TString &libs)
2472{
2473// Tries to run the specified macro and return the libraries that it loads.
2474 TString expname;
2475 if (strlen(macro)) expname = gSystem->ExpandPathName(macro);
2476 if (expname.IsNull() || gSystem->AccessPathName(expname)) {
2477 ::Error("RunMacroAndExtractLibs","Cannot find macro %s in current directory", macro);
2478 return -1;
2479 }
2480 TString oldlibs = gSystem->GetLibraries();
2481 TMacro m(expname);
2482 Int_t error = 0;
2483 Long64_t retval = m.Exec(args, &error);
2484 if (error != TInterpreter::kNoError)
2485 {
2486 ::Error("RunMacroAndExtractLibs", "Macro interpretation %s failed", macro);
2487 return -1;
2488 }
2489 libs = gSystem->GetLibraries();
2490 libs.ReplaceAll(oldlibs, "");
2491 libs.Strip(TString::kLeading);
2492 TObjArray *libTokens = libs.Tokenize(" ");
2493 libs = "";
2494 for (Int_t i=0; i<libTokens->GetEntries(); i++) {
2495 if (!libs.IsNull()) libs += " ";
2496 libs += gSystem->BaseName(libTokens->At(i)->GetName());
2497 }
2498 delete libTokens;
2499 return retval;
2500}
2501
16a4353c 2502//______________________________________________________________________________
2503void AliAnalysisAlien::Print(Option_t *) const
2504{
2505// Print current plugin settings.
84fcd93f 2506 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2507 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2508 if (mgr && mgr->IsProofMode()) {
2509 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2510 if (TestBit(AliAnalysisGrid::kTest))
2511 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2512 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2513 if (!fProofDataSet.IsNull())
2514 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2515 if (fProofReset==1)
2516 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2517 if (fProofReset>1)
2518 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
874de7ea 2519 if (!fROOTVersion.IsNull())
2520 printf("= ROOT version requested________________________ %s\n", fROOTVersion.Data());
d3b18c4c 2521 else
2522 printf("= ROOT version requested________________________ default\n");
2523 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2524 if (!fAliRootMode.IsNull())
2525 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2526 if (fNproofWorkers)
2527 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2528 if (fNproofWorkersPerSlave)
2529 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2530 if (TestSpecialBit(kClearPackages))
2531 printf("= ClearPackages requested...\n");
2532 if (fIncludePath.Data())
2533 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2534 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2535 if (fPackages && fPackages->GetEntries()) {
2536 TIter next(fPackages);
2537 TObject *obj;
2538 TString list;
2539 while ((obj=next())) list += obj->GetName();
2540 printf("= Par files to be used: ________________________ %s\n", list.Data());
2541 }
2542 if (TestSpecialBit(kProofConnectGrid))
2543 printf("= Requested PROOF connection to grid\n");
2544 return;
2545 }
e1c22e21 2546 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2547 if (fOverwriteMode) {
2548 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2549 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2550 }
348be253 2551 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
780ec36d 2552 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO:Print");
84fcd93f 2553 printf("= Production mode:______________________________ %d\n", fProductionMode);
2554 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2555 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2556 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2557 if (fUser.Length())
84fcd93f 2558 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2559 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2560 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
780ec36d 2561 TString basedatadir = fGridDataDir;
2562 TString pattern = fDataPattern;
2563 pattern.Strip();
2564 Int_t ind = pattern.Index(" ");
2565 if (ind>=0) {
2566 basedatadir += "/%run%/";
2567 basedatadir += pattern(0, ind);
2568 pattern = pattern(ind+1, pattern.Length());
2569 }
2570 printf("= Data base directory path requested: __________ %s\n", basedatadir.Data());
2571 printf("= Data search pattern: _________________________ %s\n", pattern.Data());
84fcd93f 2572 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2573 if (fRunNumbers.Length())
84fcd93f 2574 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2575 if (fRunRange[0])
74013535 2576 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2577 if (!fRunRange[0] && !fRunNumbers.Length()) {
2578 TIter next(fInputFiles);
2579 TObject *obj;
2580 TString list;
2581 while ((obj=next())) list += obj->GetName();
84fcd93f 2582 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2583 }
2584 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2585 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2586 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2587 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2588 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2589 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2590 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2591 printf("=====================================================================\n");
2592 printf("= Job price: ___________________________________ %d\n", fPrice);
2593 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2594 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2595 if (fMaxInitFailed>0)
84fcd93f 2596 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2597 if (fMasterResubmitThreshold>0)
84fcd93f 2598 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2599 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2600 if (fNrunsPerMaster>0)
84fcd93f 2601 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2602 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2603 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2604 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2605 if (fArguments.Length())
84fcd93f 2606 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2607 if (fExecutableArgs.Length())
2608 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2609 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2610 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2611 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2612 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2613 if (fDatasetName)
84fcd93f 2614 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2615 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2616 if (fIncludePath.Data())
84fcd93f 2617 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2618 if (fCloseSE.Length())
84fcd93f 2619 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2620 if (fFriendChainName.Length())
84fcd93f 2621 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2622 if (fPackages && fPackages->GetEntries()) {
16a4353c 2623 TIter next(fPackages);
2624 TObject *obj;
2625 TString list;
2626 while ((obj=next())) list += obj->GetName();
84fcd93f 2627 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2628 }
2629}
2630
c57f56b7 2631//______________________________________________________________________________
2632void AliAnalysisAlien::SetDefaults()
2633{
2634// Set default values for everything. What cannot be filled will be left empty.
2635 if (fGridJDL) delete fGridJDL;
2636 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2637 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2638 fPrice = 1;
2639 fTTL = 30000;
2640 fSplitMaxInputFileNumber = 100;
2641 fMaxInitFailed = 0;
2642 fMasterResubmitThreshold = 0;
bb885a9e 2643 fNtestFiles = 10;
149d288c 2644 fNreplicas = 2;
a8739e8a 2645 fRunRange[0] = 0;
2646 fRunRange[1] = 0;
8e3bd06a 2647 fRunPrefix = "%d";
319593fb 2648 fNrunsPerMaster = 1;
16a4353c 2649 fMaxMergeFiles = 100;
c57f56b7 2650 fRunNumbers = "";
2651 fExecutable = "analysis.sh";
5aa94dc3 2652 fExecutableCommand = "root -b -q -x";
c57f56b7 2653 fArguments = "";
631c0b05 2654 fExecutableArgs = "";
c57f56b7 2655 fAnalysisMacro = "myAnalysis.C";
2656 fAnalysisSource = "";
2657 fAdditionalLibs = "";
2658 fSplitMode = "se";
2659 fAPIVersion = "";
2660 fROOTVersion = "";
2661 fAliROOTVersion = "";
2662 fUser = ""; // Your alien user name
2663 fGridWorkingDir = "";
2664 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2665 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2666 fFriendChainName = "";
c57f56b7 2667 fGridOutputDir = "output";
b83f84f0 2668 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2669 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2670 fInputFormat = "xml-single";
2671 fJDLName = "analysis.jdl";
c6cb3634 2672 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2673 fMergeExcludes = "";
0f389141 2674 fMergeViaJDL = 0;
348be253 2675 SetUseCopy(kTRUE);
2676 SetCheckCopy(kTRUE);
149d288c 2677 SetDefaultOutputs(kTRUE);
e1c22e21 2678 fOverwriteMode = 1;
c57f56b7 2679}
2680
a41b2882 2681//______________________________________________________________________________
2682void AliAnalysisAlien::SetFriendChainName(const char *name, const char *libnames)
2683{
2684 // Set file name for the chain of friends and optionally additional libs to be loaded.
2685 // Libs should be separated by blancs.
2686 fFriendChainName = name;
06f67114 2687 fFriendChainName.ReplaceAll(",", " ");
2688 fFriendChainName.Strip();
2689 fFriendChainName.ReplaceAll(" ", " ");
2690
a41b2882 2691 fFriendLibs = libnames;
1a3f255c 2692 if (fFriendLibs.Length()) {
2693 if(!fFriendLibs.Contains(".so"))
2694 Fatal("SetFriendChainName()", "You should provide explicit library names (with extension)");
2695 fFriendLibs.ReplaceAll(",", " ");
2696 fFriendLibs.Strip();
2697 fFriendLibs.ReplaceAll(" ", " ");
a41b2882 2698 }
2699}
2700
874de7ea 2701//______________________________________________________________________________
2702void AliAnalysisAlien::SetRootVersionForProof(const char *version)
2703{
2704// Obsolete method. Use SetROOTVersion instead
2705 Warning("SetRootVersionForProof", "Obsolete. Use SetROOTVersion instead");
95e16f01 2706 if (fROOTVersion.IsNull()) SetROOTVersion(version);
2707 else Error("SetRootVersionForProof", "ROOT version already set to %s", fROOTVersion.Data());
874de7ea 2708}
2709
0f389141 2710//______________________________________________________________________________
b3e07543 2711Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2712{
b3e07543 2713// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2714 // First check if the result is already in the output directory.
2715 if (FileExists(Form("%s/%s",aliendir,filename))) {
2716 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2717 return kFALSE;
2718 }
b3e07543 2719 // Now check the last stage done.
2720 Int_t stage = 0;
2721 while (1) {
2722 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2723 stage++;
2724 }
2725 // Next stage of merging
2726 stage++;
2727 TString pattern = "*root_archive.zip";
2728 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2729 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2730 if (res) delete res;
2731 // Write standard output to file
2732 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2733 // Count the number of files inside
2734 ifstream ifile;
2735 ifile.open(Form("Stage_%d.xml",stage));
2736 if (!ifile.good()) {
2737 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2738 return kFALSE;
b3e07543 2739 }
2740 TString line;
2741 Int_t nfiles = 0;
2742 while (!ifile.eof()) {
2743 ifile >> line;
2744 if (line.Contains("/event")) nfiles++;
7c2cd90a 2745 }
b3e07543 2746 ifile.close();
2747 if (!nfiles) {
2748 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2749 return kFALSE;
2750 } else {
2751 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2752 }
b3e07543 2753 // Copy the file in the output directory
2754 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2755// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2756 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2757 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2758 // Check if this is the last stage to be done.
2759 Bool_t laststage = (nfiles<nperchunk);
2760 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
0ad3ea1f 2761 Int_t jobId = 0;
b3e07543 2762 if (laststage) {
2763 printf("### Submiting final merging stage %d\n", stage);
2764 TString finalJDL = jdl;
2765 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2766 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
0ad3ea1f 2767 jobId = SubmitSingleJob(query);
b3e07543 2768 } else {
2769 printf("### Submiting merging stage %d\n", stage);
03335324 2770 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
0ad3ea1f 2771 jobId = SubmitSingleJob(query);
7c2cd90a 2772 }
0ad3ea1f 2773 if (!jobId) return kFALSE;
2774
2775 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
2776 fGridJobIDs.Append(Form("%d", jobId));
2777 if (!fGridStages.IsNull()) fGridStages.Append(" ");
2778 fGridStages.Append(Form("%s_merge_stage%d",
2779 laststage ? "final" : "partial", stage));
2780
b3e07543 2781 return kTRUE;
2782}
7c2cd90a 2783
105631b5 2784//______________________________________________________________________________
2785AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2786{
2787// Loat the analysis manager from a file.
2788 TFile *file = TFile::Open(fname);
2789 if (!file) {
2790 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2791 return 0;
2792 }
2793 TIter nextkey(file->GetListOfKeys());
2794 AliAnalysisManager *mgr = 0;
2795 TKey *key;
2796 while ((key=(TKey*)nextkey())) {
2797 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2798 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2799 }
2800 if (!mgr)
2801 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2802 return mgr;
2803}
2804
7c2cd90a 2805//______________________________________________________________________________
2806Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2807{
2808// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2809 if (!gGrid) return 0;
2810 printf("=> %s ------> ",query);
2811 TGridResult *res = gGrid->Command(query);
2812 if (!res) return 0;
2813 TString jobId = res->GetKey(0,"jobId");
2814 delete res;
2815 if (jobId.IsNull()) {
2816 printf("submission failed. Reason:\n");
2817 gGrid->Stdout();
2818 gGrid->Stderr();
2819 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2820 return 0;
2821 }
0ad3ea1f 2822 Int_t ijobId = jobId.Atoi();
2823 printf(" Job id: '%s' (%d)\n", jobId.Data(), ijobId);
2824 return ijobId;
7c2cd90a 2825}
2826
ec5acdce 2827//______________________________________________________________________________
2828Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2829{
2830// Merges a collection of output files using concatenation.
d66aa2a9 2831 TString scoll(collection);
2832 if (!scoll.Contains(".xml")) return kFALSE;
ec5acdce 2833 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2834 if (!coll) {
2835 ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2836 return kFALSE;
2837 }
2838 // Iterate grid collection
2839 TString outtmp;
2840 Bool_t merged = kFALSE;
2841 Int_t ifile = 0;
2842 while (coll->Next()) {
2843 TString fname = gSystem->DirName(coll->GetTURL());
2844 fname += "/";
2845 fname += output;
2846 outtmp = Form("%d_%s", ifile, output);
2847 if (!TFile::Cp(fname, outtmp)) {
2848 ::Error("MergeInfo", "Could not copy %s", fname.Data());
2849 continue;
2850 }
2851 ifile++;
2852 if (ifile<2) {
2853 gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2854 continue;
2855 }
2856 gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2857 gSystem->Exec("cp tempmerged lastmerged");
2858 }
2859 if (ifile) {
2860 gSystem->Exec(Form("cp lastmerged %s", output));
2861 gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2862 merged = kTRUE;
2863 }
2864 return merged;
2865}
2866
7c2cd90a 2867//______________________________________________________________________________
b3e07543 2868Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2869{
b3e07543 2870// Merge given output files from basedir. Basedir can be an alien output directory
2871// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2872// files in a group (ignored for xml input). Merging can be done in stages:
2873// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2874// stage=1 : works with an xml of all root_archive.zip in the output directory
2875// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2876 TString outputFile = output;
0f389141 2877 TString command;
a2f5fc01 2878 TString outputChunk;
2879 TString previousChunk = "";
b3e07543 2880 TObjArray *listoffiles = new TObjArray();
2881// listoffiles->SetOwner();
a2f5fc01 2882 Int_t countChunk = 0;
2883 Int_t countZero = nmaxmerge;
0f389141 2884 Bool_t merged = kTRUE;
d66aa2a9 2885 Bool_t isGrid = kTRUE;
a2f5fc01 2886 Int_t index = outputFile.Index("@");
2887 if (index > 0) outputFile.Remove(index);
7c2cd90a 2888 TString inputFile = outputFile;
b3e07543 2889 TString sbasedir = basedir;
2890 if (sbasedir.Contains(".xml")) {
2891 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2892 nmaxmerge = 9999999;
2893 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2894 if (!coll) {
2895 ::Error("MergeOutput", "Input XML collection empty.");
2896 return kFALSE;
2897 }
2898 // Iterate grid collection
2899 while (coll->Next()) {
2900 TString fname = gSystem->DirName(coll->GetTURL());
2901 fname += "/";
2902 fname += inputFile;
2903 listoffiles->Add(new TNamed(fname.Data(),""));
2904 }
ffbe06b9 2905 } else if (sbasedir.Contains(".txt")) {
d66aa2a9 2906 // The file having the .txt extension is expected to contain a list of
2907 // folders where the output files will be looked. For alien folders,
2908 // the full folder LFN is expected (starting with alien://)
ffbe06b9 2909 // Assume lfn's on each line
2910 TString line;
2911 ifstream in;
2912 in.open(sbasedir);
d66aa2a9 2913 if (in.fail()) {
2914 ::Error("MergeOutput", "File %s cannot be opened. Merging stopped." ,sbasedir.Data());
2915 return kTRUE;
2916 }
ffbe06b9 2917 Int_t nfiles = 0;
2918 while (in.good()) {
2919 in >> line;
2920 if (line.IsNull() || line.BeginsWith("#")) continue;
d66aa2a9 2921 line.Strip();
2922 if (!line.Contains("alien:")) isGrid = kFALSE;
2923 line += "/";
2924 line += outputFile;
ffbe06b9 2925 nfiles++;
2926 listoffiles->Add(new TNamed(line.Data(),""));
2927 }
d66aa2a9 2928 in.close();
ffbe06b9 2929 if (!nfiles) {
2930 ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data());
2931 delete listoffiles;
2932 return kFALSE;
2933 }
b3e07543 2934 } else {
2935 command = Form("find %s/ *%s", basedir, inputFile.Data());
2936 printf("command: %s\n", command.Data());
2937 TGridResult *res = gGrid->Command(command);
2938 if (!res) {
2939 ::Error("MergeOutput","No result for the find command\n");
2940 delete listoffiles;
2941 return kFALSE;
2942 }
2943 TIter nextmap(res);
2944 TMap *map = 0;
2945 while ((map=(TMap*)nextmap())) {
2946 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2947 if (!objs || !objs->GetString().Length()) {
2948 // Nothing found - skip this output
2949 delete res;
2950 delete listoffiles;
2951 return kFALSE;
2952 }
2953 listoffiles->Add(new TNamed(objs->GetName(),""));
2954 }
2955 delete res;
2956 }
2957 if (!listoffiles->GetEntries()) {
7c2cd90a 2958 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2959 delete listoffiles;
0f389141 2960 return kFALSE;
2961 }
2962
2963 TFileMerger *fm = 0;
b3e07543 2964 TIter next0(listoffiles);
2965 TObjArray *listoffilestmp = new TObjArray();
2966 listoffilestmp->SetOwner();
2967 TObject *nextfile;
2968 TString snextfile;
2969 // Keep only the files at upper level
2970 Int_t countChar = 0;
2971 while ((nextfile=next0())) {
2972 snextfile = nextfile->GetName();
2973 Int_t crtCount = snextfile.CountChar('/');
2974 if (nextfile == listoffiles->First()) countChar = crtCount;
2975 if (crtCount < countChar) countChar = crtCount;
2976 }
2977 next0.Reset();
2978 while ((nextfile=next0())) {
2979 snextfile = nextfile->GetName();
2980 Int_t crtCount = snextfile.CountChar('/');
2981 if (crtCount > countChar) {
2982 delete nextfile;
2983 continue;
2984 }
2985 listoffilestmp->Add(nextfile);
2986 }
2987 delete listoffiles;
2988 listoffiles = listoffilestmp; // Now contains 'good' files
2989 listoffiles->Print();
2990 TIter next(listoffiles);
7c2cd90a 2991 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2992 outputChunk = outputFile;
2993 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2994 // Check for existent temporary merge files
e1c22e21 2995 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2996 // Preserve old merging functionality for stage 0.
2997 if (stage==0) {
2998 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2999 while (1) {
3000 // Skip as many input files as in a chunk
2ea55496 3001 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 3002 nextfile = next();
3003 if (!nextfile) {
2ea55496 3004 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 3005 delete listoffiles;
2ea55496 3006 return kFALSE;
3007 }
b3e07543 3008 snextfile = nextfile->GetName();
7c2cd90a 3009 }
3010 outputChunk = outputFile;
3011 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
3012 countChunk++;
3013 if (gSystem->AccessPathName(outputChunk)) continue;
3014 // Merged file with chunks up to <countChunk> found
3015 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
3016 previousChunk = outputChunk;
3017 break;
3018 }
3019 }
3020 countZero = nmaxmerge;
3021
b3e07543 3022 while ((nextfile=next())) {
3023 snextfile = nextfile->GetName();
2ea55496 3024 // Loop 'find' results and get next LFN
7c2cd90a 3025 if (countZero == nmaxmerge) {
3026 // First file in chunk - create file merger and add previous chunk if any.
d66aa2a9 3027 fm = new TFileMerger(isGrid);
7c2cd90a 3028 fm->SetFastMethod(kTRUE);
3029 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
3030 outputChunk = outputFile;
3031 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
3032 }
3033 // If last file found, put merged results in the output file
b3e07543 3034 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 3035 // Add file to be merged and decrement chunk counter.
b3e07543 3036 fm->AddFile(snextfile);
7c2cd90a 3037 countZero--;
b3e07543 3038 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 3039 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3040 // Nothing found - skip this output
3041 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 3042 merged = kFALSE;
3043 break;
7c2cd90a 3044 }
3045 fm->OutputFile(outputChunk);
3046 // Merge the outputs, then go to next chunk
3047 if (!fm->Merge()) {
3048 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 3049 merged = kFALSE;
3050 break;
7c2cd90a 3051 } else {
3052 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
3053 gSystem->Unlink(previousChunk);
3054 }
b3e07543 3055 if (nextfile == listoffiles->Last()) break;
7c2cd90a 3056 countChunk++;
3057 countZero = nmaxmerge;
3058 previousChunk = outputChunk;
0f389141 3059 }
0f389141 3060 }
b3e07543 3061 delete listoffiles;
3062 delete fm;
7c2cd90a 3063 return merged;
3064 }
3065 // Merging stage different than 0.
3066 // Move to the begining of the requested chunk.
d66aa2a9 3067 fm = new TFileMerger(isGrid);
7c2cd90a 3068 fm->SetFastMethod(kTRUE);
b3e07543 3069 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
3070 delete listoffiles;
7c2cd90a 3071 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3072 // Nothing found - skip this output
3073 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
3074 delete fm;
3075 return kFALSE;
3076 }
b3e07543 3077 fm->OutputFile(outputFile);
7c2cd90a 3078 // Merge the outputs
3079 if (!fm->Merge()) {
3080 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
3081 delete fm;
3082 return kFALSE;
3083 } else {
b3e07543 3084 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 3085 }
7c2cd90a 3086 delete fm;
3087 return kTRUE;
0f389141 3088}
3089
c57f56b7 3090//______________________________________________________________________________
3091Bool_t AliAnalysisAlien::MergeOutputs()
3092{
3093// Merge analysis outputs existing in the AliEn space.
3094 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
3095 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3096 if (!Connect()) {
3097 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
3098 return kFALSE;
0f389141 3099 }
c9e8f7fd 3100 if (fMergeViaJDL) {
3101 if (!TestBit(AliAnalysisGrid::kMerge)) {
3102 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
3103 return kFALSE;
3104 }
3105 if (fProductionMode) {
3106 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
3107 return kFALSE;
3108 }
0f389141 3109 Info("MergeOutputs", "Submitting merging JDL");
a03be957 3110 if (!SubmitMerging()) return kFALSE;
0f389141 3111 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 3112 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 3113 return kFALSE;
c57f56b7 3114 }
3115 // Get the output path
dc030a24 3116 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 3117 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 3118 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
3119 return kFALSE;
3120 }
3121 if (!fOutputFiles.Length()) {
3122 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3123 return kFALSE;
0f389141 3124 }
3125 // Check if fast read option was requested
e1c22e21 3126 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
3127 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 3128 if (fFastReadOption) {
d3339be3 3129 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
3130 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 3131 gEnv->SetValue("XNet.ConnectTimeout",50);
3132 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 3133 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 3134 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 3135 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 3136 }
e8b839ab 3137 // Make sure we change the temporary directory
3138 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 3139 // Set temporary compilation directory to current one
3140 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 3141 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3142 TIter next(list);
3143 TObjString *str;
a2f5fc01 3144 TString outputFile;
c57f56b7 3145 Bool_t merged = kTRUE;
3146 while((str=(TObjString*)next())) {
a2f5fc01 3147 outputFile = str->GetString();
3148 Int_t index = outputFile.Index("@");
3149 if (index > 0) outputFile.Remove(index);
3150 TString outputChunk = outputFile;
3151 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 3152 // Skip already merged outputs
a2f5fc01 3153 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 3154 if (fOverwriteMode) {
a2f5fc01 3155 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
3156 gSystem->Unlink(outputFile);
3157 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3158 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3159 outputChunk.Data());
3160 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3161 }
3162 } else {
a2f5fc01 3163 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 3164 continue;
3165 }
3166 } else {
a2f5fc01 3167 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3168 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3169 outputChunk.Data());
3170 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3171 }
3172 }
37d21c01 3173 if (fMergeExcludes.Contains(outputFile.Data()) ||
3174 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 3175 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 3176 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 3177 if (!merged) {
3178 Error("MergeOutputs", "Terminate() will NOT be executed");
09d5920f 3179 delete list;
3180 return kFALSE;
ff07ec61 3181 }
3182 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
3183 if (fileOpened) fileOpened->Close();
c57f56b7 3184 }
09d5920f 3185 delete list;
0f389141 3186 return kTRUE;
c57f56b7 3187}
3188
bb885a9e 3189//______________________________________________________________________________
3190void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
3191{
3192// Use the output files connected to output containers from the analysis manager
3193// rather than the files defined by SetOutputFiles
3194 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 3195 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 3196 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
3197}
3198
149d288c 3199//______________________________________________________________________________
3200void AliAnalysisAlien::SetOutputFiles(const char *list)
3201{
3202// Manually set the output files list.
3203// Removes duplicates. Not allowed if default outputs are not disabled.
3204 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3205 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
3206 return;
3207 }
3208 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
3209 fOutputFiles = "";
3210 TString slist = list;
3211 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
3212 TObjArray *arr = slist.Tokenize(" ");
3213 TObjString *os;
3214 TIter next(arr);
3215 TString sout;
3216 while ((os=(TObjString*)next())) {
3217 sout = os->GetString();
3218 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
3219 if (fOutputFiles.Contains(sout)) continue;
3220 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
3221 fOutputFiles += sout;
3222 }
3223 delete arr;
f790bc1b 3224}
149d288c 3225
3226//______________________________________________________________________________
3227void AliAnalysisAlien::SetOutputArchive(const char *list)
3228{
3229// Manually set the output archive list. Free text - you are on your own...
3230// Not allowed if default outputs are not disabled.
3231 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3232 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
3233 return;
3234 }
3235 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
3236 fOutputArchive = list;
3237}
3238
3239//______________________________________________________________________________
3240void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
3241{
3242// Setting a prefered output SE is not allowed anymore.
3243 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
3244}
3245
28ce6c9c 3246//______________________________________________________________________________
3247void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
3248{
3249// Set some PROOF special parameter.
3250 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3251 if (pair) {
3252 TObject *old = pair->Key();
3253 TObject *val = pair->Value();
3254 fProofParam.Remove(old);
3255 delete old;
3256 delete val;
3257 }
3258 fProofParam.Add(new TObjString(pname), new TObjString(value));
3259}
3260
3261//______________________________________________________________________________
3262const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
3263{
3264// Returns a special PROOF parameter.
3265 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3266 if (!pair) return 0;
3267 return pair->Value()->GetName();
3268}
3269
5513444a 3270//______________________________________________________________________________
3271Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 3272{
3273// Start remote grid analysis.
3bdcb562 3274 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3275 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
3276 if (!mgr || !mgr->IsInitialized()) {
3277 Error("StartAnalysis", "You need an initialized analysis manager for this");
3278 return kFALSE;
3279 }
3280 // Are we in PROOF mode ?
3281 if (mgr->IsProofMode()) {
baa34407 3282 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3283 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3284 if (fProofCluster.IsNull()) {
3285 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3286 return kFALSE;
3287 }
3288 if (fProofDataSet.IsNull() && !testMode) {
3289 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3290 return kFALSE;
3291 }
3292 // Set the needed environment
3293 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3294 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3295 if (fProofReset && !testMode) {
3296 if (fProofReset==1) {
3297 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3298 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3299 } else {
3300 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3301 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3302 }
3303 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3304 return kFALSE;
3305 }
baa34407 3306
3307 if (!testMode) {
3308 // Check if there is an old active session
3309 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3310 if (nsessions) {
3311 Error("StartAnalysis","You have to reset your old session first\n");
3312 return kFALSE;
3313 }
53911ca6 3314 }
3bdcb562 3315 // Do we need to change the ROOT version ? The success of this cannot be checked.
874de7ea 3316 if (!fROOTVersion.IsNull() && !testMode) {
cbccef47 3317 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"VO_ALICE@ROOT::%s\");",
874de7ea 3318 fProofCluster.Data(), fROOTVersion.Data()));
3bdcb562 3319 }
3320 // Connect to PROOF and check the status
3321 Long_t proof = 0;
27734f0e 3322 TString sworkers;
3323 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3324 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3325 if (!testMode) {
27734f0e 3326 if (!sworkers.IsNull())
3327 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3328 else
3329 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3330 } else {
3331 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3332 if (!proof) {
3333 Error("StartAnalysis", "Could not start PROOF in test mode");
3334 return kFALSE;
3335 }
3336 }
3337 if (!proof) {
3338 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3339 return kFALSE;
3340 }
27734f0e 3341 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3342 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3343 // Set proof special parameters if any
3344 TIter nextpp(&fProofParam);
3345 TObject *proofparam;
3346 while ((proofparam=nextpp())) {
3347 TString svalue = GetProofParameter(proofparam->GetName());
3348 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3349 }
3bdcb562 3350 // Is dataset existing ?
3351 if (!testMode) {
3352 TString dataset = fProofDataSet;
3353 Int_t index = dataset.Index("#");
3354 if (index>=0) dataset.Remove(index);
830acc4c 3355// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3356// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3357// return kFALSE;
3358// }
3359// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3360 }
3361 // Is ClearPackages() needed ?
3362 if (TestSpecialBit(kClearPackages)) {
3363 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3364 gROOT->ProcessLine("gProof->ClearPackages();");
3365 }
3366 // Is a given aliroot mode requested ?
3367 TList optionsList;
1ac846e9 3368 TString parLibs;
830acc4c 3369 if (!fAliRootMode.IsNull()) {
3bdcb562 3370 TString alirootMode = fAliRootMode;
3371 if (alirootMode == "default") alirootMode = "";
3372 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3373 optionsList.SetOwner();
3374 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3375 // Check the additional libs to be loaded
3376 TString extraLibs;
1ac846e9 3377 Bool_t parMode = kFALSE;
dc1ffbcb 3378 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3379 // Parse the extra libs for .so
3380 if (fAdditionalLibs.Length()) {
a41b2882 3381 TString additionalLibs = fAdditionalLibs;
3382 additionalLibs.Strip();
3383 if (additionalLibs.Length() && fFriendLibs.Length())
3384 additionalLibs += " ";
3385 additionalLibs += fFriendLibs;
3386 TObjArray *list = additionalLibs.Tokenize(" ");
3bdcb562 3387 TIter next(list);
3388 TObjString *str;
1ac846e9 3389 while((str=(TObjString*)next())) {
3390 if (str->GetString().Contains(".so")) {
3391 if (parMode) {
3392 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3393 break;
3394 }
3395 TString stmp = str->GetName();
3396 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3397 stmp.ReplaceAll(".so","");
3398 if (!extraLibs.IsNull()) extraLibs += ":";
3399 extraLibs += stmp;
3400 continue;
3401 }
3402 if (str->GetString().Contains(".par")) {
3403 // The first par file found in the list will not allow any further .so
3404 parMode = kTRUE;
3405 if (!parLibs.IsNull()) parLibs += ":";
3406 parLibs += str->GetName();
3407 continue;
3408 }
3bdcb562 3409 }
3410 if (list) delete list;
3411 }
a41b2882 3412 if (!extraLibs.IsNull()) {
3413 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3414 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3415 }
830acc4c 3416 // Check extra includes
3417 if (!fIncludePath.IsNull()) {
3418 TString includePath = fIncludePath;
3419 includePath.ReplaceAll(" ",":");
dc030a24 3420 includePath.ReplaceAll("$ALICE_ROOT/","");
3421 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3422 includePath.ReplaceAll("-I","");
dc030a24 3423 includePath.Remove(TString::kTrailing, ':');
830acc4c 3424 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3425 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3426 }
27734f0e 3427 // Check if connection to grid is requested
3428 if (TestSpecialBit(kProofConnectGrid))
3429 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3430 // Enable AliRoot par
3431 if (testMode) {
3432 // Enable proof lite package
3433 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3434 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3435 TNamed *obj = (TNamed*)optionsList.At(i);
3436 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3437 }
3438 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3439 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3440 Info("StartAnalysis", "AliRootProofLite enabled");
3441 } else {
3442 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3443 return kFALSE;
3444 }
3445 } else {
dc030a24 3446 if ( ! fAliROOTVersion.IsNull() ) {
3447 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3448 fAliROOTVersion.Data(), &optionsList))) {
3449 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3450 return kFALSE;
3451 }
3452 }
27734f0e 3453 }
1ac846e9 3454 // Enable first par files from fAdditionalLibs
3455 if (!parLibs.IsNull()) {
3456 TObjArray *list = parLibs.Tokenize(":");
3457 TIter next(list);
3458 TObjString *package;
3459 while((package=(TObjString*)next())) {
40501bb7 3460 TString spkg = package->GetName();
3461 spkg.ReplaceAll(".par", "");
3462 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3463 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3464 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3465 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3466 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3467 return kFALSE;
3468 }
3469 } else {
3470 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3471 return kFALSE;
3472 }
3473 }
3474 if (list) delete list;
3475 }
3bdcb562 3476 } else {
3477 if (fAdditionalLibs.Contains(".so") && !testMode) {
3478 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3479 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3480 return kFALSE;
3481 }
3482 }
3483 // Enable par files if requested
3484 if (fPackages && fPackages->GetEntries()) {
3485 TIter next(fPackages);
3486 TObject *package;
3487 while ((package=next())) {
1ac846e9 3488 // Skip packages already enabled
3489 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3490 TString spkg = package->GetName();
3491 spkg.ReplaceAll(".par", "");
3492 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3493 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3494 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3495 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3496 return kFALSE;
3497 }
3498 } else {
3499 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3500 return kFALSE;
3501 }
3502 }
3503 }
3504 // Do we need to load analysis source files ?
3505 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3506 if (fAnalysisSource.Length()) {
3507 TObjArray *list = fAnalysisSource.Tokenize(" ");
3508 TIter next(list);
3509 TObjString *str;
3510 while((str=(TObjString*)next())) {
3511 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3512 }
3bdcb562 3513 if (list) delete list;
3514 }
830acc4c 3515 if (testMode) {
3516 // Register dataset to proof lite.
3517 if (fFileForTestMode.IsNull()) {
3518 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3519 return kFALSE;
3520 }
3521 if (gSystem->AccessPathName(fFileForTestMode)) {
3522 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3523 return kFALSE;
3524 }
3525 TFileCollection *coll = new TFileCollection();
3526 coll->AddFromFile(fFileForTestMode);
8ddad121 3527 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3528 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3529 }
3bdcb562 3530 return kTRUE;
3531 }
c57f56b7 3532
43da816a 3533 // Check if output files have to be taken from the analysis manager
3534 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3535 // Add output files and AOD files
3536 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3537 // Add extra files registered to the analysis manager
f790bc1b 3538 TString extra = GetListOfFiles("ext");
3539 if (!extra.IsNull()) {
149d288c 3540 extra.ReplaceAll(".root", "*.root");
f790bc1b 3541 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3542 fOutputFiles += extra;
84fcd93f 3543 }
149d288c 3544 // Compose the output archive.
b83f84f0 3545 fOutputArchive = "log_archive.zip:std*@disk=1 ";
ec5acdce 3546 if (mgr->IsCollectThroughput())
95e16f01 3547 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",fOutputFiles.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 3548 else
3549 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3550 }
f7b1cbc2 3551// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3552 if (TestBit(AliAnalysisGrid::kOffline)) {
3553 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3554 \n there nor any job run. You can revise the JDL and analysis \
3555 \n macro then run the same in \"submit\" mode.");
3556 } else if (TestBit(AliAnalysisGrid::kTest)) {
3557 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3558 \n dataset.");
3559 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3560 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3561 \n space and job submitted.");
3562 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3563 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3564 if (fMergeViaJDL) CheckInputData();
5513444a 3565 return kTRUE;
c57f56b7 3566 } else {
3567 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3568 }
3569
348be253 3570 Print();
c57f56b7 3571 if (!Connect()) {
3572 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3573 return kFALSE;
16a4353c 3574 }
d3b18c4c 3575 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3576 if (!CheckInputData()) {
3577 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3578 return kFALSE;
c57f56b7 3579 }
d3339be3 3580 if (!CreateDataset(fDataPattern)) {
3581 TString serror;
3582 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3583 if (fRunNumbers.Length()) serror = "run numbers";
3584 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3585 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3586 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3587 return kFALSE;
3588 }
21ca8e59 3589 WriteAnalysisFile();
c57f56b7 3590 WriteAnalysisMacro();
3591 WriteExecutable();
3592 WriteValidationScript();
0f389141 3593 if (fMergeViaJDL) {
3594 WriteMergingMacro();
3595 WriteMergeExecutable();
3596 WriteValidationScript(kTRUE);
3597 }
5513444a 3598 if (!CreateJDL()) return kFALSE;
3599 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3600 if (testMode) {
c57f56b7 3601 // Locally testing the analysis
3602 Info("StartAnalysis", "\n_______________________________________________________________________ \
3603 \n Running analysis script in a daughter shell as on a worker node \
3604 \n_______________________________________________________________________");
149d288c 3605 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3606 TIter next(list);
3607 TObjString *str;
a2f5fc01 3608 TString outputFile;
c57f56b7 3609 while((str=(TObjString*)next())) {
a2f5fc01 3610 outputFile = str->GetString();
3611 Int_t index = outputFile.Index("@");
3612 if (index > 0) outputFile.Remove(index);
3613 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3614 }
3615 delete list;
3616 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3617 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3618// gSystem->Exec("cat stdout");
5513444a 3619 return kFALSE;
c57f56b7 3620 }
5513444a 3621 // Check if submitting is managed by LPM manager
a3e84053 3622 if (fProductionMode) {
7ac61826 3623 //TString prodfile = fJDLName;
3624 //prodfile.ReplaceAll(".jdl", ".prod");
3625 //WriteProductionFile(prodfile);
5513444a 3626 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3627 return kFALSE;
3628 }
a8739e8a 3629 // Submit AliEn job(s)
d2a409b2 3630 gGrid->Cd(fGridOutputDir);
a8739e8a 3631 TGridResult *res;
c57f56b7 3632 TString jobID = "";
0ad3ea1f 3633 fGridJobIDs = "";
3634 fGridStages = "";
d2a409b2 3635 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3636 // Submit a given xml or a set of runs
a8739e8a 3637 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3638 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3639 if (res) {
3640 const char *cjobId = res->GetKey(0,"jobId");
3641 if (!cjobId) {
a03be957 3642 gGrid->Stdout();
3643 gGrid->Stderr();
a8739e8a 3644 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3645 return kFALSE;
a8739e8a 3646 } else {
3647 Info("StartAnalysis", "\n_______________________________________________________________________ \
3648 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3649 \n_______________________________________________________________________",
3650 fJDLName.Data(), cjobId);
3651 jobID = cjobId;
0ad3ea1f 3652 if (jobID.Atoi()) {
3653 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3654 fGridJobIDs.Append(jobID);
3655 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3656 fGridStages.Append("full");
3657 }
a8739e8a 3658 }
3659 delete res;
a03be957 3660 } else {
3661 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3662 return kFALSE;
a8739e8a 3663 }
3664 } else {
d2a409b2 3665 // Submit for a range of enumeration of runs.
a03be957 3666 if (!Submit()) return kFALSE;
0ad3ea1f 3667 jobID = fGridJobIDs;
c57f56b7 3668 }
a8739e8a 3669
d88e4159 3670 if (fDropToShell) {
3671 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3672 \n You may exit at any time and terminate the job later using the option <terminate> \
3673 \n ##################################################################################", jobID.Data());
3674 gSystem->Exec("aliensh");
3675 } else {
3676 Info("StartAnalysis", "\n#### SUBMITTED JOB %s TO ALIEN QUEUE #### \
3677 \n Remember to terminate the job later using the option <terminate> \
3678 \n ##################################################################################", jobID.Data());
3679 }
5513444a 3680 return kTRUE;
c57f56b7 3681}
3682
f790bc1b 3683//______________________________________________________________________________
3684const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3685{
3686// Get a comma-separated list of output files of the requested type.
3687// Type can be (case unsensitive):
3688// aod - list of aod files (std, extensions and filters)
3689// out - list of output files connected to containers (but not aod's or extras)
3690// ext - list of extra files registered to the manager
3691// ter - list of files produced in terminate
3692 static TString files;
3693 files = "";
3694 TString stype = type;
3695 stype.ToLower();
3696 TString aodfiles, extra;
3697 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3698 if (!mgr) {
3699 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3700 return files.Data();
3701 }
3702 if (mgr->GetOutputEventHandler()) {
6be131bd 3703 aodfiles = "";
3704 if (mgr->GetOutputEventHandler()->GetFillAOD())
3705 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
f6eaa73b 3706 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs(kTRUE);
6be131bd 3707 if (!extraaod.IsNull() && mgr->GetOutputEventHandler()->GetFillExtension()) {
0c0825e4 3708 if (!aodfiles.IsNull()) aodfiles += ",";
f790bc1b 3709 aodfiles += extraaod;
3710 }
3711 }
3712 if (stype.Contains("aod")) {
3713 files = aodfiles;
3714 if (stype == "aod") return files.Data();
3715 }
3716 // Add output files that are not in the list of AOD files
3717 TString outputfiles = "";
3718 TIter next(mgr->GetOutputs());
3719 AliAnalysisDataContainer *output;
3720 const char *filename = 0;
3721 while ((output=(AliAnalysisDataContainer*)next())) {
3722 filename = output->GetFileName();
3723 if (!(strcmp(filename, "default"))) continue;
3724 if (outputfiles.Contains(filename)) continue;
3725 if (aodfiles.Contains(filename)) continue;
0c0825e4 3726 if (!outputfiles.IsNull() && strlen(filename)) outputfiles += ",";
f790bc1b 3727 outputfiles += filename;
3728 }
3729 if (stype.Contains("out")) {
0c0825e4 3730 if (!files.IsNull() && !outputfiles.IsNull()) files += ",";
f790bc1b 3731 files += outputfiles;
3732 if (stype == "out") return files.Data();
3733 }
3734 // Add extra files registered to the analysis manager
3735 TString sextra;
3736 extra = mgr->GetExtraFiles();
3737 if (!extra.IsNull()) {
3738 extra.Strip();
3739 extra.ReplaceAll(" ", ",");
3740 TObjArray *fextra = extra.Tokenize(",");
3741 TIter nextx(fextra);
3742 TObject *obj;
3743 while ((obj=nextx())) {
3744 if (aodfiles.Contains(obj->GetName())) continue;
3745 if (outputfiles.Contains(obj->GetName())) continue;
3746 if (sextra.Contains(obj->GetName())) continue;
3747 if (!sextra.IsNull()) sextra += ",";
3748 sextra += obj->GetName();
3749 }
3750 delete fextra;
3751 if (stype.Contains("ext")) {
0c0825e4 3752 if (!files.IsNull() && !sextra.IsNull()) files += ",";
f790bc1b 3753 files += sextra;
3754 }
3755 }
3756 if (stype == "ext") return files.Data();
3757 TString termfiles;
3758 if (!fTerminateFiles.IsNull()) {
3759 fTerminateFiles.Strip();
3760 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3761 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3762 TIter nextx(fextra);
3763 TObject *obj;
3764 while ((obj=nextx())) {
3765 if (aodfiles.Contains(obj->GetName())) continue;
3766 if (outputfiles.Contains(obj->GetName())) continue;
3767 if (termfiles.Contains(obj->GetName())) continue;
3768 if (sextra.Contains(obj->GetName())) continue;
3769 if (!termfiles.IsNull()) termfiles += ",";
3770 termfiles += obj->GetName();
3771 }
3772 delete fextra;
3773 }
3774 if (stype.Contains("ter")) {
3775 if (!files.IsNull() && !termfiles.IsNull()) {
3776 files += ",";
3777 files += termfiles;
3778 }
3779 }
3780 return files.Data();
3781}
3782
d2a409b2 3783//______________________________________________________________________________
a03be957 3784Bool_t AliAnalysisAlien::Submit()
d2a409b2 3785{
3786// Submit all master jobs.
3787 Int_t nmasterjobs = fInputFiles->GetEntries();
3788 Long_t tshoot = gSystem->Now();
a03be957 3789 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3790 while (fNsubmitted < nmasterjobs) {
3791 Long_t now = gSystem->Now();
3792 if ((now-tshoot)>30000) {
3793 tshoot = now;
a03be957 3794 if (!SubmitNext()) return kFALSE;
d2a409b2 3795 }
3796 }
a03be957 3797 return kTRUE;
d2a409b2 3798}
3799
0f389141 3800//______________________________________________________________________________
a03be957 3801Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3802{
3803// Submit all merging jobs.
dc030a24 3804 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3805 gGrid->Cd(fGridOutputDir);
3806 TString mergeJDLName = fExecutable;
3807 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3808 if (!fInputFiles) {
3809 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3810 return kFALSE;
3811 }
0f389141 3812 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3813 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3814 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3815 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3816 if (fOutputToRunNo) {
3817 // The output directory is the run number
3818 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3819 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3820 } else {
17b4e9a4 3821 if (!fRunNumbers.Length() && !fRunRange[0]) {
3822 // The output directory is the grid outdir
3823 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3824 runOutDir = fGridOutputDir;
3825 } else {
3826 // The output directory is the master number in 3 digits format
3827 printf("### Submitting merging job for master <%03d>\n", i);
3828 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3829 }
7c2cd90a 3830 }
3831 // Check now the number of merging stages.
2b222f7c 3832 TObjArray *list = fOutputFiles.Tokenize(",");
3833 TIter next(list);
3834 TObjString *str;
3835 TString outputFile;
3836 while((str=(TObjString*)next())) {
3837 outputFile = str->GetString();
3838 Int_t index = outputFile.Index("@");
3839 if (index > 0) outputFile.Remove(index);
37d21c01 3840 if (!fMergeExcludes.Contains(outputFile) &&
3841 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3842 }
3843 delete list;
b3e07543 3844 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3845 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3846 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3847 }
a03be957 3848 if (!ntosubmit) return kTRUE;
d88e4159 3849 if (fDropToShell) {
3850 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3851 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3852 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3853 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3854 \n ################################################################################################################");
3855 gSystem->Exec("aliensh");
3856 } else {
3857 Info("StartAnalysis", "\n #### STARTED MERGING JOBS FOR YOU #### \
3858 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL') \
3859 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3860 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3861 \n ################################################################################################################");
3862 }
a03be957 3863 return kTRUE;
0f389141 3864}
3865
d2a409b2 3866//______________________________________________________________________________
a03be957 3867Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3868{
08d5b699 3869// Submit next bunch of master jobs if the queue is free. The first master job is
3870// submitted right away, while the next will not be unless the previous was split.
3871// The plugin will not submit new master jobs if there are more that 500 jobs in
3872// waiting phase.
d2a409b2 3873 static Bool_t iscalled = kFALSE;
3874 static Int_t firstmaster = 0;
3875 static Int_t lastmaster = 0;
3876 static Int_t npermaster = 0;
a03be957 3877 if (iscalled) return kTRUE;
d2a409b2 3878 iscalled = kTRUE;
3879 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3880 Int_t ntosubmit = 0;
3881 TGridResult *res;
3882 TString jobID = "";
3bdcb562 3883 Int_t nmasterjobs = fInputFiles->GetEntries();
3884 if (!fNsubmitted) {
3885 ntosubmit = 1;
3886 if (!IsUseSubmitPolicy()) {
90d50a8c 3887 if (nmasterjobs>5)
3bdcb562 3888 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3889 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3890 ntosubmit = nmasterjobs;
3891 }
3892 } else {
d2a409b2 3893 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3894 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3895 // If last master not split, just return
a03be957 3896 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3897 // No more than 100 waiting jobs
08d5b699 3898 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3899 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3900 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3901 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3902 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3903 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3904 }
d2a409b2 3905 for (Int_t i=0; i<ntosubmit; i++) {
3906 // Submit for a range of enumeration of runs.
a03be957 3907 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3908 TString query;
cd11251e 3909 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3910 runOutDir.ReplaceAll(".xml", "");
3911 if (fOutputToRunNo)
3912 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3913 else
3914 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3915 printf("********* %s\n",query.Data());
d2a409b2 3916 res = gGrid->Command(query);
3917 if (res) {
98ca124f 3918 TString cjobId1 = res->GetKey(0,"jobId");
3919 if (!cjobId1.Length()) {
d2a409b2 3920 iscalled = kFALSE;
a03be957 3921 gGrid->Stdout();
3922 gGrid->Stderr();
3923 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3924 return kFALSE;
d2a409b2 3925 } else {
3926 Info("StartAnalysis", "\n_______________________________________________________________________ \
3927 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3928 \n_______________________________________________________________________",
98ca124f 3929 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
0ad3ea1f 3930 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3931 fGridJobIDs.Append(cjobId1);
3932 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3933 fGridStages.Append("full");
d2a409b2 3934 jobID += cjobId1;
3935 jobID += " ";
98ca124f 3936 lastmaster = cjobId1.Atoi();
d2a409b2 3937 if (!firstmaster) firstmaster = lastmaster;
3938 fNsubmitted++;
3939 }
3940 delete res;
a03be957 3941 } else {
3942 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3943 return kFALSE;
d2a409b2 3944 }
3945 }
3946 iscalled = kFALSE;
a03be957 3947 return kTRUE;
d2a409b2 3948}
3949
c57f56b7 3950//______________________________________________________________________________
3951void AliAnalysisAlien::WriteAnalysisFile()
3952{
f10e8481 3953// Write current analysis manager into the file <analysisFile>
3954 TString analysisFile = fExecutable;
3955 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3956 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3957 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3958 if (!mgr || !mgr->IsInitialized()) {
3959 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3960 return;
3961 }
3962 // Check analysis type
3963 TObject *handler;
3964 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3965 handler = (TObject*)mgr->GetInputEventHandler();
3966 if (handler) {
65eb22f8 3967 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3968 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3969 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3970 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3971 } else {
3972 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3973 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3974 }
c57f56b7 3975 }
3976 TDirectory *cdir = gDirectory;
f10e8481 3977 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3978 if (file) {
e16a394c 3979 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3980 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3981 // Unless merging makes no sense
3982 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3983 mgr->Write();
3984 delete file;
fe2d7fc2 3985 // Enable termination for local jobs
3986 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3987 }
3988 if (cdir) cdir->cd();
f10e8481 3989 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3990 }
3991 Bool_t copy = kTRUE;
d3b18c4c 3992 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3993 if (copy) {
3994 CdWork();
3995 TString workdir = gGrid->GetHomeDirectory();
3996 workdir += fGridWorkingDir;
d3b18c4c 3997 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3998 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3999 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
4000 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 4001 }
4002}
4003
4004//______________________________________________________________________________
4005void AliAnalysisAlien::WriteAnalysisMacro()
4006{
4007// Write the analysis macro that will steer the analysis in grid mode.
4008 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4009 ofstream out;
4010 out.open(fAnalysisMacro.Data(), ios::out);
4011 if (!out.good()) {
4012 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4013 return;
4014 }
5b273635 4015 Bool_t hasSTEERBase = kFALSE;
4016 Bool_t hasESD = kFALSE;
4017 Bool_t hasAOD = kFALSE;
4018 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4019 Bool_t hasOADB = kFALSE;
5b273635 4020 Bool_t hasANALYSISalice = kFALSE;
4021 Bool_t hasCORRFW = kFALSE;
c57f56b7 4022 TString func = fAnalysisMacro;
4023 TString type = "ESD";
4024 TString comment = "// Analysis using ";
715da79c 4025 if (fMCLoop) {
4026 type = "MCGEN";
4027 comment += "MCGEN";
4028 } else {
4029 if (IsUseMCchain()) {
4030 type = "MC";
4031 comment += "MC";
4032 } else {
4033 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
4034 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
4035 type = "AOD";
4036 comment += "AOD";
4037 }
4038 }
4039 }
0df6ccf2 4040 if (type!="AOD" && fFriendChainName!="") {
4041 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
4042 return;
4043 }
c57f56b7 4044 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
4045 else comment += " data";
4046 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
4047 func.ReplaceAll(".C", "");
4048 out << "void " << func.Data() << "()" << endl;
4049 out << "{" << endl;
4050 out << comment.Data() << endl;
4051 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 4052 out << " TStopwatch timer;" << endl;
4053 out << " timer.Start();" << endl << endl;
13f0c00b 4054 // Change temp directory to current one
fdee57f1 4055 if (!IsLocalTest()) {
4056 out << "// connect to AliEn and make the chain" << endl;
4057 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
4058 }
13f0c00b 4059 out << "// Set temporary merging directory to current one" << endl;
4060 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4061 out << "// Set temporary compilation directory to current one" << endl;
4062 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 4063 // Reset existing include path
4064 out << "// Reset existing include path and add current directory first in the search" << endl;
4065 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 4066 if (!fExecutableCommand.Contains("aliroot")) {
4067 out << "// load base root libraries" << endl;
4068 out << " gSystem->Load(\"libTree\");" << endl;
4069 out << " gSystem->Load(\"libGeom\");" << endl;
4070 out << " gSystem->Load(\"libVMC\");" << endl;
4071 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4072 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4073 }
d5c6455a 4074 if (fAdditionalRootLibs.Length()) {
47a4137d 4075 // in principle libtree /lib geom libvmc etc. can go into this list, too
4076 out << "// Add aditional libraries" << endl;
4077 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4078 TIter next(list);
4079 TObjString *str;
4080 while((str=(TObjString*)next())) {
4081 if (str->GetString().Contains(".so"))
4082 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 4083 }
47a4137d 4084 if (list) delete list;
d5c6455a 4085 }
57377eb5 4086 out << "// Load analysis framework libraries" << endl;
652561d5 4087 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 4088 if (!fPackages) {
f790bc1b 4089 if (!fExecutableCommand.Contains("aliroot")) {
4090 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4091 out << " gSystem->Load(\"libESD\");" << endl;
4092 out << " gSystem->Load(\"libAOD\");" << endl;
4093 }
4e5c5506 4094 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4095 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 4096 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4097 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 4098 } else {
4e5c5506 4099 TIter next(fPackages);
4100 TObject *obj;
57377eb5 4101 TString pkgname;
57377eb5 4102 while ((obj=next())) {
4103 pkgname = obj->GetName();
4478e6f1 4104 if (pkgname == "STEERBase" ||
4105 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4106 if (pkgname == "ESD" ||
4107 pkgname == "ESD.par") hasESD = kTRUE;
4108 if (pkgname == "AOD" ||
4109 pkgname == "AOD.par") hasAOD = kTRUE;
4110 if (pkgname == "ANALYSIS" ||
4111 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4112 if (pkgname == "OADB" ||
4113 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 4114 if (pkgname == "ANALYSISalice" ||
4115 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4116 if (pkgname == "CORRFW" ||
4117 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 4118 }
4119 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 4120 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4121 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 4122 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4123 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 4124 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4125 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 4126 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4127 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4128 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4129 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 4130 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4131 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 4132 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4133 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 4134 out << "// Compile other par packages" << endl;
4135 next.Reset();
fcc9bb6f 4136 while ((obj=next())) {
4137 pkgname = obj->GetName();
4478e6f1 4138 if (pkgname == "STEERBase" ||
4139 pkgname == "STEERBase.par" ||
4140 pkgname == "ESD" ||
4141 pkgname == "ESD.par" ||
4142 pkgname == "AOD" ||
4143 pkgname == "AOD.par" ||
4144 pkgname == "ANALYSIS" ||
4145 pkgname == "ANALYSIS.par" ||
7ea40ac8 4146 pkgname == "OADB" ||
4147 pkgname == "OADB.par" ||
4478e6f1 4148 pkgname == "ANALYSISalice" ||
4149 pkgname == "ANALYSISalice.par" ||
4150 pkgname == "CORRFW" ||
4151 pkgname == "CORRFW.par") continue;
5b273635 4152 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 4153 }
4e5c5506 4154 }
b3e07543 4155 out << "// include path" << endl;
4156 // Get the include path from the interpreter and remove entries pointing to AliRoot
4157 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4158 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4159 out << " TIter nextpath(listpaths);" << endl;
4160 out << " TObjString *pname;" << endl;
4161 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4162 out << " TString current = pname->GetName();" << endl;
4163 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4164 out << " gSystem->AddIncludePath(current);" << endl;
4165 out << " }" << endl;
4166 out << " if (listpaths) delete listpaths;" << endl;
4167 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4168 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4169 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
715da79c 4170 if (fMCLoop && !fGeneratorLibs.IsNull()) {
4171 out << "// MC generator libraries" << endl;
4172 TObjArray *list = fGeneratorLibs.Tokenize(" ");
4173 TIter next(list);
4174 TObjString *str;
4175 while((str=(TObjString*)next())) {
4176 out << " gSystem->Load(\"" << str->GetName() << "\");" << endl;
4177 }
4178 delete list;
4179 }
6da75e0b 4180 if (fAdditionalLibs.Length()) {
4181 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4182 TString additionalLibs = fAdditionalLibs;
4183 additionalLibs.Strip();
4184 if (additionalLibs.Length() && fFriendLibs.Length())
4185 additionalLibs += " ";
4186 additionalLibs += fFriendLibs;
4187 TObjArray *list = additionalLibs.Tokenize(" ");
6da75e0b 4188 TIter next(list);
4189 TObjString *str;
4190 while((str=(TObjString*)next())) {
4191 if (str->GetString().Contains(".so"))
4192 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 4193 if (str->GetString().Contains(".par"))
4194 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 4195 }
715da79c 4196 delete list;
6da75e0b 4197 }
4198 out << endl;
c57f56b7 4199 out << "// analysis source to be compiled at runtime (if any)" << endl;
4200 if (fAnalysisSource.Length()) {
4201 TObjArray *list = fAnalysisSource.Tokenize(" ");
4202 TIter next(list);
4203 TObjString *str;
4204 while((str=(TObjString*)next())) {
4205 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4206 }
4207 if (list) delete list;
4208 }
4209 out << endl;
b3e07543 4210// out << " printf(\"Currently load libraries:\\n\");" << endl;
4211// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 4212 if (fFastReadOption) {
d3339be3 4213 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
4214 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 4215 out << "// fast xrootd reading enabled" << endl;
4216 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4217 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4218 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4219 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4220 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4221 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 4222 }
c57f56b7 4223 out << "// read the analysis manager from file" << endl;
f10e8481 4224 TString analysisFile = fExecutable;
4225 analysisFile.ReplaceAll(".sh", ".root");
105631b5 4226 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4227 << analysisFile << "\");" << endl;
4228 out << " if (!mgr) return;" << endl;
21ca8e59 4229 if (IsLocalTest()) {
4230 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
4231 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 4232 if (fFileForTestMode.IsNull())
4233 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
4234 else
4235 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 4236 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4b5d9e21 4237 if (!fFriendChainName.IsNull())
b9f1efa5 4238 out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\",\"" << fFriendLibs << "\");" << endl;
024dfa52 4239 if (IsUseMCchain())
4240 out << " plugin->SetUseMCchain();" << endl;
715da79c 4241 if (fMCLoop)
4242 out << " plugin->SetMCLoop(kTRUE);" << endl;
21ca8e59 4243 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 4244 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 4245 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 4246 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
4247 } else {
7a366e0b 4248 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 4249 out << " mgr->SetNSysInfo(100);" << endl;
4250 }
21ca8e59 4251 }
c57f56b7 4252 out << " mgr->PrintStatus();" << endl;
52b6a92b 4253 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 4254 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 4255 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 4256 } else {
9a8288ff 4257 if (TestBit(AliAnalysisGrid::kTest))
4258 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4259 else
4260 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 4261 }
4262 }
21ca8e59 4263 if (!IsLocalTest()) {
715da79c 4264 if (fMCLoop) {
4265 out << " mgr->SetCacheSize(0);" << endl;
4266 out << " mgr->EventLoop(" << fNMCevents << ");" << endl;
4267 } else {
4268 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
4269 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
4270 }
21ca8e59 4271 } else {
715da79c 4272 if (fMCLoop) {
4273 out << " mgr->SetCacheSize(0);" << endl;
4274 out << " mgr->EventLoop(" << fNMCevents << ");" << endl;
7435f77f
AG		 4275 } else {
4276 out << " mgr->StartAnalysis(\"localfile\");" << endl;
4277 }
242accb2 4278 }
f7498086 4279 out << " timer.Stop();" << endl;
4280 out << " timer.Print();" << endl;
c57f56b7 4281 out << "}" << endl << endl;
715da79c 4282 if (!IsLocalTest() && !fMCLoop) {
fcc9bb6f 4283 out <<"//________________________________________________________________________________" << endl;
c57f56b7 4284 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
4285 out << "{" << endl;
4286 out << "// Create a chain using url's from xml file" << endl;
242accb2 4287 out << " TString filename;" << endl;
4288 out << " Int_t run = 0;" << endl;
c85cfc0f 4289 if (IsUseMCchain()) {
4290 out << " TString treename = \"TE\";" << endl;
4291 } else {
780ec36d 4292 if (!fTreeName.IsNull()) {
4293 out << " TString treename = \"" << fTreeName << "\";" << endl;
4294 } else {
4295 out << " TString treename = type;" << endl;
4296 out << " treename.ToLower();" << endl;
4297 out << " treename += \"Tree\";" << endl;
4298 }
c85cfc0f 4299 }
e02fee64 4300 out << " printf(\"***************************************\\n\");" << endl;
4301 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
4302 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 4303 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
4304 out << " if (!coll) {" << endl;
4305 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
4306 out << " return NULL;" << endl;
4307 out << " }" << endl;
242accb2 4308 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 4309 out << " TChain *chain = new TChain(treename);" << endl;
06f67114 4310 if(!fFriendChainName.IsNull()) {
4311 out << " TList *friends = new TList();" << endl;
4312 out << " TIter nextfriend(friends);" << endl;
4313 out << " TChain *cfriend = 0;" << endl;
4314 TObjArray *list = fFriendChainName.Tokenize(" ");
4315 TIter next(list);
4316 TObjString *str;
4317 while((str=(TObjString*)next())) {
4318 out << " cfriend = new TChain(treename, \"" << str->GetName() << "\");" << endl;
4319 out << " friends->Add(cfriend);" << endl;
4320 out << " chain->AddFriend(cfriend);" << endl;
4321 }
4322 delete list;
4323// out << " TChain *chainFriend = new TChain(treename);" << endl;
0df6ccf2 4324 }
c57f56b7 4325 out << " coll->Reset();" << endl;
0df6ccf2 4326 out << " while (coll->Next()) {" << endl;
242accb2 4327 out << " filename = coll->GetTURL("");" << endl;
4328 out << " if (mgr) {" << endl;
4329 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
4330 out << " if (nrun && nrun != run) {" << endl;
4331 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
4332 out << " mgr->SetRunFromPath(nrun);" << endl;
4333 out << " run = nrun;" << endl;
4334 out << " }" << endl;
4335 out << " }" << endl;
4336 out << " chain->Add(filename);" << endl;
06f67114 4337 if(!fFriendChainName.IsNull()) {
4338 out << " TString bpath=coll->GetTURL(\"\");" << endl;
4339 out << " if (bpath.Index(\"#\") > -1) bpath.Remove(bpath.Index(\"#\"));" << endl;
4340 out << " bpath = gSystem->DirName(bpath);" << endl;
4341 out << " bpath += \"/\";" << endl;
4342 out << " TString fileFriend;" << endl;
4343 out << " nextfriend.Reset();" << endl;
4344 out << " while ((cfriend=(TChain*)nextfriend())) {" << endl;
4345 out << " fileFriend = bpath;" << endl;
4346 out << " fileFriend += cfriend->GetTitle();" << endl;
4347 out << " TFile *file = TFile::Open(fileFriend);" << endl;
4348 out << " if (file) {" << endl;
4349 out << " file->Close();" << endl;
4350 out << " cfriend->Add(fileFriend.Data());" << endl;
4351 out << " } else {" << endl;
4352 out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
4353 out << " return 0;" << endl;
4354 out << " }" << endl;
4b5d9e21 4355 out << " }" << endl;
0df6ccf2 4356 }
4357 out << " }" << endl;
c57f56b7 4358 out << " if (!chain->GetNtrees()) {" << endl;
4359 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
4360 out << " return NULL;" << endl;
4361 out << " }" << endl;
4362 out << " return chain;" << endl;
fcc9bb6f 4363 out << "}" << endl << endl;
c57f56b7 4364 }
5b273635 4365 if (hasANALYSISalice) {
4366 out <<"//________________________________________________________________________________" << endl;
4367 out << "Bool_t SetupPar(const char *package) {" << endl;
4368 out << "// Compile the package and set it up." << endl;
4369 out << " TString pkgdir = package;" << endl;
4370 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4371 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4372 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4373 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4374 out << " // Check for BUILD.sh and execute" << endl;
4375 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4376 out << " printf(\"*******************************\\n\");" << endl;
4377 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4378 out << " printf(\"*******************************\\n\");" << endl;
4379 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4380 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4381 out << " gSystem->ChangeDirectory(cdir);" << endl;
4382 out << " return kFALSE;" << endl;
4383 out << " }" << endl;
4384 out << " } else {" << endl;
4385 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4386 out << " gSystem->ChangeDirectory(cdir);" << endl;
4387 out << " return kFALSE;" << endl;
4388 out << " }" << endl;
4389 out << " // Check for SETUP.C and execute" << endl;
4390 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4391 out << " printf(\"*******************************\\n\");" << endl;
4392 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4393 out << " printf(\"*******************************\\n\");" << endl;
4394 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4395 out << " } else {" << endl;
4396 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4397 out << " gSystem->ChangeDirectory(cdir);" << endl;
4398 out << " return kFALSE;" << endl;
4399 out << " }" << endl;
4400 out << " // Restore original workdir" << endl;
4401 out << " gSystem->ChangeDirectory(cdir);" << endl;
4402 out << " return kTRUE;" << endl;
4403 out << "}" << endl;
4404 }
c57f56b7 4405 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4406 }
4407 Bool_t copy = kTRUE;
d3b18c4c 4408 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4409 if (copy) {
4410 CdWork();
4411 TString workdir = gGrid->GetHomeDirectory();
4412 workdir += fGridWorkingDir;
4413 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4414 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4415// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4416 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4417 Form("alien://%s/%s", workdir.Data(),
4418 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4419 }
4420}
4421
0f389141 4422//______________________________________________________________________________
4423void AliAnalysisAlien::WriteMergingMacro()
4424{
4425// Write a macro to merge the outputs per master job.
4426 if (!fMergeViaJDL) return;
4427 if (!fOutputFiles.Length()) {
4428 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4429 return;
4430 }
ec5acdce 4431 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
0f389141 4432 TString mergingMacro = fExecutable;
4433 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4434 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4435 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4436 ofstream out;
4437 out.open(mergingMacro.Data(), ios::out);
4438 if (!out.good()) {
4439 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4440 return;
4441 }
5b273635 4442 Bool_t hasSTEERBase = kFALSE;
4443 Bool_t hasESD = kFALSE;
4444 Bool_t hasAOD = kFALSE;
4445 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4446 Bool_t hasOADB = kFALSE;
5b273635 4447 Bool_t hasANALYSISalice = kFALSE;
4448 Bool_t hasCORRFW = kFALSE;
0f389141 4449 TString func = mergingMacro;
4450 TString comment;
4451 func.ReplaceAll(".C", "");
f47d5cb4 4452 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4453 out << "{" << endl;
4454 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4455 out << " TStopwatch timer;" << endl;
4456 out << " timer.Start();" << endl << endl;
b3e07543 4457 // Reset existing include path
4458 out << "// Reset existing include path and add current directory first in the search" << endl;
4459 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4460 if (!fExecutableCommand.Contains("aliroot")) {
4461 out << "// load base root libraries" << endl;
4462 out << " gSystem->Load(\"libTree\");" << endl;
4463 out << " gSystem->Load(\"libGeom\");" << endl;
4464 out << " gSystem->Load(\"libVMC\");" << endl;
4465 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4466 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4467 }
0f389141 4468 if (fAdditionalRootLibs.Length()) {
4469 // in principle libtree /lib geom libvmc etc. can go into this list, too
4470 out << "// Add aditional libraries" << endl;
4471 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4472 TIter next(list);
4473 TObjString *str;
4474 while((str=(TObjString*)next())) {
4475 if (str->GetString().Contains(".so"))
4476 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4477 }
4478 if (list) delete list;
4479 }
0f389141 4480 out << "// Load analysis framework libraries" << endl;
4481 if (!fPackages) {
7c2cd90a 4482 if (!fExecutableCommand.Contains("aliroot")) {
4483 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4484 out << " gSystem->Load(\"libESD\");" << endl;
4485 out << " gSystem->Load(\"libAOD\");" << endl;
4486 }
0f389141 4487 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4488 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4489 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4490 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4491 } else {
4492 TIter next(fPackages);
4493 TObject *obj;
4494 TString pkgname;
5b273635 4495 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4496 while ((obj=next())) {
4497 pkgname = obj->GetName();
4498 if (pkgname == "STEERBase" ||
4499 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4500 if (pkgname == "ESD" ||
4501 pkgname == "ESD.par") hasESD = kTRUE;
4502 if (pkgname == "AOD" ||
4503 pkgname == "AOD.par") hasAOD = kTRUE;
4504 if (pkgname == "ANALYSIS" ||
4505 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4506 if (pkgname == "OADB" ||
4507 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4508 if (pkgname == "ANALYSISalice" ||
4509 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4510 if (pkgname == "CORRFW" ||
4511 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4512 }
5b273635 4513 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4514 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4515 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4516 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4517 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4518 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4519 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4520 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4521 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4522 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4523 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4524 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4525 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4526 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4527 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4528 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4529 out << "// Compile other par packages" << endl;
4530 next.Reset();
4531 while ((obj=next())) {
4532 pkgname = obj->GetName();
4533 if (pkgname == "STEERBase" ||
4534 pkgname == "STEERBase.par" ||
4535 pkgname == "ESD" ||
4536 pkgname == "ESD.par" ||
4537 pkgname == "AOD" ||
4538 pkgname == "AOD.par" ||
4539 pkgname == "ANALYSIS" ||
4540 pkgname == "ANALYSIS.par" ||
7ea40ac8 4541 pkgname == "OADB" ||
4542 pkgname == "OADB.par" ||
0f389141 4543 pkgname == "ANALYSISalice" ||
4544 pkgname == "ANALYSISalice.par" ||
4545 pkgname == "CORRFW" ||
4546 pkgname == "CORRFW.par") continue;
5b273635 4547 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4548 }
4549 }
b3e07543 4550 out << "// include path" << endl;
4551 // Get the include path from the interpreter and remove entries pointing to AliRoot
4552 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4553 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4554 out << " TIter nextpath(listpaths);" << endl;
4555 out << " TObjString *pname;" << endl;
4556 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4557 out << " TString current = pname->GetName();" << endl;
4558 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4559 out << " gSystem->AddIncludePath(current);" << endl;
4560 out << " }" << endl;
4561 out << " if (listpaths) delete listpaths;" << endl;
4562 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4563 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4564 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0d239a91
AG		 4565 if (fMCLoop && !fGeneratorLibs.IsNull()) {
4566 out << "// MC generator libraries" << endl;
4567 TObjArray *list = fGeneratorLibs.Tokenize(" ");
4568 TIter next(list);
4569 TObjString *str;
4570 while((str=(TObjString*)next())) {
4571 out << " gSystem->Load(\"" << str->GetName() << "\");" << endl;
4572 }
4573 delete list;
4574 }
0f389141 4575 if (fAdditionalLibs.Length()) {
4576 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4577 TString additionalLibs = fAdditionalLibs;
4578 additionalLibs.Strip();
4579 if (additionalLibs.Length() && fFriendLibs.Length())
4580 additionalLibs += " ";
4581 additionalLibs += fFriendLibs;
4582 TObjArray *list = additionalLibs.Tokenize(" ");
0f389141 4583 TIter next(list);
4584 TObjString *str;
4585 while((str=(TObjString*)next())) {
4586 if (str->GetString().Contains(".so"))
4587 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4588 }
4589 if (list) delete list;
4590 }
4591 out << endl;
4592 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4593 if (fAnalysisSource.Length()) {
4594 TObjArray *list = fAnalysisSource.Tokenize(" ");
4595 TIter next(list);
4596 TObjString *str;
4597 while((str=(TObjString*)next())) {
4598 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4599 }
4600 if (list) delete list;
4601 }
149d288c 4602 out << endl;
4603
0f389141 4604 if (fFastReadOption) {
4605 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4606 out << "// fast xrootd reading enabled" << endl;
4607 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4608 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4609 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4610 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4611 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4612 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4613 }
4614 // Change temp directory to current one
fdee57f1 4615 out << "// Connect to AliEn" << endl;
4616 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
e8b839ab 4617 out << "// Set temporary merging directory to current one" << endl;
4618 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4619 out << "// Set temporary compilation directory to current one" << endl;
4620 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
7c2cd90a 4621 out << " TString outputDir = dir;" << endl;
d87f72d5 4622 if (IsMergeAOD())
4623 out << " TString outputFiles = \"" << GetListOfFiles("outaod") << "\";" << endl;
4624 else
4625 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4626 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4627 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4628 out << " TIter *iter = new TIter(list);" << endl;
4629 out << " TObjString *str;" << endl;
a2f5fc01 4630 out << " TString outputFile;" << endl;
0f389141 4631 out << " Bool_t merged = kTRUE;" << endl;
ec5acdce 4632 TString analysisFile = fExecutable;
4633 analysisFile.ReplaceAll(".sh", ".root");
4634 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4635 << analysisFile << "\");" << endl;
4636 out << " if (!mgr) {" << endl;
4637 out << " printf(\"ERROR: Analysis manager could not be extracted from file \");" << endl;
4638 out << " return;" << endl;
4639 out << " }" << endl;
d66aa2a9 4640 if (IsLocalTest()) {
0c0825e4 4641 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4642 out << " printf(\"Testing merging...\\n\");" << endl;
0c0825e4 4643 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4644 }
0f389141 4645 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4646 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4647 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4648 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4649 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4650 out << " // Skip already merged outputs" << endl;
a2f5fc01 4651 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
d66aa2a9 4652 out << " printf(\"Output file <%s> found. Not merging again.\\n\",outputFile.Data());" << endl;
0f389141 4653 out << " continue;" << endl;
4654 out << " }" << endl;
a2f5fc01 4655 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4656 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4657 out << " if (!merged) {" << endl;
a2f5fc01 4658 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4659 out << " return;" << endl;
0f389141 4660 out << " }" << endl;
4661 out << " }" << endl;
d66aa2a9 4662 if (mgr && mgr->IsCollectThroughput() && !IsLocalTest()) {
ec5acdce 4663 out << " TString infolog = \"" << mgr->GetFileInfoLog() << "\";" << endl;
4664 out << " AliAnalysisAlien::MergeInfo(infolog, outputDir);" << endl;
4665 }
7c2cd90a 4666 out << " // all outputs merged, validate" << endl;
4667 out << " ofstream out;" << endl;
4668 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4669 out << " out.close();" << endl;
4670 out << " // read the analysis manager from file" << endl;
d66aa2a9 4671 if (IsLocalTest()) {
0c0825e4 4672 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4673 out << " printf(\"Testing Terminate()...\\n\");" << endl;
0c0825e4 4674 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4675 } else {
4676 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
4677 }
5c4250fc 4678 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4679 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4680 out << " mgr->PrintStatus();" << endl;
ec5acdce 4681 if (mgr) {
4682 if (mgr->GetDebugLevel()>3) {
f866cba5 4683 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4684 } else {
9a8288ff 4685 if (TestBit(AliAnalysisGrid::kTest))
4686 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4687 else
4688 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4689 }
4690 }
9a8288ff 4691 out << " TTree *tree = NULL;" << endl;
4692 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4693 out << "}" << endl << endl;
5b273635 4694 if (hasANALYSISalice) {
4695 out <<"//________________________________________________________________________________" << endl;
4696 out << "Bool_t SetupPar(const char *package) {" << endl;
4697 out << "// Compile the package and set it up." << endl;
4698 out << " TString pkgdir = package;" << endl;
4699 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4700 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4701 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4702 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4703 out << " // Check for BUILD.sh and execute" << endl;
4704 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4705 out << " printf(\"*******************************\\n\");" << endl;
4706 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4707 out << " printf(\"*******************************\\n\");" << endl;
4708 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4709 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4710 out << " gSystem->ChangeDirectory(cdir);" << endl;
4711 out << " return kFALSE;" << endl;
4712 out << " }" << endl;
4713 out << " } else {" << endl;
4714 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4715 out << " gSystem->ChangeDirectory(cdir);" << endl;
4716 out << " return kFALSE;" << endl;
4717 out << " }" << endl;
4718 out << " // Check for SETUP.C and execute" << endl;
4719 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4720 out << " printf(\"*******************************\\n\");" << endl;
4721 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4722 out << " printf(\"*******************************\\n\");" << endl;
4723 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4724 out << " } else {" << endl;
4725 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4726 out << " gSystem->ChangeDirectory(cdir);" << endl;
4727 out << " return kFALSE;" << endl;
4728 out << " }" << endl;
4729 out << " // Restore original workdir" << endl;
4730 out << " gSystem->ChangeDirectory(cdir);" << endl;
4731 out << " return kTRUE;" << endl;
4732 out << "}" << endl;
4733 }
0f389141 4734 }
4735 Bool_t copy = kTRUE;
d3b18c4c 4736 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4737 if (copy) {
4738 CdWork();
4739 TString workdir = gGrid->GetHomeDirectory();
4740 workdir += fGridWorkingDir;
4741 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4742 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4743// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4744 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4745 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4746 }
4747}
4748
4749//______________________________________________________________________________
4750Bool_t AliAnalysisAlien::SetupPar(const char *package)
4751{
205b201f 4752// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4753// Note that for loading the compiled library. The current directory should have precedence in
4754// LD_LIBRARY_PATH
4755 TString pkgdir = package;
4756 pkgdir.ReplaceAll(".par","");
b50a41c3 4757 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4758 TString cdir = gSystem->WorkingDirectory();
4759 gSystem->ChangeDirectory(pkgdir);
4760 // Check for BUILD.sh and execute
4761 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4762 printf("**************************************************\n");
4763 printf("*** Building PAR archive %s\n", package);
4764 printf("**************************************************\n");
4765 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4766 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4767 gSystem->ChangeDirectory(cdir);
4768 return kFALSE;
4769 }
4770 } else {
4771 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4772 gSystem->ChangeDirectory(cdir);
4773 return kFALSE;
4774 }
4775 // Check for SETUP.C and execute
4776 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4777 printf("**************************************************\n");
4778 printf("*** Setup PAR archive %s\n", package);
4779 printf("**************************************************\n");
4780 gROOT->Macro("PROOF-INF/SETUP.C");
4781 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4782 } else {
4783 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4784 gSystem->ChangeDirectory(cdir);
4785 return kFALSE;
4786 }
4787 // Restore original workdir
4788 gSystem->ChangeDirectory(cdir);
4789 return kTRUE;
4790}
4791
c57f56b7 4792//______________________________________________________________________________
4793void AliAnalysisAlien::WriteExecutable()
4794{
4795// Generate the alien executable script.
5aa94dc3 4796 // Patch executable with -x to catch error code
4797 if (fExecutableCommand.Contains("root") &&
4798 fExecutableCommand.Contains("-q") &&
4799 !fExecutableCommand.Contains("-x")) fExecutableCommand += " -x";
c57f56b7 4800 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4801 ofstream out;
4802 out.open(fExecutable.Data(), ios::out);
4803 if (out.bad()) {
5513444a 4804 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4805 return;
4806 }
4807 out << "#!/bin/bash" << endl;
2a4d5166 4808 // Make sure we can properly compile par files
4809 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4810 out << "echo \"=========================================\"" << endl;
4811 out << "echo \"############## PATH : ##############\"" << endl;
4812 out << "echo $PATH" << endl;
4813 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4814 out << "echo $LD_LIBRARY_PATH" << endl;
4815 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4816 out << "echo $ROOTSYS" << endl;
4817 out << "echo \"############## which root : ##############\"" << endl;
4818 out << "which root" << endl;
4819 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4820 out << "echo $ALICE_ROOT" << endl;
4821 out << "echo \"############## which aliroot : ##############\"" << endl;
4822 out << "which aliroot" << endl;
9c5ddadc 4823 out << "echo \"############## system limits : ##############\"" << endl;
4824 out << "ulimit -a" << endl;
4825 out << "echo \"############## memory : ##############\"" << endl;
4826 out << "free -m" << endl;
c57f56b7 4827 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4828 out << fExecutableCommand << " ";
370c9880 4829 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4830 out << "RET=$?" << endl;
4831 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4832 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4833 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4834 out << " let sig=\"$RET - 128\""<<endl;
4835 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4836 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4837 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4838 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4839 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4840 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4841 out << " fi"<<endl;
4842 out << " exit $RET"<< endl;
4843 out << "fi" << endl << endl ;
4844 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4845 out << "echo \"############## memory after: ##############\"" << endl;
4846 out << "free -m" << endl;
c57f56b7 4847 }
4848 Bool_t copy = kTRUE;
d3b18c4c 4849 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4850 if (copy) {
4851 CdWork();
4852 TString workdir = gGrid->GetHomeDirectory();
4853 workdir += fGridWorkingDir;
6da4839e 4854 TString executable = TString::Format("%s/%s", workdir.Data(), fExecutable.Data());
c57f56b7 4855 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4856 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4857 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4858 executable.Data())) Fatal("","Terminating");
c57f56b7 4859 }
4860}
4861
0f389141 4862//______________________________________________________________________________
4863void AliAnalysisAlien::WriteMergeExecutable()
4864{
4865// Generate the alien executable script for the merging job.
4866 if (!fMergeViaJDL) return;
4867 TString mergeExec = fExecutable;
4868 mergeExec.ReplaceAll(".sh", "_merge.sh");
4869 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4870 ofstream out;
4871 out.open(mergeExec.Data(), ios::out);
4872 if (out.bad()) {
4873 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4874 return;
4875 }
4876 out << "#!/bin/bash" << endl;
2a4d5166 4877 // Make sure we can properly compile par files
4878 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4879 out << "echo \"=========================================\"" << endl;
4880 out << "echo \"############## PATH : ##############\"" << endl;
4881 out << "echo $PATH" << endl;
4882 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4883 out << "echo $LD_LIBRARY_PATH" << endl;
4884 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4885 out << "echo $ROOTSYS" << endl;
4886 out << "echo \"############## which root : ##############\"" << endl;
4887 out << "which root" << endl;
4888 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4889 out << "echo $ALICE_ROOT" << endl;
4890 out << "echo \"############## which aliroot : ##############\"" << endl;
4891 out << "which aliroot" << endl;
4892 out << "echo \"############## system limits : ##############\"" << endl;
4893 out << "ulimit -a" << endl;
4894 out << "echo \"############## memory : ##############\"" << endl;
4895 out << "free -m" << endl;
4896 out << "echo \"=========================================\"" << endl << endl;
0f389141 4897 TString mergeMacro = fExecutable;
4898 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4899 if (IsOneStageMerging())
4900 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4901 else
f47d5cb4 4902 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4903 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4904 out << "RET=$?" << endl;
4905 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4906 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4907 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4908 out << " let sig=\"$RET - 128\""<<endl;
4909 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4910 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4911 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4912 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4913 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4914 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4915 out << " fi"<<endl;
4916 out << " exit $RET"<< endl;
4917 out << "fi" << endl << endl ;
0f389141 4918 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4919 out << "echo \"############## memory after: ##############\"" << endl;
4920 out << "free -m" << endl;
0f389141 4921 }
4922 Bool_t copy = kTRUE;
d3b18c4c 4923 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4924 if (copy) {
4925 CdWork();
4926 TString workdir = gGrid->GetHomeDirectory();
0f389141 4927 workdir += fGridWorkingDir;
6da4839e 4928 TString executable = TString::Format("%s/%s", workdir.Data(), mergeExec.Data());
0f389141 4929 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4930 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6da4839e
AG		 4931 if (!copyLocal2Alien("WriteMergeExecutable",mergeExec.Data(),
4932 executable.Data())) Fatal("","Terminating");
0f389141 4933 }
4934}
4935
c57f56b7 4936//______________________________________________________________________________
5513444a 4937void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4938{
4939// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4940// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4941// Next lines: full_path_to_dataset XXX (XXX is a string)
4942// To submit, one has to: submit jdl XXX for all lines
4943 ofstream out;
4944 out.open(filename, ios::out);
4945 if (out.bad()) {
4946 Error("WriteProductionFile", "Bad file name: %s", filename);
4947 return;
4948 }
d3b18c4c 4949 TString workdir;
4950 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4951 workdir = gGrid->GetHomeDirectory();
5513444a 4952 workdir += fGridWorkingDir;
f5e8c702 4953 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4954 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4955 out << locjdl << " " << njobspermaster << endl;
5513444a 4956 Int_t nmasterjobs = fInputFiles->GetEntries();
4957 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4958 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4959 runOutDir.ReplaceAll(".xml", "");
4960 if (fOutputToRunNo)
4961 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4962 else
4963 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4964 }
d3b18c4c 4965 if (gGrid) {
4966 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4967 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4968// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4969 if (!copyLocal2Alien("WriteProductionFile", filename,
4970 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4971 }
5513444a 4972}
4973
4974//______________________________________________________________________________
0f389141 4975void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4976{
4977// Generate the alien validation script.
4978 // Generate the validation script
4979 TObjString *os;
d3b18c4c 4980 if (fValidationScript.IsNull()) {
4981 fValidationScript = fExecutable;
4982 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4983 }
4984 TString validationScript = fValidationScript;
4985 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4986 if (!Connect()) {
4987 Error("WriteValidationScript", "Alien connection required");
4988 return;
4989 }
d3b18c4c 4990 if (!fTerminateFiles.IsNull()) {
4991 fTerminateFiles.Strip();
4992 fTerminateFiles.ReplaceAll(" ",",");
4993 }
a2f5fc01 4994 TString outStream = "";
4995 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4996 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4997 ofstream out;
0d5d317c 4998 out.open(validationScript, ios::out);
c57f56b7 4999 out << "#!/bin/bash" << endl;
5000 out << "##################################################" << endl;
5001 out << "validateout=`dirname $0`" << endl;
5002 out << "validatetime=`date`" << endl;
5003 out << "validated=\"0\";" << endl;
5004 out << "error=0" << endl;
5005 out << "if [ -z $validateout ]" << endl;
5006 out << "then" << endl;
5007 out << " validateout=\".\"" << endl;
5008 out << "fi" << endl << endl;
5009 out << "cd $validateout;" << endl;
5010 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 5011 out << "echo \"*******************************************************\"" << outStream << endl;
5012 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 5013 out << "" << endl;
a2f5fc01 5014 out << "echo \"* Time: $validatetime \"" << outStream << endl;
5015 out << "echo \"* Dir: $validateout\"" << outStream << endl;
5016 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
5017 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5018 out << "ls -la ./" << outStream << endl;
5019 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 5020 out << "##################################################" << endl;
ebec370a 5021 out << "" << endl;
5022
5023 out << "if [ ! -f stderr ] ; then" << endl;
5024 out << " error=1" << endl;
a2f5fc01 5025 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
5026 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5027 out << "fi" << endl;
5028
b34c9f51 5029 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
5030 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
5031 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
5032 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
5033 out << "" << endl;
5034
ebec370a 5035 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
5036 out << " error=1" << endl;
a2f5fc01 5037 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
5038 out << " echo \"$parArch\" " << outStream << endl;
5039 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5040 out << "fi" << endl;
5041
5042 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
5043 out << " error=1" << endl;
a2f5fc01 5044 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
5045 out << " echo \"$segViol\" " << outStream << endl;
5046 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5047 out << "fi" << endl;
5048
5049 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
5050 out << " error=1" << endl;
a2f5fc01 5051 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
5052 out << " echo \"$segFault\" " << outStream << endl;
5053 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5054 out << "fi" << endl;
5055
b34c9f51 5056 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
5057 out << " error=1" << endl;
a2f5fc01 5058 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
5059 out << " echo \"$glibcErr\" " << outStream << endl;
5060 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 5061 out << "fi" << endl;
5062
ebec370a 5063 // Part dedicated to the specific analyses running into the train
5064
d3b18c4c 5065 TString outputFiles = fOutputFiles;
5066 if (merge && !fTerminateFiles.IsNull()) {
0c0825e4 5067 if (!outputFiles.IsNull()) outputFiles += ",";
d3b18c4c 5068 outputFiles += fTerminateFiles;
5069 }
5070 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 5071 TIter next1(arr);
a2f5fc01 5072 TString outputFile;
bbd88db8 5073 while (!merge && (os=(TObjString*)next1())) {
5074 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 5075 outputFile = os->GetString();
5076 Int_t index = outputFile.Index("@");
5077 if (index > 0) outputFile.Remove(index);
bbd88db8 5078 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 5079 if (outputFile.Contains("*")) continue;
5080 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 5081 out << " error=1" << endl;
7c2cd90a 5082 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
5083 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 5084 out << "fi" << endl;
5085 }
5086 delete arr;
7c2cd90a 5087 out << "if ! [ -f outputs_valid ] ; then" << endl;
5088 out << " error=1" << endl;
5089 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
5090 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
5091 out << "fi" << endl;
923e2ca5 5092
c57f56b7 5093 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 5094 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 5095 if (!IsKeepLogs()) {
5096 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 5097 outStream = "";
149d288c 5098 out << " rm -f std*" << endl;
5099 }
c57f56b7 5100 out << "fi" << endl;
5101
a2f5fc01 5102 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5103 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 5104 out << "cd -" << endl;
5105 out << "exit $error" << endl;
5106 }
5107 Bool_t copy = kTRUE;
d3b18c4c 5108 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 5109 if (copy) {
5110 CdWork();
5111 TString workdir = gGrid->GetHomeDirectory();
5112 workdir += fGridWorkingDir;
d3b18c4c 5113 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 5114 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 5115// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
5116 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
5117 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 5118 }
5119}