]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Bug in dispCore filling corrected; cleanup
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
715da79c 32#include "TInterpreter.h"
c57f56b7 33#include "TFile.h"
830acc4c 34#include "TFileCollection.h"
3bdcb562 35#include "TChain.h"
c57f56b7 36#include "TObjString.h"
37#include "TObjArray.h"
77f34eae 38#include "TMacro.h"
c57f56b7 39#include "TGrid.h"
40#include "TGridResult.h"
41#include "TGridCollection.h"
42#include "TGridJDL.h"
d2a409b2 43#include "TGridJobStatusList.h"
44#include "TGridJobStatus.h"
c57f56b7 45#include "TFileMerger.h"
46#include "AliAnalysisManager.h"
77f34eae 47#include "AliAnalysisTaskCfg.h"
bb885a9e 48#include "AliVEventHandler.h"
49#include "AliAnalysisDataContainer.h"
65eb22f8 50#include "AliMultiInputEventHandler.h"
c57f56b7 51
c82bb898 52using std::ofstream;
53using std::ifstream;
54using std::ios;
55using std::endl;
c57f56b7 56ClassImp(AliAnalysisAlien)
6c395669 57#if 0
58;
59#endif
c57f56b7 60
6c395669 61namespace {
62 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
63 {
64 TString sl(Form("file:%s", loc));
65 TString sr(Form("alien://%s", rem));
66 Bool_t ret = TFile::Cp(sl, sr);
67 if (!ret) {
68 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
69 }
70 return ret;
71 }
72}
73
c57f56b7 74//______________________________________________________________________________
75AliAnalysisAlien::AliAnalysisAlien()
76 :AliAnalysisGrid(),
77 fGridJDL(NULL),
0f389141 78 fMergingJDL(NULL),
c57f56b7 79 fPrice(0),
80 fTTL(0),
81 fSplitMaxInputFileNumber(0),
82 fMaxInitFailed(0),
83 fMasterResubmitThreshold(0),
bb885a9e 84 fNtestFiles(0),
319593fb 85 fNrunsPerMaster(0),
16a4353c 86 fMaxMergeFiles(0),
b3e07543 87 fMaxMergeStages(0),
d2a409b2 88 fNsubmitted(0),
a3e84053 89 fProductionMode(0),
cd11251e 90 fOutputToRunNo(0),
0f389141 91 fMergeViaJDL(0),
92 fFastReadOption(0),
e1c22e21 93 fOverwriteMode(1),
149d288c 94 fNreplicas(2),
3bdcb562 95 fNproofWorkers(0),
27734f0e 96 fNproofWorkersPerSlave(0),
3bdcb562 97 fProofReset(0),
715da79c 98 fNMCevents(0),
99 fNMCjobs(0),
c57f56b7 100 fRunNumbers(),
101 fExecutable(),
0a1c1f7f 102 fExecutableCommand(),
c57f56b7 103 fArguments(),
631c0b05 104 fExecutableArgs(),
c57f56b7 105 fAnalysisMacro(),
106 fAnalysisSource(),
d3b18c4c 107 fValidationScript(),
d5c6455a 108 fAdditionalRootLibs(),
c57f56b7 109 fAdditionalLibs(),
715da79c 110 fGeneratorLibs(),
c57f56b7 111 fSplitMode(),
112 fAPIVersion(),
113 fROOTVersion(),
114 fAliROOTVersion(),
648174cf 115 fExternalPackages(),
c57f56b7 116 fUser(),
117 fGridWorkingDir(),
118 fGridDataDir(),
119 fDataPattern(),
120 fGridOutputDir(),
121 fOutputArchive(),
122 fOutputFiles(),
123 fInputFormat(),
e7c71df0 124 fDatasetName(),
c57f56b7 125 fJDLName(),
d3b18c4c 126 fTerminateFiles(),
bb885a9e 127 fMergeExcludes(),
37d21c01 128 fRegisterExcludes(),
f965131e 129 fIncludePath(),
bb885a9e 130 fCloseSE(),
0df6ccf2 131 fFriendChainName(),
c6cb3634 132 fJobTag(),
648174cf 133 fOutputSingle(),
5fce53f4 134 fRunPrefix(),
3bdcb562 135 fProofCluster(),
136 fProofDataSet(),
137 fFileForTestMode(),
3bdcb562 138 fAliRootMode(),
d88e4159 139 fProofProcessOpt(),
f47d5cb4 140 fMergeDirName(),
4e5c5506 141 fInputFiles(0),
28ce6c9c 142 fPackages(0),
77f34eae 143 fModules(0),
d88e4159 144 fProofParam(),
a41b2882 145 fDropToShell(true),
715da79c 146 fMCLoop(false),
a41b2882 147 fGridJobIDs(""),
148 fGridStages(""),
780ec36d 149 fFriendLibs(""),
150 fTreeName()
c57f56b7 151{
152// Dummy ctor.
153 SetDefaults();
154}
155
156//______________________________________________________________________________
157AliAnalysisAlien::AliAnalysisAlien(const char *name)
158 :AliAnalysisGrid(name),
159 fGridJDL(NULL),
0f389141 160 fMergingJDL(NULL),
c57f56b7 161 fPrice(0),
162 fTTL(0),
163 fSplitMaxInputFileNumber(0),
164 fMaxInitFailed(0),
165 fMasterResubmitThreshold(0),
bb885a9e 166 fNtestFiles(0),
319593fb 167 fNrunsPerMaster(0),
16a4353c 168 fMaxMergeFiles(0),
b3e07543 169 fMaxMergeStages(0),
d2a409b2 170 fNsubmitted(0),
a3e84053 171 fProductionMode(0),
cd11251e 172 fOutputToRunNo(0),
0f389141 173 fMergeViaJDL(0),
174 fFastReadOption(0),
e1c22e21 175 fOverwriteMode(1),
149d288c 176 fNreplicas(2),
3bdcb562 177 fNproofWorkers(0),
27734f0e 178 fNproofWorkersPerSlave(0),
3bdcb562 179 fProofReset(0),
715da79c 180 fNMCevents(0),
181 fNMCjobs(0),
c57f56b7 182 fRunNumbers(),
183 fExecutable(),
0a1c1f7f 184 fExecutableCommand(),
c57f56b7 185 fArguments(),
631c0b05 186 fExecutableArgs(),
c57f56b7 187 fAnalysisMacro(),
188 fAnalysisSource(),
d3b18c4c 189 fValidationScript(),
d5c6455a 190 fAdditionalRootLibs(),
c57f56b7 191 fAdditionalLibs(),
715da79c 192 fGeneratorLibs(),
c57f56b7 193 fSplitMode(),
194 fAPIVersion(),
195 fROOTVersion(),
196 fAliROOTVersion(),
648174cf 197 fExternalPackages(),
c57f56b7 198 fUser(),
199 fGridWorkingDir(),
200 fGridDataDir(),
201 fDataPattern(),
202 fGridOutputDir(),
203 fOutputArchive(),
204 fOutputFiles(),
205 fInputFormat(),
e7c71df0 206 fDatasetName(),
c57f56b7 207 fJDLName(),
d3b18c4c 208 fTerminateFiles(),
bb885a9e 209 fMergeExcludes(),
37d21c01 210 fRegisterExcludes(),
f965131e 211 fIncludePath(),
bb885a9e 212 fCloseSE(),
0df6ccf2 213 fFriendChainName(),
c6cb3634 214 fJobTag(),
648174cf 215 fOutputSingle(),
5fce53f4 216 fRunPrefix(),
3bdcb562 217 fProofCluster(),
218 fProofDataSet(),
219 fFileForTestMode(),
3bdcb562 220 fAliRootMode(),
d88e4159 221 fProofProcessOpt(),
f47d5cb4 222 fMergeDirName(),
4e5c5506 223 fInputFiles(0),
28ce6c9c 224 fPackages(0),
77f34eae 225 fModules(0),
d88e4159 226 fProofParam(),
0ad3ea1f 227 fDropToShell(true),
715da79c 228 fMCLoop(false),
0ad3ea1f 229 fGridJobIDs(""),
a41b2882 230 fGridStages(""),
780ec36d 231 fFriendLibs(""),
232 fTreeName()
c57f56b7 233{
234// Default ctor.
235 SetDefaults();
236}
237
238//______________________________________________________________________________
239AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
240 :AliAnalysisGrid(other),
241 fGridJDL(NULL),
0f389141 242 fMergingJDL(NULL),
c57f56b7 243 fPrice(other.fPrice),
244 fTTL(other.fTTL),
245 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
246 fMaxInitFailed(other.fMaxInitFailed),
247 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 248 fNtestFiles(other.fNtestFiles),
319593fb 249 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 250 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 251 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 252 fNsubmitted(other.fNsubmitted),
a3e84053 253 fProductionMode(other.fProductionMode),
cd11251e 254 fOutputToRunNo(other.fOutputToRunNo),
0f389141 255 fMergeViaJDL(other.fMergeViaJDL),
256 fFastReadOption(other.fFastReadOption),
257 fOverwriteMode(other.fOverwriteMode),
149d288c 258 fNreplicas(other.fNreplicas),
3bdcb562 259 fNproofWorkers(other.fNproofWorkers),
27734f0e 260 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 261 fProofReset(other.fProofReset),
715da79c 262 fNMCevents(other.fNMCevents),
263 fNMCjobs(other.fNMCjobs),
c57f56b7 264 fRunNumbers(other.fRunNumbers),
265 fExecutable(other.fExecutable),
0a1c1f7f 266 fExecutableCommand(other.fExecutableCommand),
c57f56b7 267 fArguments(other.fArguments),
631c0b05 268 fExecutableArgs(other.fExecutableArgs),
c57f56b7 269 fAnalysisMacro(other.fAnalysisMacro),
270 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 271 fValidationScript(other.fValidationScript),
d5c6455a 272 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 273 fAdditionalLibs(other.fAdditionalLibs),
715da79c 274 fGeneratorLibs(other.fGeneratorLibs),
c57f56b7 275 fSplitMode(other.fSplitMode),
276 fAPIVersion(other.fAPIVersion),
277 fROOTVersion(other.fROOTVersion),
278 fAliROOTVersion(other.fAliROOTVersion),
648174cf 279 fExternalPackages(other.fExternalPackages),
c57f56b7 280 fUser(other.fUser),
281 fGridWorkingDir(other.fGridWorkingDir),
282 fGridDataDir(other.fGridDataDir),
283 fDataPattern(other.fDataPattern),
284 fGridOutputDir(other.fGridOutputDir),
285 fOutputArchive(other.fOutputArchive),
286 fOutputFiles(other.fOutputFiles),
287 fInputFormat(other.fInputFormat),
e7c71df0 288 fDatasetName(other.fDatasetName),
c57f56b7 289 fJDLName(other.fJDLName),
d3b18c4c 290 fTerminateFiles(other.fTerminateFiles),
bb885a9e 291 fMergeExcludes(other.fMergeExcludes),
37d21c01 292 fRegisterExcludes(other.fRegisterExcludes),
f965131e 293 fIncludePath(other.fIncludePath),
bb885a9e 294 fCloseSE(other.fCloseSE),
0df6ccf2 295 fFriendChainName(other.fFriendChainName),
c6cb3634 296 fJobTag(other.fJobTag),
648174cf 297 fOutputSingle(other.fOutputSingle),
5fce53f4 298 fRunPrefix(other.fRunPrefix),
3bdcb562 299 fProofCluster(other.fProofCluster),
300 fProofDataSet(other.fProofDataSet),
301 fFileForTestMode(other.fFileForTestMode),
3bdcb562 302 fAliRootMode(other.fAliRootMode),
d88e4159 303 fProofProcessOpt(other.fProofProcessOpt),
f47d5cb4 304 fMergeDirName(other.fMergeDirName),
4e5c5506 305 fInputFiles(0),
28ce6c9c 306 fPackages(0),
77f34eae 307 fModules(0),
d88e4159 308 fProofParam(),
0ad3ea1f 309 fDropToShell(other.fDropToShell),
715da79c 310 fMCLoop(other.fMCLoop),
0ad3ea1f 311 fGridJobIDs(other.fGridJobIDs),
a41b2882 312 fGridStages(other.fGridStages),
780ec36d 313 fFriendLibs(other.fFriendLibs),
314 fTreeName(other.fTreeName)
c57f56b7 315{
316// Copy ctor.
317 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 318 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 319 fRunRange[0] = other.fRunRange[0];
320 fRunRange[1] = other.fRunRange[1];
c57f56b7 321 if (other.fInputFiles) {
322 fInputFiles = new TObjArray();
323 TIter next(other.fInputFiles);
324 TObject *obj;
325 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
326 fInputFiles->SetOwner();
327 }
4e5c5506 328 if (other.fPackages) {
329 fPackages = new TObjArray();
330 TIter next(other.fPackages);
331 TObject *obj;
332 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
333 fPackages->SetOwner();
334 }
77f34eae 335 if (other.fModules) {
336 fModules = new TObjArray();
337 fModules->SetOwner();
338 TIter next(other.fModules);
339 AliAnalysisTaskCfg *mod, *crt;
340 while ((crt=(AliAnalysisTaskCfg*)next())) {
341 mod = new AliAnalysisTaskCfg(*crt);
342 fModules->Add(mod);
343 }
344 }
c57f56b7 345}
346
347//______________________________________________________________________________
348AliAnalysisAlien::~AliAnalysisAlien()
349{
350// Destructor.
77f34eae 351 delete fGridJDL;
352 delete fMergingJDL;
353 delete fInputFiles;
354 delete fPackages;
355 delete fModules;
28ce6c9c 356 fProofParam.DeleteAll();
c57f56b7 357}
358
359//______________________________________________________________________________
360AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
361{
362// Assignment.
363 if (this != &other) {
364 AliAnalysisGrid::operator=(other);
365 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 366 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 367 fPrice = other.fPrice;
368 fTTL = other.fTTL;
369 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
370 fMaxInitFailed = other.fMaxInitFailed;
371 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 372 fNtestFiles = other.fNtestFiles;
a3e84053 373 fNrunsPerMaster = other.fNrunsPerMaster;
374 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 375 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 376 fNsubmitted = other.fNsubmitted;
377 fProductionMode = other.fProductionMode;
cd11251e 378 fOutputToRunNo = other.fOutputToRunNo;
0f389141 379 fMergeViaJDL = other.fMergeViaJDL;
380 fFastReadOption = other.fFastReadOption;
381 fOverwriteMode = other.fOverwriteMode;
149d288c 382 fNreplicas = other.fNreplicas;
3bdcb562 383 fNproofWorkers = other.fNproofWorkers;
27734f0e 384 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 385 fProofReset = other.fProofReset;
715da79c 386 fNMCevents = other.fNMCevents;
387 fNMCjobs = other.fNMCjobs;
c57f56b7 388 fRunNumbers = other.fRunNumbers;
389 fExecutable = other.fExecutable;
0a1c1f7f 390 fExecutableCommand = other.fExecutableCommand;
c57f56b7 391 fArguments = other.fArguments;
631c0b05 392 fExecutableArgs = other.fExecutableArgs;
c57f56b7 393 fAnalysisMacro = other.fAnalysisMacro;
394 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 395 fValidationScript = other.fValidationScript;
d5c6455a 396 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 397 fAdditionalLibs = other.fAdditionalLibs;
715da79c 398 fGeneratorLibs = other.fGeneratorLibs;
c57f56b7 399 fSplitMode = other.fSplitMode;
400 fAPIVersion = other.fAPIVersion;
401 fROOTVersion = other.fROOTVersion;
402 fAliROOTVersion = other.fAliROOTVersion;
648174cf 403 fExternalPackages = other.fExternalPackages;
c57f56b7 404 fUser = other.fUser;
405 fGridWorkingDir = other.fGridWorkingDir;
406 fGridDataDir = other.fGridDataDir;
407 fDataPattern = other.fDataPattern;
408 fGridOutputDir = other.fGridOutputDir;
409 fOutputArchive = other.fOutputArchive;
410 fOutputFiles = other.fOutputFiles;
411 fInputFormat = other.fInputFormat;
e7c71df0 412 fDatasetName = other.fDatasetName;
c57f56b7 413 fJDLName = other.fJDLName;
d3b18c4c 414 fTerminateFiles = other.fTerminateFiles;
bb885a9e 415 fMergeExcludes = other.fMergeExcludes;
37d21c01 416 fRegisterExcludes = other.fRegisterExcludes;
f965131e 417 fIncludePath = other.fIncludePath;
bb885a9e 418 fCloseSE = other.fCloseSE;
0df6ccf2 419 fFriendChainName = other.fFriendChainName;
c6cb3634 420 fJobTag = other.fJobTag;
648174cf 421 fOutputSingle = other.fOutputSingle;
5fce53f4 422 fRunPrefix = other.fRunPrefix;
3bdcb562 423 fProofCluster = other.fProofCluster;
424 fProofDataSet = other.fProofDataSet;
425 fFileForTestMode = other.fFileForTestMode;
3bdcb562 426 fAliRootMode = other.fAliRootMode;
d88e4159 427 fProofProcessOpt = other.fProofProcessOpt;
f47d5cb4 428 fMergeDirName = other.fMergeDirName;
d88e4159 429 fDropToShell = other.fDropToShell;
715da79c 430 fMCLoop = other.fMCLoop;
0ad3ea1f 431 fGridJobIDs = other.fGridJobIDs;
432 fGridStages = other.fGridStages;
a41b2882 433 fFriendLibs = other.fFriendLibs;
780ec36d 434 fTreeName = other.fTreeName;
c57f56b7 435 if (other.fInputFiles) {
436 fInputFiles = new TObjArray();
437 TIter next(other.fInputFiles);
438 TObject *obj;
439 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
440 fInputFiles->SetOwner();
441 }
4e5c5506 442 if (other.fPackages) {
443 fPackages = new TObjArray();
444 TIter next(other.fPackages);
445 TObject *obj;
446 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
447 fPackages->SetOwner();
448 }
77f34eae 449 if (other.fModules) {
450 fModules = new TObjArray();
451 fModules->SetOwner();
452 TIter next(other.fModules);
453 AliAnalysisTaskCfg *mod, *crt;
454 while ((crt=(AliAnalysisTaskCfg*)next())) {
455 mod = new AliAnalysisTaskCfg(*crt);
456 fModules->Add(mod);
457 }
458 }
c57f56b7 459 }
460 return *this;
461}
462
37d21c01 463//______________________________________________________________________________
464void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
465{
466// Add a single additional library to be loaded. Extension must be present.
467 TString lib(name);
468 if (!lib.Contains(".")) {
469 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
470 return;
471 }
472 if (fAdditionalLibs.Contains(name)) {
473 Warning("AddAdditionalLibrary", "Library %s already added.", name);
474 return;
475 }
476 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
477 fAdditionalLibs += lib;
478}
479
77f34eae 480//______________________________________________________________________________
481void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
482{
483// Adding a module. Checks if already existing. Becomes owned by this.
484 if (!module) return;
485 if (GetModule(module->GetName())) {
486 Error("AddModule", "A module having the same name %s already added", module->GetName());
487 return;
488 }
489 if (!fModules) {
490 fModules = new TObjArray();
491 fModules->SetOwner();
492 }
493 fModules->Add(module);
494}
495
496//______________________________________________________________________________
497void AliAnalysisAlien::AddModules(TObjArray *list)
498{
499// Adding a list of modules. Checks if already existing. Becomes owned by this.
500 TIter next(list);
501 AliAnalysisTaskCfg *module;
502 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
503}
504
505//______________________________________________________________________________
506Bool_t AliAnalysisAlien::CheckDependencies()
507{
508// Check if all dependencies are satisfied. Reorder modules if needed.
509 Int_t nmodules = GetNmodules();
510 if (!nmodules) {
511 Warning("CheckDependencies", "No modules added yet to check their dependencies");
512 return kTRUE;
513 }
514 AliAnalysisTaskCfg *mod = 0;
515 AliAnalysisTaskCfg *dep = 0;
516 TString depname;
517 Int_t i, j, k;
518 for (i=0; i<nmodules; i++) {
519 mod = (AliAnalysisTaskCfg*) fModules->At(i);
520 Int_t ndeps = mod->GetNdeps();
521 Int_t istart = i;
522 for (j=0; j<ndeps; j++) {
523 depname = mod->GetDependency(j);
524 dep = GetModule(depname);
525 if (!dep) {
526 Error("CheckDependencies","Dependency %s not added for module %s",
527 depname.Data(), mod->GetName());
528 return kFALSE;
529 }
530 if (dep->NeedsDependency(mod->GetName())) {
531 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
532 mod->GetName(), dep->GetName());
533 return kFALSE;
534 }
535 Int_t idep = fModules->IndexOf(dep);
536 // The dependency task must come first
537 if (idep>i) {
538 // Remove at idep and move all objects below up one slot
539 // down to index i included.
540 fModules->RemoveAt(idep);
3e40fd4c 541 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 542 fModules->AddAt(dep, i++);
543 }
544 //Redo from istart if dependencies were inserted
545 if (i>istart) i=istart-1;
546 }
547 }
548 return kTRUE;
549}
550
3e40fd4c 551//______________________________________________________________________________
552AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
553{
554// Create the analysis manager and optionally execute the macro in filename.
555 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
715da79c 556 if (mgr) {
557 mgr->SetMCLoop(fMCLoop);
558 return mgr;
559 }
3e40fd4c 560 mgr = new AliAnalysisManager(name);
561 mgr->SetGridHandler((AliAnalysisGrid*)this);
715da79c 562 mgr->SetMCLoop(fMCLoop);
3e40fd4c 563 if (strlen(filename)) {
564 TString line = gSystem->ExpandPathName(filename);
565 line.Prepend(".x ");
566 gROOT->ProcessLine(line.Data());
567 }
568 return mgr;
569}
570
77f34eae 571//______________________________________________________________________________
572Int_t AliAnalysisAlien::GetNmodules() const
573{
574// Get number of modules.
575 if (!fModules) return 0;
576 return fModules->GetEntries();
577}
578
579//______________________________________________________________________________
580AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
581{
582// Get a module by name.
583 if (!fModules) return 0;
584 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
585}
586
587//______________________________________________________________________________
588Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
589{
590// Load a given module.
591 if (mod->IsLoaded()) return kTRUE;
192ddca2 592 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
593 if (!mgr) {
594 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
595 return kFALSE;
596 }
77f34eae 597 Int_t ndeps = mod->GetNdeps();
598 TString depname;
599 for (Int_t j=0; j<ndeps; j++) {
600 depname = mod->GetDependency(j);
601 AliAnalysisTaskCfg *dep = GetModule(depname);
602 if (!dep) {
603 Error("LoadModule","Dependency %s not existing for module %s",
604 depname.Data(), mod->GetName());
605 return kFALSE;
606 }
607 if (!LoadModule(dep)) {
608 Error("LoadModule","Dependency %s for module %s could not be loaded",
609 depname.Data(), mod->GetName());
610 return kFALSE;
611 }
612 }
613 // Load libraries for the module
614 if (!mod->CheckLoadLibraries()) {
615 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
616 return kFALSE;
617 }
192ddca2 618 // Check if a custom file name was requested
619 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
620
621 // Check if a custom terminate file name was requested
622 if (strlen(mod->GetTerminateFileName())) {
623 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
624 fTerminateFiles += mod->GetTerminateFileName();
625 }
626
77f34eae 627 // Execute the macro
628 if (mod->ExecuteMacro()<0) {
629 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
630 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
631 return kFALSE;
632 }
633 // Configure dependencies
634 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
635 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
636 mod->GetConfigMacro()->GetTitle(), mod->GetName());
637 return kFALSE;
638 }
5e8c7c22 639 // Adjust extra libraries
640 Int_t nlibs = mod->GetNlibs();
641 TString lib;
642 for (Int_t i=0; i<nlibs; i++) {
643 lib = mod->GetLibrary(i);
5e8c7c22 644 lib = Form("lib%s.so", lib.Data());
c9d6bd12 645 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 646 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
647 fAdditionalLibs += lib;
648 }
77f34eae 649 return kTRUE;
650}
651
139fe6ba 652//______________________________________________________________________________
653Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
654{
655// Generate the full train.
656 fAdditionalLibs = "";
657 if (!LoadModules()) return kFALSE;
658 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
659 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 660 mgr->RunLocalInit();
139fe6ba 661 mgr->PrintStatus();
662 Int_t productionMode = fProductionMode;
663 SetProductionMode();
664 TString macro = fAnalysisMacro;
665 TString executable = fExecutable;
666 TString validation = fValidationScript;
667 TString execCommand = fExecutableCommand;
668 SetAnalysisMacro(Form("%s.C", name));
669 SetExecutable(Form("%s.sh", name));
a8e31fb0 670// SetExecutableCommand("aliroot -b -q ");
139fe6ba 671 SetValidationScript(Form("%s_validation.sh", name));
672 StartAnalysis();
673 SetProductionMode(productionMode);
674 fAnalysisMacro = macro;
675 fExecutable = executable;
676 fExecutableCommand = execCommand;
677 fValidationScript = validation;
678 return kTRUE;
679}
680
77f34eae 681//______________________________________________________________________________
21ca8e59 682Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 683{
684// Generate test macros for a single module or for the full train.
5e8c7c22 685 fAdditionalLibs = "";
77f34eae 686 if (strlen(modname)) {
21ca8e59 687 if (!CheckDependencies()) return kFALSE;
688 AliAnalysisTaskCfg *mod = GetModule(modname);
689 if (!mod) {
690 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
691 return kFALSE;
692 }
693 if (!LoadModule(mod)) return kFALSE;
a41b2882 694 if (!LoadFriendLibs()) return kFALSE;
21ca8e59 695 } else if (!LoadModules()) return kFALSE;
696 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
697 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 698 mgr->RunLocalInit();
21ca8e59 699 mgr->PrintStatus();
700 SetLocalTest(kTRUE);
701 Int_t productionMode = fProductionMode;
702 SetProductionMode();
703 TString macro = fAnalysisMacro;
704 TString executable = fExecutable;
705 TString validation = fValidationScript;
706 TString execCommand = fExecutableCommand;
707 SetAnalysisMacro(Form("%s.C", name));
708 SetExecutable(Form("%s.sh", name));
d66aa2a9 709 fOutputFiles = GetListOfFiles("outaod");
710 // Add extra files registered to the analysis manager
711 TString extra = GetListOfFiles("ext");
712 if (!extra.IsNull()) {
713 extra.ReplaceAll(".root", "*.root");
714 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
715 fOutputFiles += extra;
716 }
a8e31fb0 717// SetExecutableCommand("aliroot -b -q ");
21ca8e59 718 SetValidationScript(Form("%s_validation.sh", name));
719 WriteAnalysisFile();
720 WriteAnalysisMacro();
721 WriteExecutable();
722 WriteValidationScript();
ffbe06b9 723 WriteMergingMacro();
724 WriteMergeExecutable();
725 WriteValidationScript(kTRUE);
21ca8e59 726 SetLocalTest(kFALSE);
727 SetProductionMode(productionMode);
728 fAnalysisMacro = macro;
729 fExecutable = executable;
730 fExecutableCommand = execCommand;
731 fValidationScript = validation;
77f34eae 732 return kTRUE;
733}
734
735//______________________________________________________________________________
736Bool_t AliAnalysisAlien::LoadModules()
737{
738// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 739 fAdditionalLibs = "";
21ca8e59 740 Int_t nmodules = GetNmodules();
741 if (!nmodules) {
742 Warning("LoadModules", "No module to be loaded");
743 return kTRUE;
744 }
3e40fd4c 745 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
746 if (!mgr) {
747 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
748 return kFALSE;
749 }
77f34eae 750 if (!CheckDependencies()) return kFALSE;
21ca8e59 751 nmodules = GetNmodules();
77f34eae 752 AliAnalysisTaskCfg *mod;
21ca8e59 753 for (Int_t imod=0; imod<nmodules; imod++) {
754 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 755 if (!LoadModule(mod)) return kFALSE;
756 }
a41b2882 757 // Load additional friend libraries
758 return LoadFriendLibs();
77f34eae 759}
760
a41b2882 761//______________________________________________________________________________
762Bool_t AliAnalysisAlien::LoadFriendLibs() const
763{
764// Load libraries required for reading friends.
765 if (fFriendLibs.Length()) {
766 TObjArray *list = 0;
767 TString lib;
768 if (fFriendLibs.Contains(",")) list = fFriendLibs.Tokenize(",");
769 else list = fFriendLibs.Tokenize(" ");
770 for (Int_t ilib=0; ilib<list->GetEntriesFast(); ilib++) {
771 lib = list->At(ilib)->GetName();
772 lib.ReplaceAll(".so","");
2dffffd6 773 lib.ReplaceAll(".dylib","");
a41b2882 774 lib.ReplaceAll(" ","");
775 if (lib.BeginsWith("lib")) lib.Remove(0, 3);
776 lib.Prepend("lib");
777 Int_t loaded = strlen(gSystem->GetLibraries(lib,"",kFALSE));
778 if (!loaded) loaded = gSystem->Load(lib);
779 if (loaded < 0) {
780 Error("LoadModules", "Cannot load library for friends %s", lib.Data());
781 return kFALSE;
782 }
783 }
784 delete list;
785 }
786 return kTRUE;
787}
788
74013535 789//______________________________________________________________________________
790void AliAnalysisAlien::SetRunPrefix(const char *prefix)
791{
792// Set the run number format. Can be a prefix or a format like "%09d"
793 fRunPrefix = prefix;
794 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
795}
796
f965131e 797//______________________________________________________________________________
798void AliAnalysisAlien::AddIncludePath(const char *path)
799{
800// Add include path in the remote analysis macro.
801 TString p(path);
802 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
803 else fIncludePath += Form("-I%s ", path);
804}
805
c57f56b7 806//______________________________________________________________________________
807void AliAnalysisAlien::AddRunNumber(Int_t run)
808{
809// Add a run number to the list of runs to be processed.
810 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 811 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 812}
813
98690895 814//______________________________________________________________________________
815void AliAnalysisAlien::AddRunList(const char* runList)
816{
817// Add several runs into the list of runs; they are expected to be separated by a blank character.
818 TString sList = runList;
819 TObjArray *list = sList.Tokenize(" ");
820 Int_t n = list->GetEntries();
821 for (Int_t i = 0; i < n; i++) {
822 TObjString *os = (TObjString*)list->At(i);
823 AddRunNumber(os->GetString().Atoi());
824 }
825 delete list;
826}
827
ee75cfc3 828//______________________________________________________________________________
829void AliAnalysisAlien::AddRunNumber(const char* run)
830{
831// Add a run number to the list of runs to be processed.
7aa5d539 832 TString runs = run;
833 TObjString *os;
834 TObjArray *arr = runs.Tokenize(" ");
835 TIter next(arr);
836 TString prefix;
837 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
838 while ((os=(TObjString*)next())){
839 if (fRunNumbers.Length()) fRunNumbers += " ";
840 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
841 }
842 delete arr;
ee75cfc3 843}
844
c57f56b7 845//______________________________________________________________________________
846void AliAnalysisAlien::AddDataFile(const char *lfn)
847{
848// Adds a data file to the input to be analysed. The file should be a valid LFN
849// or point to an existing file in the alien workdir.
850 if (!fInputFiles) fInputFiles = new TObjArray();
851 fInputFiles->Add(new TObjString(lfn));
852}
648174cf 853
854//______________________________________________________________________________
855void AliAnalysisAlien::AddExternalPackage(const char *package)
856{
857// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
858 if (fExternalPackages) fExternalPackages += " ";
859 fExternalPackages += package;
860}
861
c57f56b7 862//______________________________________________________________________________
863Bool_t AliAnalysisAlien::Connect()
864{
865// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
866 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 867 if (fProductionMode) return kTRUE;
c57f56b7 868 if (!gGrid) {
869 Info("Connect", "Trying to connect to AliEn ...");
870 TGrid::Connect("alien://");
871 }
872 if (!gGrid || !gGrid->IsConnected()) {
873 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
874 return kFALSE;
875 }
876 fUser = gGrid->GetUser();
877 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
878 return kTRUE;
879}
880
881//______________________________________________________________________________
882void AliAnalysisAlien::CdWork()
883{
884// Check validity of alien workspace. Create directory if possible.
885 if (!Connect()) {
886 Error("CdWork", "Alien connection required");
887 return;
888 }
889 TString homedir = gGrid->GetHomeDirectory();
890 TString workdir = homedir + fGridWorkingDir;
923e2ca5 891 if (DirectoryExists(workdir)) {
892 gGrid->Cd(workdir);
893 return;
894 }
895 // Work directory not existing - create it
896 gGrid->Cd(homedir);
b93f8109 897 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 898 gGrid->Cd(fGridWorkingDir);
d3b18c4c 899 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 900 } else {
d3b18c4c 901 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 902 workdir.Data(), homedir.Data());
903 fGridWorkingDir = "";
904 }
c57f56b7 905}
906
348be253 907//______________________________________________________________________________
908Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
909{
910// Check if file copying is possible.
d3b18c4c 911 if (fProductionMode) return kTRUE;
b5ae4fe0 912 TString salienpath(alienpath);
913 if (salienpath.Contains(" ")) {
914 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
915 return kFALSE;
916 }
348be253 917 if (!Connect()) {
918 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
919 return kFALSE;
920 }
d3339be3 921 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
922 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 923 // Check if alien_CLOSE_SE is defined
924 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
925 if (!closeSE.IsNull()) {
926 Info("CheckFileCopy", "Your current close storage is pointing to: \
927 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
928 } else {
929 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
930 }
931 // Check if grid directory exists.
932 if (!DirectoryExists(alienpath)) {
933 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
934 return kFALSE;
935 }
b5ae4fe0 936 TString stest = "plugin_test_copy";
937 TFile f(stest, "RECREATE");
348be253 938 // User may not have write permissions to current directory
939 if (f.IsZombie()) {
940 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
941 gSystem->WorkingDirectory());
942 return kFALSE;
943 }
944 f.Close();
b5ae4fe0 945 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
946 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 947 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 948 \n# 1. Make sure you have write permissions there. If this is the case: \
949 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
950 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
951 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
952 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 953 gSystem->Unlink(stest.Data());
348be253 954 return kFALSE;
955 }
b5ae4fe0 956 gSystem->Unlink(stest.Data());
957 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 958 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 959 return kTRUE;
960}
961
c57f56b7 962//______________________________________________________________________________
963Bool_t AliAnalysisAlien::CheckInputData()
964{
965// Check validity of input data. If necessary, create xml files.
d3b18c4c 966 if (fProductionMode) return kTRUE;
a8739e8a 967 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 968 if (!fGridDataDir.Length()) {
969 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
970 return kFALSE;
971 }
9f020952 972 if (fMergeViaJDL) {
973 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
974 return kFALSE;
975 }
d2a409b2 976 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 977 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
978 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 979 return kTRUE;
c57f56b7 980 }
981 // Process declared files
a2f5fc01 982 Bool_t isCollection = kFALSE;
983 Bool_t isXml = kFALSE;
984 Bool_t useTags = kFALSE;
c57f56b7 985 Bool_t checked = kFALSE;
d3b18c4c 986 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 987 TString file;
988 TString workdir = gGrid->GetHomeDirectory();
989 workdir += fGridWorkingDir;
990 if (fInputFiles) {
991 TObjString *objstr;
992 TIter next(fInputFiles);
993 while ((objstr=(TObjString*)next())) {
994 file = workdir;
995 file += "/";
996 file += objstr->GetString();
997 // Store full lfn path
998 if (FileExists(file)) objstr->SetString(file);
999 else {
1000 file = objstr->GetName();
1001 if (!FileExists(objstr->GetName())) {
1002 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
1003 objstr->GetName(), workdir.Data());
1004 return kFALSE;
1005 }
1006 }
1007 Bool_t iscoll, isxml, usetags;
1008 CheckDataType(file, iscoll, isxml, usetags);
1009 if (!checked) {
1010 checked = kTRUE;
a2f5fc01 1011 isCollection = iscoll;
1012 isXml = isxml;
1013 useTags = usetags;
1014 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1015 } else {
a2f5fc01 1016 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 1017 Error("CheckInputData", "Some conflict was found in the types of inputs");
1018 return kFALSE;
1019 }
1020 }
1021 }
1022 }
1023 // Process requested run numbers
a8739e8a 1024 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 1025 // Check validity of alien data directory
1026 if (!fGridDataDir.Length()) {
1027 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
1028 return kFALSE;
1029 }
923e2ca5 1030 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 1031 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
1032 return kFALSE;
1033 }
a2f5fc01 1034 if (isCollection) {
c57f56b7 1035 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
1036 return kFALSE;
1037 }
1038
a2f5fc01 1039 if (checked && !isXml) {
c57f56b7 1040 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
1041 return kFALSE;
1042 }
1043 // Check validity of run number(s)
1044 TObjArray *arr;
1045 TObjString *os;
74013535 1046 TString format;
319593fb 1047 Int_t nruns = 0;
904f9f5f 1048 TString schunk, schunk2;
c57f56b7 1049 TString path;
1050 if (!checked) {
1051 checked = kTRUE;
a2f5fc01 1052 useTags = fDataPattern.Contains("tag");
1053 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1054 }
a2f5fc01 1055 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 1056 Error("CheckInputData", "Cannot mix input files using/not using tags");
1057 return kFALSE;
1058 }
1059 if (fRunNumbers.Length()) {
a8739e8a 1060 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 1061 arr = fRunNumbers.Tokenize(" ");
1062 TIter next(arr);
1063 while ((os=(TObjString*)next())) {
1064 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1065 if (!DirectoryExists(path)) {
1066 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 1067 continue;
c57f56b7 1068 }
1069 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
1070 TString msg = "\n##### file: ";
1071 msg += path;
1072 msg += " type: xml_collection;";
a2f5fc01 1073 if (useTags) msg += " using_tags: Yes";
c57f56b7 1074 else msg += " using_tags: No";
23329835 1075 Info("CheckDataType", "%s", msg.Data());
319593fb 1076 if (fNrunsPerMaster<2) {
d2a409b2 1077 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 1078 } else {
1079 nruns++;
1080 if (((nruns-1)%fNrunsPerMaster) == 0) {
1081 schunk = os->GetString();
1082 }
1083 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1084 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1085 AddDataFile(schunk);
319593fb 1086 }
c57f56b7 1087 }
1088 delete arr;
a8739e8a 1089 } else {
1090 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1091 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1092 format = Form("%%s/%s ", fRunPrefix.Data());
1093 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1094 if (!DirectoryExists(path)) {
a8739e8a 1095 continue;
1096 }
74013535 1097 format = Form("%%s/%s.xml", fRunPrefix.Data());
1098 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1099 TString msg = "\n##### file: ";
1100 msg += path;
1101 msg += " type: xml_collection;";
a2f5fc01 1102 if (useTags) msg += " using_tags: Yes";
a8739e8a 1103 else msg += " using_tags: No";
23329835 1104 Info("CheckDataType", "%s", msg.Data());
319593fb 1105 if (fNrunsPerMaster<2) {
74013535 1106 format = Form("%s.xml", fRunPrefix.Data());
1107 AddDataFile(Form(format.Data(),irun));
319593fb 1108 } else {
1109 nruns++;
1110 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1111 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1112 }
74013535 1113 format = Form("_%s.xml", fRunPrefix.Data());
1114 schunk2 = Form(format.Data(), irun);
319593fb 1115 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1116 schunk += schunk2;
d2a409b2 1117 AddDataFile(schunk);
319593fb 1118 }
a8739e8a 1119 }
904f9f5f 1120 if (!fInputFiles) {
1121 schunk += schunk2;
1122 AddDataFile(schunk);
1123 }
c57f56b7 1124 }
1125 return kTRUE;
1126}
1127
21ca8e59 1128//______________________________________________________________________________
4a8b667c 1129Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1130{
1131// Copy data from the given grid directory according a pattern and make a local
1132// dataset.
2ba4c287 1133// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1134 if (!Connect()) {
1135 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
4a8b667c 1136 return 0;
21ca8e59 1137 }
1138 if (!DirectoryExists(griddir)) {
1139 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
4a8b667c 1140 return 0;
21ca8e59 1141 }
1142 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1143 printf("Running command: %s\n", command.Data());
1144 TGridResult *res = gGrid->Command(command);
1145 Int_t nfound = res->GetEntries();
1146 if (!nfound) {
1147 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
4a8b667c 1148 return 0;
21ca8e59 1149 }
1150 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1151
1152 // archives
1153 TObjArray* additionalArchives = 0;
1154 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1155 additionalArchives = TString(archivefile).Tokenize(";");
1156 archivefile = additionalArchives->At(0)->GetName();
1157 additionalArchives->RemoveAt(0);
1158 additionalArchives->Compress();
1159 }
1160
21ca8e59 1161 // Copy files locally
1162 ofstream out;
1163 out.open(output, ios::out);
1164 TMap *map;
1165 TString turl, dirname, filename, temp;
1166 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1167 gSystem->MakeDirectory(outputdir);
1168 gSystem->ChangeDirectory(outputdir);
4a8b667c 1169 Int_t ncopied = 0;
21ca8e59 1170 for (Int_t i=0; i<nfound; i++) {
1171 map = (TMap*)res->At(i);
1172 turl = map->GetValue("turl")->GetName();
1173 filename = gSystem->BaseName(turl.Data());
1174 dirname = gSystem->DirName(turl.Data());
1175 dirname = gSystem->BaseName(dirname.Data());
1176 gSystem->MakeDirectory(dirname);
43b807f6 1177
1178 TString source(turl);
1179 TString targetFileName(filename);
1180
1181 if (strlen(archivefile) > 0) {
4a8b667c 1182// TODO here the archive in which the file resides should be determined
1183// however whereis returns only a guid, and guid2lfn does not work
1184// Therefore we use the one provided as argument for now
1185 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1186 targetFileName = archivefile;
43b807f6 1187 }
1188 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
4a8b667c 1189 Bool_t success = kTRUE;
1190 if (additionalArchives) {
1191 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1192 TString target;
1193 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1194 gSystem->MakeDirectory(gSystem->DirName(target));
1195 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1196 }
1197 }
2ba4c287 1198
4a8b667c 1199 if (success) {
1200 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1201 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1202 ncopied++;
1203 }
21ca8e59 1204 }
1205 }
1206 gSystem->ChangeDirectory(cdir);
1207 delete res;
4a8b667c 1208 delete additionalArchives;
1209 return ncopied;
21ca8e59 1210}
1211
c57f56b7 1212//______________________________________________________________________________
1213Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1214{
1215// Create dataset for the grid data directory + run number.
3af861ff 1216 const Int_t gMaxEntries = 15000;
d3b18c4c 1217 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1218 if (!Connect()) {
1219 Error("CreateDataset", "Cannot create dataset with no grid connection");
1220 return kFALSE;
1221 }
1222
1223 // Cd workspace
d3b18c4c 1224 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1225 TString workdir = gGrid->GetHomeDirectory();
1226 workdir += fGridWorkingDir;
1227
1228 // Compose the 'find' command arguments
74013535 1229 TString format;
c57f56b7 1230 TString command;
780ec36d 1231 TString delimiter = pattern;
1232 delimiter.Strip();
1233 if (delimiter.Contains(" ")) delimiter = "";
1234 else delimiter = " ";
c57f56b7 1235 TString options = "-x collection ";
bb885a9e 1236 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1237 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1238 TString conditions = "";
3af861ff 1239 Int_t nstart = 0;
1240 Int_t ncount = 0;
1241 Int_t stage = 0;
c57f56b7 1242 TString file;
1243 TString path;
319593fb 1244 Int_t nruns = 0;
904f9f5f 1245 TString schunk, schunk2;
ab254fd1 1246 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1247 if (!fRunNumbers.Length() && !fRunRange[0]) {
1248 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1249 // Make a single data collection from data directory.
1250 path = fGridDataDir;
923e2ca5 1251 if (!DirectoryExists(path)) {
d2a409b2 1252 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1253 return kFALSE;
780ec36d 1254 }
923e2ca5 1255// CdWork();
d2a409b2 1256 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1257 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1258 while (1) {
1259 ncount = 0;
1260 stage++;
1261 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1262 command = "find ";
1263 command += Form("%s -o %d ",options.Data(), nstart);
1264 command += path;
780ec36d 1265 command += delimiter;
3af861ff 1266 command += pattern;
1267 command += conditions;
1268 printf("command: %s\n", command.Data());
1269 TGridResult *res = gGrid->Command(command);
1270 if (res) delete res;
1271 // Write standard output to file
1272 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1273 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1274 Bool_t nullFile = kFALSE;
1275 if (!hasGrep) {
1276 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1277 } else {
1278 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1279 if (nullFile) {
1280 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1281 gSystem->Exec("rm -f __tmp*");
1282 return kFALSE;
1283 }
1284 TString line;
1285 ifstream in;
1286 in.open("__tmp__");
1287 in >> line;
1288 in.close();
1289 gSystem->Exec("rm -f __tmp__");
1290 ncount = line.Atoi();
1291 }
1292 }
1293 if (ncount == gMaxEntries) {
1294 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1295 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1296 if (!cbase) cbase = cadd;
1297 else {
1298 cbase->Add(cadd);
1299 delete cadd;
defd7a3a 1300 }
3af861ff 1301 nstart += ncount;
1302 } else {
1303 if (cbase) {
1304 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1305 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1306 cbase->Add(cadd);
1307 delete cadd;
1308 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1309 delete cbase; cbase = 0;
1310 } else {
1311 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1312 }
1313 gSystem->Exec("rm -f __tmp*");
1314 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1315 break;
1316 }
0f389141 1317 }
1318 Bool_t fileExists = FileExists(file);
1319 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1320 // Copy xml file to alien space
0f389141 1321 if (fileExists) gGrid->Rm(file);
d2a409b2 1322 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1323 if (!FileExists(file)) {
1324 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1325 return kFALSE;
1326 }
1327 // Update list of files to be processed.
1328 }
1329 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1330 return kTRUE;
1331 }
c57f56b7 1332 // Several runs
a2f5fc01 1333 Bool_t nullResult = kTRUE;
a8739e8a 1334 if (fRunNumbers.Length()) {
1335 TObjArray *arr = fRunNumbers.Tokenize(" ");
1336 TObjString *os;
1337 TIter next(arr);
1338 while ((os=(TObjString*)next())) {
3af861ff 1339 nstart = 0;
1340 stage = 0;
0dab9d6d 1341 path = Form("%s/%s", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1342 if (!DirectoryExists(path)) continue;
1343// CdWork();
a8739e8a 1344 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1345 else file = Form("%s.xml", os->GetString().Data());
319593fb 1346 // If local collection file does not exist, create it via 'find' command.
3af861ff 1347 while (1) {
1348 ncount = 0;
1349 stage++;
1350 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1351 command = "find ";
1352 command += Form("%s -o %d ",options.Data(), nstart);
1353 command += path;
780ec36d 1354 command += delimiter;
3af861ff 1355 command += pattern;
1356 command += conditions;
1357 TGridResult *res = gGrid->Command(command);
1358 if (res) delete res;
1359 // Write standard output to file
1360 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1361 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1362 Bool_t nullFile = kFALSE;
1363 if (!hasGrep) {
1364 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1365 } else {
1366 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1367 if (nullFile) {
1368 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1369 gSystem->Exec("rm -f __tmp*");
1370 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1371 break;
3af861ff 1372 }
1373 TString line;
1374 ifstream in;
1375 in.open("__tmp__");
1376 in >> line;
1377 in.close();
1378 gSystem->Exec("rm -f __tmp__");
1379 ncount = line.Atoi();
1380 }
1381 nullResult = kFALSE;
1382 }
1383 if (ncount == gMaxEntries) {
1384 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1385 if (fNrunsPerMaster > 1) {
1386 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1387 file.Data(),gMaxEntries);
1388 return kFALSE;
1389 }
1390 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1391 if (!cbase) cbase = cadd;
1392 else {
1393 cbase->Add(cadd);
1394 delete cadd;
defd7a3a 1395 }
3af861ff 1396 nstart += ncount;
1397 } else {
c89a84bc 1398 if (cbase && fNrunsPerMaster<2) {
3af861ff 1399 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1400 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1401 cbase->Add(cadd);
1402 delete cadd;
1403 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1404 delete cbase; cbase = 0;
1405 } else {
1406 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1407 }
1408 gSystem->Exec("rm -f __tmp*");
1409 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1410 break;
d3339be3 1411 }
3af861ff 1412 }
a8739e8a 1413 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1414 // Check if there is one run per master job.
1415 if (fNrunsPerMaster<2) {
1416 if (FileExists(file)) {
0f389141 1417 if (fOverwriteMode) gGrid->Rm(file);
1418 else {
1419 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1420 continue;
1421 }
319593fb 1422 }
1423 // Copy xml file to alien space
1424 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1425 if (!FileExists(file)) {
1426 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1427 delete arr;
1428 return kFALSE;
1429 }
1430 } else {
1431 nruns++;
1432 if (((nruns-1)%fNrunsPerMaster) == 0) {
1433 schunk = os->GetString();
1434 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1435 } else {
1436 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1437 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1438 cbase->Add(cadd);
1439 delete cadd;
1440 }
1441 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1442 continue;
1443 }
1444 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1445 if (FileExists(schunk)) {
1446 if (fOverwriteMode) gGrid->Rm(file);
1447 else {
1448 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1449 continue;
1450 }
319593fb 1451 }
84fcd93f 1452 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1453 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1454 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1455 if (!FileExists(schunk)) {
1456 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1457 delete arr;
1458 return kFALSE;
1459 }
d3339be3 1460 }
a8739e8a 1461 }
1462 delete arr;
a2f5fc01 1463 if (nullResult) {
d3339be3 1464 Error("CreateDataset", "No valid dataset corresponding to the query!");
1465 return kFALSE;
1466 }
a8739e8a 1467 } else {
1468 // Process a full run range.
1469 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
0dab9d6d 1470 format = Form("%%s/%s", fRunPrefix.Data());
3af861ff 1471 nstart = 0;
1472 stage = 0;
74013535 1473 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1474 if (!DirectoryExists(path)) continue;
1475// CdWork();
74013535 1476 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1477 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1478 else file = Form(format.Data(), irun);
0f389141 1479 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1480 if (fOverwriteMode) gGrid->Rm(file);
1481 else {
1482 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1483 continue;
1484 }
a8739e8a 1485 }
319593fb 1486 // If local collection file does not exist, create it via 'find' command.
3af861ff 1487 while (1) {
1488 ncount = 0;
1489 stage++;
1490 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1491 command = "find ";
1492 command += Form("%s -o %d ",options.Data(), nstart);
1493 command += path;
780ec36d 1494 command += delimiter;
3af861ff 1495 command += pattern;
1496 command += conditions;
1497 TGridResult *res = gGrid->Command(command);
1498 if (res) delete res;
1499 // Write standard output to file
1500 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1501 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1502 Bool_t nullFile = kFALSE;
1503 if (!hasGrep) {
1504 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1505 } else {
1506 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1507 if (nullFile) {
1508 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1509 gSystem->Exec("rm -f __tmp*");
1604c9ed 1510 break;
3af861ff 1511 }
1512 TString line;
1513 ifstream in;
1514 in.open("__tmp__");
1515 in >> line;
1516 in.close();
1517 gSystem->Exec("rm -f __tmp__");
1518 ncount = line.Atoi();
1519 }
1520 nullResult = kFALSE;
1521 }
1522 if (ncount == gMaxEntries) {
1523 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1524 if (fNrunsPerMaster > 1) {
1525 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1526 file.Data(),gMaxEntries);
1527 return kFALSE;
1528 }
1529 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1530 if (!cbase) cbase = cadd;
1531 else {
1532 cbase->Add(cadd);
1533 delete cadd;
defd7a3a 1534 }
3af861ff 1535 nstart += ncount;
1536 } else {
c89a84bc 1537 if (cbase && fNrunsPerMaster<2) {
3af861ff 1538 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1539 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1540 cbase->Add(cadd);
1541 delete cadd;
1542 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1543 delete cbase; cbase = 0;
1544 } else {
1545 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1546 }
1547 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1548 break;
d3339be3 1549 }
319593fb 1550 }
a8739e8a 1551 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1552 // Check if there is one run per master job.
1553 if (fNrunsPerMaster<2) {
1554 if (FileExists(file)) {
0f389141 1555 if (fOverwriteMode) gGrid->Rm(file);
1556 else {
1557 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1558 continue;
1559 }
319593fb 1560 }
1561 // Copy xml file to alien space
1562 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1563 if (!FileExists(file)) {
1564 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1565 return kFALSE;
1566 }
1567 } else {
1568 nruns++;
95e5b448 1569 // Check if the collection for the chunk exist locally.
1570 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1571 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1572 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1573 else continue;
1574 }
84fcd93f 1575 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1576 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1577 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1578 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1579 } else {
1580 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1581 cbase->Add(cadd);
1582 delete cadd;
1583 }
74013535 1584 format = Form("%%s_%s.xml", fRunPrefix.Data());
1585 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1586 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1587 continue;
1588 }
904f9f5f 1589 schunk = schunk2;
319593fb 1590 if (FileExists(schunk)) {
0f389141 1591 if (fOverwriteMode) gGrid->Rm(schunk);
1592 else {
1593 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1594 continue;
1595 }
319593fb 1596 }
84fcd93f 1597 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1598 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1599 if (FileExists(schunk)) {
0f389141 1600 if (fOverwriteMode) gGrid->Rm(schunk);
1601 else {
1602 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1603 continue;
1604 }
95e5b448 1605 }
319593fb 1606 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1607 if (!FileExists(schunk)) {
1608 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1609 return kFALSE;
1610 }
1611 }
c57f56b7 1612 }
a2f5fc01 1613 if (nullResult) {
d3339be3 1614 Error("CreateDataset", "No valid dataset corresponding to the query!");
1615 return kFALSE;
1616 }
a8739e8a 1617 }
c57f56b7 1618 return kTRUE;
1619}
1620
1621//______________________________________________________________________________
1622Bool_t AliAnalysisAlien::CreateJDL()
1623{
1624// Generate a JDL file according to current settings. The name of the file is
1625// specified by fJDLName.
ec5acdce 1626 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
c57f56b7 1627 Bool_t error = kFALSE;
1628 TObjArray *arr = 0;
1629 Bool_t copy = kTRUE;
d3b18c4c 1630 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1631 Bool_t generate = kTRUE;
1632 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1633 if (!Connect()) {
1634 Error("CreateJDL", "Alien connection required");
1635 return kFALSE;
1636 }
1637 // Check validity of alien workspace
d3b18c4c 1638 TString workdir;
1639 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1640 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1641 workdir += fGridWorkingDir;
1642 if (generate) {
1643 TObjString *os;
1e466628 1644 if (!fInputFiles && !fMCLoop) {
c57f56b7 1645 Error("CreateJDL()", "Define some input files for your analysis.");
1646 error = kTRUE;
1647 }
1648 // Compose list of input files
1649 // Check if output files were defined
1650 if (!fOutputFiles.Length()) {
1651 Error("CreateJDL", "You must define at least one output file");
1652 error = kTRUE;
1653 }
1654 // Check if an output directory was defined and valid
1655 if (!fGridOutputDir.Length()) {
1656 Error("CreateJDL", "You must define AliEn output directory");
1657 error = kTRUE;
1658 } else {
d3b18c4c 1659 if (!fProductionMode) {
1660 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1661 if (!DirectoryExists(fGridOutputDir)) {
1662 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1663 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1664 } else {
1665 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1666 // error = kTRUE;
1667 }
67a3b254 1668 } else {
1669 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1670 }
1671 gGrid->Cd(workdir);
1672 }
c57f56b7 1673 }
1674 // Exit if any error up to now
1675 if (error) return kFALSE;
1676 // Set JDL fields
0f389141 1677 if (!fUser.IsNull()) {
1678 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1679 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
6da4839e
AG		 1680 }
1681 TString executable = fExecutable;
1682 if (!executable.BeginsWith("/"))
1683 executable.Prepend(Form("%s/", workdir.Data()));
1684 fGridJDL->SetExecutable(executable, "This is the startup script");
1685 TString mergeExec = executable;
0f389141 1686 mergeExec.ReplaceAll(".sh", "_merge.sh");
1687 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1688 mergeExec.ReplaceAll(".sh", ".C");
9c5cde18 1689 fMergingJDL->AddToInputSandbox(Form("LF:%s", mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1690 if (!fArguments.IsNull())
1691 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1692 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1693 else {
1694 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1695 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1696 }
b3e07543 1697
1f0d1ca2 1698 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1699 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1700 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1701 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1702
0f389141 1703 if (fMaxInitFailed > 0) {
c57f56b7 1704 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1705 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1706 }
6da4839e 1707 if (fSplitMaxInputFileNumber > 0 && !fMCLoop) {
c57f56b7 1708 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1709 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1710 }
d7eb1f39 1711 if (!IsOneStageMerging()) {
f47d5cb4 1712 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1713 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1714 }
1715 if (fSplitMode.Length()) {
c57f56b7 1716 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1717 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1718 }
1719 fMergingJDL->SetValue("Split", "\"se\"");
1720 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1721 if (!fAliROOTVersion.IsNull()) {
1722 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1723 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1724 }
1725 if (!fROOTVersion.IsNull()) {
c57f56b7 1726 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1727 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1728 }
1729 if (!fAPIVersion.IsNull()) {
c57f56b7 1730 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1731 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1732 }
648174cf 1733 if (!fExternalPackages.IsNull()) {
1734 arr = fExternalPackages.Tokenize(" ");
1735 TIter next(arr);
1736 while ((os=(TObjString*)next())) {
1737 TString pkgname = os->GetString();
1738 Int_t index = pkgname.Index("::");
1739 TString pkgversion = pkgname(index+2, pkgname.Length());
1740 pkgname.Remove(index);
1741 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1742 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1743 }
1744 delete arr;
1745 }
6da4839e
AG		 1746 if (!fMCLoop) {
1747 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1748 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
6da4839e 1749 }
d7eb1f39
AG		 1750 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1751 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1752 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1753 TString analysisFile = fExecutable;
1754 analysisFile.ReplaceAll(".sh", ".root");
1755 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1756 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1757 if (fAdditionalLibs.Length()) {
1758 arr = fAdditionalLibs.Tokenize(" ");
1759 TIter next(arr);
1760 while ((os=(TObjString*)next())) {
2dffffd6 1761 if (os->GetString().Contains(".so") ||
1762 os->GetString().Contains(".dylib")) continue;
c57f56b7 1763 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1764 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1765 }
1766 delete arr;
1767 }
4e5c5506 1768 if (fPackages) {
1769 TIter next(fPackages);
1770 TObject *obj;
0f389141 1771 while ((obj=next())) {
4e5c5506 1772 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1773 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1774 }
4e5c5506 1775 }
95e16f01 1776 const char *comment = "List of output files and archives";
c57f56b7 1777 if (fOutputArchive.Length()) {
e4d9a069 1778 TString outputArchive = fOutputArchive;
1779 if (!fRegisterExcludes.IsNull()) {
1780 arr = fRegisterExcludes.Tokenize(" ");
1781 TIter next1(arr);
1782 while ((os=(TObjString*)next1())) {
1783 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1784 outputArchive.ReplaceAll(os->GetString(),"");
1785 }
1786 delete arr;
1787 }
1788 arr = outputArchive.Tokenize(" ");
c57f56b7 1789 TIter next(arr);
0f389141 1790 Bool_t first = kTRUE;
0f389141 1791 while ((os=(TObjString*)next())) {
0f389141 1792 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1793 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
0f389141 1794 else
95e16f01 1795 fGridJDL->AddToSet("Output", os->GetString());
1796 if (first) fGridJDL->AddToSetDescription("Output", comment);
0f389141 1797 first = kFALSE;
1798 }
c57f56b7 1799 delete arr;
f790bc1b 1800 // Output archive for the merging jdl
f790bc1b 1801 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1802 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1803 // Add normal output files, extra files + terminate files
0c0825e4 1804 TString files;
1805 if (IsMergeAOD()) files = GetListOfFiles("outaodextter");
1806 else files = GetListOfFiles("outextter");
37d21c01 1807 // Do not register files in fRegisterExcludes
1808 if (!fRegisterExcludes.IsNull()) {
1809 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1810 TIter next1(arr);
1811 while ((os=(TObjString*)next1())) {
1812 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1813 files.ReplaceAll(os->GetString(),"");
1814 }
66d71516 1815 delete arr;
0f389141 1816 }
f790bc1b 1817 files.ReplaceAll(".root", "*.root");
ec5acdce 1818
1819 if (mgr->IsCollectThroughput())
95e16f01 1820 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",files.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 1821 else
1822 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1823 } else {
95f84779 1824 TString files = fOutputArchive;
1825 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1826 outputArchive = files;
f790bc1b 1827 }
0f389141 1828 arr = outputArchive.Tokenize(" ");
705adb3e 1829 TIter next2(arr);
0f389141 1830 first = kTRUE;
705adb3e 1831 while ((os=(TObjString*)next2())) {
0cdf65a8 1832 TString currentfile = os->GetString();
0cdf65a8 1833 if (!currentfile.Contains("@") && fCloseSE.Length())
95e16f01 1834 fMergingJDL->AddToSet("Output", Form("%s@%s",currentfile.Data(), fCloseSE.Data()));
0f389141 1835 else
95e16f01 1836 fMergingJDL->AddToSet("Output", currentfile);
1837 if (first) fMergingJDL->AddToSetDescription("Output", comment);
0f389141 1838 first = kFALSE;
1839 }
1840 delete arr;
c57f56b7 1841 }
149d288c 1842 arr = fOutputFiles.Tokenize(",");
c57f56b7 1843 TIter next(arr);
0f389141 1844 Bool_t first = kTRUE;
43da816a 1845 while ((os=(TObjString*)next())) {
e1eaf596 1846 // Ignore ouputs in jdl that are also in outputarchive
1847 TString sout = os->GetString();
f790bc1b 1848 sout.ReplaceAll("*", "");
1849 sout.ReplaceAll(".root", "");
e1eaf596 1850 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1851 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1852 // Ignore fRegisterExcludes
1853 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1854 if (!first) comment = NULL;
43da816a 1855 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1856 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
43da816a 1857 else
95e16f01 1858 fGridJDL->AddToSet("Output", os->GetString());
1859 if (first) fGridJDL->AddToSetDescription("Output", comment);
f790bc1b 1860 if (fMergeExcludes.Contains(sout)) continue;
1861 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1862 fMergingJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
f790bc1b 1863 else
95e16f01 1864 fMergingJDL->AddToSet("Output", os->GetString());
1865 if (first) fMergingJDL->AddToSetDescription("Output", comment);
1866 first = kFALSE;
43da816a 1867 }
c57f56b7 1868 delete arr;
0f389141 1869 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1870 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1871 TString validationScript = fValidationScript;
0f389141 1872 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1873 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1874 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1875 if (fMasterResubmitThreshold) {
1876 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1877 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1878 }
d2a409b2 1879 // Write a jdl with 2 input parameters: collection name and output dir name.
1880 WriteJDL(copy);
c57f56b7 1881 }
1882 // Copy jdl to grid workspace
a8739e8a 1883 if (copy) {
b5e4aaa7 1884 // Check if an output directory was defined and valid
1885 if (!fGridOutputDir.Length()) {
1886 Error("CreateJDL", "You must define AliEn output directory");
1887 return kFALSE;
1888 } else {
1889 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1890 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1891 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1892 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1893 } else {
1894 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1895 return kFALSE;
1896 }
1897 }
1898 gGrid->Cd(workdir);
1899 }
648174cf 1900 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1901 TString mergeJDLName = fExecutable;
1902 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1903 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1904 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1905 if (fProductionMode) {
648174cf 1906 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1907 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1908 }
648174cf 1909 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1910 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1911 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1912 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1913 Fatal("","Terminating");
1914// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1915 if (fMergeViaJDL) {
1916 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1917// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1918 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1919 Fatal("","Terminating");
0f389141 1920 }
648174cf 1921 }
c57f56b7 1922 if (fAdditionalLibs.Length()) {
e7c71df0 1923 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1924 TObjString *os;
1925 TIter next(arr);
1926 while ((os=(TObjString*)next())) {
2dffffd6 1927 if (os->GetString().Contains(".so") ||
1928 os->GetString().Contains(".dylib")) continue;
4e5c5506 1929 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1930 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1931// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1932 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1933 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1934 Fatal("","Terminating");
c57f56b7 1935 }
1936 delete arr;
1937 }
4e5c5506 1938 if (fPackages) {
1939 TIter next(fPackages);
1940 TObject *obj;
1941 while ((obj=next())) {
fdbbc7be 1942 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1943 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1944// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1945 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1946 Form("%s/%s", workdir.Data(), obj->GetName())))
1947 Fatal("","Terminating");
4e5c5506 1948 }
1949 }
c57f56b7 1950 }
1951 return kTRUE;
1952}
1953
a8739e8a 1954//______________________________________________________________________________
d2a409b2 1955Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1956{
1957// Writes one or more JDL's corresponding to findex. If findex is negative,
1958// all run numbers are considered in one go (jdl). For non-negative indices
1959// they correspond to the indices in the array fInputFiles.
dec33edc 1960 if (!fInputFiles && !fMCLoop) return kFALSE;
d3b18c4c 1961 TObject *os;
1962 TString workdir;
1963 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1964 workdir += fGridWorkingDir;
f47d5cb4 1965 TString stageName = "$2";
1966 if (fProductionMode) stageName = "$4";
1967 if (!fMergeDirName.IsNull()) {
1968 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1969 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1970 } else {
1971 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1972 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1973 }
d3b18c4c 1974 if (fProductionMode) {
a8739e8a 1975 TIter next(fInputFiles);
b3e07543 1976 while ((os=next())) {
d3b18c4c 1977 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1978 }
56754bf6 1979 if (!fOutputToRunNo)
1980 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1981 else
1982 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1983 } else {
1984 if (!fRunNumbers.Length() && !fRunRange[0]) {
1985 // One jdl with no parameters in case input data is specified by name.
1986 TIter next(fInputFiles);
1987 while ((os=next()))
1988 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1989 if (!fOutputSingle.IsNull())
1990 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1991 else {
1992 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
b1dade98 1993// fMergingJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1994 }
1995 } else {
1996 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1997 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1998 if (!fOutputSingle.IsNull()) {
1999 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
2000 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
2001 } else {
2002 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 2003 }
2004 }
a8739e8a 2005 }
2006
a8739e8a 2007 // Generate the JDL as a string
2008 TString sjdl = fGridJDL->Generate();
0f389141 2009 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 2010 // Final merge jdl
2011 if (!fMergeDirName.IsNull()) {
2012 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
2013 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
2014 } else {
2015 fMergingJDL->SetOutputDirectory("$1", "Output directory");
2016 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
2017 }
b3e07543 2018 TString sjdl2 = fMergingJDL->Generate();
2019 Int_t index, index1;
95e16f01 2020 sjdl.ReplaceAll("\",\"", "\",\n \"");
a8739e8a 2021 sjdl.ReplaceAll("(member", "\n (member");
2022 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2023 sjdl.ReplaceAll("{", "{\n ");
2024 sjdl.ReplaceAll("};", "\n};");
2025 sjdl.ReplaceAll("{\n \n", "{\n");
2026 sjdl.ReplaceAll("\n\n", "\n");
2027 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2028 sjdl1.ReplaceAll("\",\"", "\",\n \"");
0f389141 2029 sjdl1.ReplaceAll("(member", "\n (member");
2030 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2031 sjdl1.ReplaceAll("{", "{\n ");
2032 sjdl1.ReplaceAll("};", "\n};");
2033 sjdl1.ReplaceAll("{\n \n", "{\n");
2034 sjdl1.ReplaceAll("\n\n", "\n");
2035 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2036 sjdl2.ReplaceAll("\",\"", "\",\n \"");
b3e07543 2037 sjdl2.ReplaceAll("(member", "\n (member");
2038 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2039 sjdl2.ReplaceAll("{", "{\n ");
2040 sjdl2.ReplaceAll("};", "\n};");
2041 sjdl2.ReplaceAll("{\n \n", "{\n");
2042 sjdl2.ReplaceAll("\n\n", "\n");
2043 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 2044 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 2045 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 2046 index = sjdl.Index("JDLVariables");
2047 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 2048 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2049 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 2050 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 2051 index = fJobTag.Index(":");
2052 if (index < 0) index = fJobTag.Length();
2053 TString jobTag = fJobTag;
f47d5cb4 2054 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 2055 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 2056 if (fProductionMode) {
2057 sjdl1.Prepend("# Generated merging jdl (production mode) \
2058 \n# $1 = full alien path to output directory to be merged \
2059 \n# $2 = train number \
2060 \n# $3 = production (like LHC10b) \
2061 \n# $4 = merging stage \
2062 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2063 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2064 sjdl2.Prepend("# Generated merging jdl \
2065 \n# $1 = full alien path to output directory to be merged \
2066 \n# $2 = train number \
2067 \n# $3 = production (like LHC10b) \
2068 \n# $4 = merging stage \
2069 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2070 } else {
2071 sjdl1.Prepend("# Generated merging jdl \
2072 \n# $1 = full alien path to output directory to be merged \
2073 \n# $2 = merging stage \
03335324 2074 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2075 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2076 sjdl2.Prepend("# Generated merging jdl \
2077 \n# $1 = full alien path to output directory to be merged \
2078 \n# $2 = merging stage \
03335324 2079 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2080 }
0f389141 2081 index = sjdl1.Index("JDLVariables");
2082 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 2083 index = sjdl2.Index("JDLVariables");
2084 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 2085 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2086 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
2087 index = sjdl2.Index("Split =");
2088 if (index>=0) {
2089 index1 = sjdl2.Index("\n", index);
2090 sjdl2.Remove(index, index1-index+1);
2091 }
2092 index = sjdl2.Index("SplitMaxInputFileNumber");
2093 if (index>=0) {
2094 index1 = sjdl2.Index("\n", index);
2095 sjdl2.Remove(index, index1-index+1);
2096 }
2097 index = sjdl2.Index("InputDataCollection");
2098 if (index>=0) {
2099 index1 = sjdl2.Index(";", index);
2100 sjdl2.Remove(index, index1-index+1);
2101 }
2102 index = sjdl2.Index("InputDataListFormat");
2103 if (index>=0) {
2104 index1 = sjdl2.Index("\n", index);
2105 sjdl2.Remove(index, index1-index+1);
2106 }
2107 index = sjdl2.Index("InputDataList");
2108 if (index>=0) {
2109 index1 = sjdl2.Index("\n", index);
2110 sjdl2.Remove(index, index1-index+1);
2111 }
f47d5cb4 2112 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2113 // Write jdl to file
a8739e8a 2114 ofstream out;
d2a409b2 2115 out.open(fJDLName.Data(), ios::out);
a8739e8a 2116 if (out.bad()) {
d3b18c4c 2117 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2118 return kFALSE;
2119 }
2120 out << sjdl << endl;
b3e07543 2121 out.close();
0f389141 2122 TString mergeJDLName = fExecutable;
2123 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2124 if (fMergeViaJDL) {
2125 ofstream out1;
2126 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2127 if (out1.bad()) {
d3b18c4c 2128 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2129 return kFALSE;
2130 }
2131 out1 << sjdl1 << endl;
b3e07543 2132 out1.close();
2133 ofstream out2;
2134 TString finalJDL = mergeJDLName;
2135 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2136 out2.open(finalJDL.Data(), ios::out);
2137 if (out2.bad()) {
2138 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2139 return kFALSE;
2140 }
2141 out2 << sjdl2 << endl;
2142 out2.close();
0f389141 2143 }
a8739e8a 2144
2145 // Copy jdl to grid workspace
2146 if (!copy) {
d3b18c4c 2147 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2148 } else {
d2a409b2 2149 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2150 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2151 TString finalJDL = mergeJDLName;
2152 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2153 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2154 if (fProductionMode) {
b5fe9cba 2155 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2156 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2157 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2158 }
d2a409b2 2159 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2160 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2161 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2162 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2163// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2164 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2165 Fatal("","Terminating");
0f389141 2166 if (fMergeViaJDL) {
b3e07543 2167 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2168// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2169// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2170 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2171 Fatal("","Terminating");
2172 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2173 Fatal("","Terminating");
0f389141 2174 }
a8739e8a 2175 }
2176 return kTRUE;
2177}
2178
c57f56b7 2179//______________________________________________________________________________
5513444a 2180Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2181{
2182// Returns true if file exists.
5513444a 2183 if (!gGrid) return kFALSE;
2ea55496 2184 TString slfn = lfn;
2185 slfn.ReplaceAll("alien://","");
2186 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2187 if (!res) return kFALSE;
2188 TMap *map = dynamic_cast<TMap*>(res->At(0));
2189 if (!map) {
2190 delete res;
2191 return kFALSE;
2192 }
2193 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2194 if (!objs || !objs->GetString().Length()) {
2195 delete res;
2196 return kFALSE;
2197 }
2198 delete res;
2199 return kTRUE;
2200}
2201
923e2ca5 2202//______________________________________________________________________________
2203Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2204{
2205// Returns true if directory exists. Can be also a path.
d7aac489 2206// Since there is not API in TAlien, we use the Cd trick:
923e2ca5 2207 if (!gGrid) return kFALSE;
d7aac489 2208 // Backup current path
2209 TString cpath = gGrid->Pwd();
2210 TString command = "cd ";
2211 TString sdir(dirname);
2212 sdir.ReplaceAll("alien://", "");
2213 command += sdir;
2214 TGridResult *res = gGrid->Command(command);
2215 if (!res) {
2216 gGrid->Cd(cpath);
2217 return kFALSE;
2218 }
2219 TMap *map = (TMap*)res->At(0);
2220 if (!map) {
2221 gGrid->Cd(cpath);
2222 delete res;
2223 return kFALSE;
923e2ca5 2224 }
d7aac489 2225 TString sval = map->GetValue("__result__")->GetName();
2226 Bool_t retval = (Bool_t)sval.Atoi();
2227 gGrid->Cd(cpath);
923e2ca5 2228 delete res;
d7aac489 2229 return retval;
2230}
923e2ca5 2231
c57f56b7 2232//______________________________________________________________________________
a2f5fc01 2233void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2234{
2235// Check input data type.
a2f5fc01 2236 isCollection = kFALSE;
2237 isXml = kFALSE;
2238 useTags = kFALSE;
c57f56b7 2239 if (!gGrid) {
2240 Error("CheckDataType", "No connection to grid");
2241 return;
2242 }
a2f5fc01 2243 isCollection = IsCollection(lfn);
c57f56b7 2244 TString msg = "\n##### file: ";
2245 msg += lfn;
a2f5fc01 2246 if (isCollection) {
c57f56b7 2247 msg += " type: raw_collection;";
2248 // special treatment for collections
a2f5fc01 2249 isXml = kFALSE;
c57f56b7 2250 // check for tag files in the collection
2251 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2252 if (!res) {
2253 msg += " using_tags: No (unknown)";
23329835 2254 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2255 return;
2256 }
2257 const char* typeStr = res->GetKey(0, "origLFN");
2258 if (!typeStr || !strlen(typeStr)) {
2259 msg += " using_tags: No (unknown)";
23329835 2260 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2261 return;
2262 }
2263 TString file = typeStr;
a2f5fc01 2264 useTags = file.Contains(".tag");
2265 if (useTags) msg += " using_tags: Yes";
c57f56b7 2266 else msg += " using_tags: No";
23329835 2267 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2268 return;
2269 }
2270 TString slfn(lfn);
2271 slfn.ToLower();
a2f5fc01 2272 isXml = slfn.Contains(".xml");
2273 if (isXml) {
c57f56b7 2274 // Open xml collection and check if there are tag files inside
2275 msg += " type: xml_collection;";
2276 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2277 if (!coll) {
2278 msg += " using_tags: No (unknown)";
23329835 2279 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2280 return;
2281 }
2282 TMap *map = coll->Next();
2283 if (!map) {
2284 msg += " using_tags: No (unknown)";
23329835 2285 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2286 return;
2287 }
2288 map = (TMap*)map->GetValue("");
2289 TString file;
2290 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2291 useTags = file.Contains(".tag");
c57f56b7 2292 delete coll;
a2f5fc01 2293 if (useTags) msg += " using_tags: Yes";
c57f56b7 2294 else msg += " using_tags: No";
23329835 2295 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2296 return;
2297 }
a2f5fc01 2298 useTags = slfn.Contains(".tag");
c57f56b7 2299 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2300 else msg += " type: unknown file;";
a2f5fc01 2301 if (useTags) msg += " using_tags: Yes";
c57f56b7 2302 else msg += " using_tags: No";
23329835 2303 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2304}
2305
4e5c5506 2306//______________________________________________________________________________
2307void AliAnalysisAlien::EnablePackage(const char *package)
2308{
2309// Enables a par file supposed to exist in the current directory.
2310 TString pkg(package);
2311 pkg.ReplaceAll(".par", "");
2312 pkg += ".par";
2313 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2314 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2315 return;
2316 }
2317 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2318 Info("EnablePackage", "AliEn plugin will use .par packages");
2319 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2320 if (!fPackages) {
2321 fPackages = new TObjArray();
2322 fPackages->SetOwner();
2323 }
2324 fPackages->Add(new TObjString(pkg));
2325}
2326
3bdcb562 2327//______________________________________________________________________________
2328TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2329{
2330// Make a tree from files having the location specified in fFileForTestMode.
2331// Inspired from JF's CreateESDChain.
2332 if (fFileForTestMode.IsNull()) {
2333 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2334 return NULL;
2335 }
2336 if (gSystem->AccessPathName(fFileForTestMode)) {
2337 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2338 return NULL;
2339 }
2340 // Open the file
2341 ifstream in;
2342 in.open(fFileForTestMode);
2343 Int_t count = 0;
2344 // Read the input list of files and add them to the chain
b505f1f8 2345 TString line;
d6c60197 2346 TString streeName(treeName);
2347 if (IsUseMCchain()) streeName = "TE";
2348 TChain *chain = new TChain(streeName);
06f67114 2349 TList *friends = new TList();
2350 TChain *cfriend = 0;
2351 if (!fFriendChainName.IsNull()) {
2352 TObjArray *list = fFriendChainName.Tokenize(" ");
2353 TIter next(list);
2354 TObjString *str;
2355 while((str=(TObjString*)next())) {
2356 cfriend = new TChain(streeName, str->GetName());
2357 friends->Add(cfriend);
2358 chain->AddFriend(cfriend);
2359 }
2360 delete list;
2361 }
2362 TString bpath;
2363 TIter nextfriend(friends);
b505f1f8 2364 while (in.good())
2365 {
3bdcb562 2366 in >> line;
ffbe06b9 2367 if (line.IsNull() || line.BeginsWith("#")) continue;
3bdcb562 2368 if (count++ == fNtestFiles) break;
2369 TString esdFile(line);
2370 TFile *file = TFile::Open(esdFile);
b505f1f8 2371 if (file && !file->IsZombie()) {
2372 chain->Add(esdFile);
3bdcb562 2373 file->Close();
b505f1f8 2374 if (!fFriendChainName.IsNull()) {
4b5d9e21 2375 if (esdFile.Index("#") > -1)
2376 esdFile.Remove(esdFile.Index("#"));
06f67114 2377 bpath = gSystem->DirName(esdFile);
2378 bpath += "/";
2379 TString fileFriend;
2380 nextfriend.Reset();
2381 while ((cfriend=(TChain*)nextfriend())) {
2382 fileFriend = bpath;
2383 fileFriend += cfriend->GetTitle();
2384 file = TFile::Open(fileFriend);
2385 if (file && !file->IsZombie()) {
2386 file->Close();
2387 cfriend->Add(fileFriend);
2388 } else {
2389 Fatal("GetChainForTestMode", "Cannot open friend file: %s", fileFriend.Data());
2390 return 0;
2391 }
2392 }
4b5d9e21 2393 }
3bdcb562 2394 } else {
2395 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2396 }
b505f1f8 2397 }
2398 in.close();
2399 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2400 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2401 delete chain;
06f67114 2402 friends->Delete();
2403 delete friends;
3bdcb562 2404 return NULL;
b505f1f8 2405 }
b505f1f8 2406 return chain;
3bdcb562 2407}
2408
d2a409b2 2409//______________________________________________________________________________
2410const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2411{
2412// Get job status for all jobs with jobid>jobidstart.
2413 static char mstatus[20];
2414 mstatus[0] = '\0';
2415 nrunning = 0;
2416 nwaiting = 0;
2417 nerror = 0;
2418 ndone = 0;
2419 TGridJobStatusList *list = gGrid->Ps("");
2420 if (!list) return mstatus;
2421 Int_t nentries = list->GetSize();
2422 TGridJobStatus *status;
2423 Int_t pid;
2424 for (Int_t ijob=0; ijob<nentries; ijob++) {
2425 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2426 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2427 if (pid<jobidstart) continue;
2428 if (pid == lastid) {
8ddad121 2429 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2430 }
2431 switch (status->GetStatus()) {
2432 case TGridJobStatus::kWAITING:
2433 nwaiting++; break;
2434 case TGridJobStatus::kRUNNING:
2435 nrunning++; break;
2436 case TGridJobStatus::kABORTED:
2437 case TGridJobStatus::kFAIL:
2438 case TGridJobStatus::kUNKNOWN:
2439 nerror++; break;
2440 case TGridJobStatus::kDONE:
2441 ndone++;
2442 }
2443 }
2444 list->Delete();
2445 delete list;
2446 return mstatus;
2447}
2448
c57f56b7 2449//______________________________________________________________________________
2450Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2451{
2452// Returns true if file is a collection. Functionality duplicated from
2453// TAlien::Type() because we don't want to directly depend on TAlien.
2454 if (!gGrid) {
2455 Error("IsCollection", "No connection to grid");
2456 return kFALSE;
2457 }
2458 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2459 if (!res) return kFALSE;
2460 const char* typeStr = res->GetKey(0, "type");
2461 if (!typeStr || !strlen(typeStr)) return kFALSE;
2462 if (!strcmp(typeStr, "collection")) return kTRUE;
2463 delete res;
2464 return kFALSE;
2465}
2466
fe2d7fc2 2467//______________________________________________________________________________
2468Bool_t AliAnalysisAlien::IsSingleOutput() const
2469{
2470// Check if single-ouput option is on.
2471 return (!fOutputSingle.IsNull());
2472}
715da79c 2473
2474//______________________________________________________________________________
2475Long64_t AliAnalysisAlien::RunMacroAndExtractLibs(const char* macro, const char *args, TString &libs)
2476{
2477// Tries to run the specified macro and return the libraries that it loads.
2478 TString expname;
2479 if (strlen(macro)) expname = gSystem->ExpandPathName(macro);
2480 if (expname.IsNull() || gSystem->AccessPathName(expname)) {
2481 ::Error("RunMacroAndExtractLibs","Cannot find macro %s in current directory", macro);
2482 return -1;
2483 }
2484 TString oldlibs = gSystem->GetLibraries();
2485 TMacro m(expname);
2486 Int_t error = 0;
2487 Long64_t retval = m.Exec(args, &error);
2488 if (error != TInterpreter::kNoError)
2489 {
2490 ::Error("RunMacroAndExtractLibs", "Macro interpretation %s failed", macro);
2491 return -1;
2492 }
2493 libs = gSystem->GetLibraries();
2494 libs.ReplaceAll(oldlibs, "");
2495 libs.Strip(TString::kLeading);
2496 TObjArray *libTokens = libs.Tokenize(" ");
2497 libs = "";
2498 for (Int_t i=0; i<libTokens->GetEntries(); i++) {
2499 if (!libs.IsNull()) libs += " ";
2500 libs += gSystem->BaseName(libTokens->At(i)->GetName());
2501 }
2502 delete libTokens;
2503 return retval;
2504}
2505
16a4353c 2506//______________________________________________________________________________
2507void AliAnalysisAlien::Print(Option_t *) const
2508{
2509// Print current plugin settings.
84fcd93f 2510 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2511 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2512 if (mgr && mgr->IsProofMode()) {
2513 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2514 if (TestBit(AliAnalysisGrid::kTest))
2515 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2516 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2517 if (!fProofDataSet.IsNull())
2518 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2519 if (fProofReset==1)
2520 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2521 if (fProofReset>1)
2522 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
874de7ea 2523 if (!fROOTVersion.IsNull())
2524 printf("= ROOT version requested________________________ %s\n", fROOTVersion.Data());
d3b18c4c 2525 else
2526 printf("= ROOT version requested________________________ default\n");
2527 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2528 if (!fAliRootMode.IsNull())
2529 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2530 if (fNproofWorkers)
2531 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2532 if (fNproofWorkersPerSlave)
2533 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2534 if (TestSpecialBit(kClearPackages))
2535 printf("= ClearPackages requested...\n");
2536 if (fIncludePath.Data())
2537 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2538 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2539 if (fPackages && fPackages->GetEntries()) {
2540 TIter next(fPackages);
2541 TObject *obj;
2542 TString list;
2543 while ((obj=next())) list += obj->GetName();
2544 printf("= Par files to be used: ________________________ %s\n", list.Data());
2545 }
2546 if (TestSpecialBit(kProofConnectGrid))
2547 printf("= Requested PROOF connection to grid\n");
2548 return;
2549 }
e1c22e21 2550 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2551 if (fOverwriteMode) {
2552 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2553 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2554 }
348be253 2555 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
780ec36d 2556 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO:Print");
84fcd93f 2557 printf("= Production mode:______________________________ %d\n", fProductionMode);
2558 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2559 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2560 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2561 if (fUser.Length())
84fcd93f 2562 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2563 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2564 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
780ec36d 2565 TString basedatadir = fGridDataDir;
2566 TString pattern = fDataPattern;
2567 pattern.Strip();
2568 Int_t ind = pattern.Index(" ");
2569 if (ind>=0) {
2570 basedatadir += "/%run%/";
2571 basedatadir += pattern(0, ind);
2572 pattern = pattern(ind+1, pattern.Length());
2573 }
2574 printf("= Data base directory path requested: __________ %s\n", basedatadir.Data());
2575 printf("= Data search pattern: _________________________ %s\n", pattern.Data());
84fcd93f 2576 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2577 if (fRunNumbers.Length())
84fcd93f 2578 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2579 if (fRunRange[0])
74013535 2580 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2581 if (!fRunRange[0] && !fRunNumbers.Length()) {
2582 TIter next(fInputFiles);
2583 TObject *obj;
2584 TString list;
2585 while ((obj=next())) list += obj->GetName();
84fcd93f 2586 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2587 }
2588 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2589 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2590 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2591 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2592 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2593 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2594 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2595 printf("=====================================================================\n");
2596 printf("= Job price: ___________________________________ %d\n", fPrice);
2597 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2598 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2599 if (fMaxInitFailed>0)
84fcd93f 2600 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2601 if (fMasterResubmitThreshold>0)
84fcd93f 2602 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2603 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2604 if (fNrunsPerMaster>0)
84fcd93f 2605 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2606 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2607 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2608 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2609 if (fArguments.Length())
84fcd93f 2610 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2611 if (fExecutableArgs.Length())
2612 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2613 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2614 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2615 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2616 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2617 if (fDatasetName)
84fcd93f 2618 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2619 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2620 if (fIncludePath.Data())
84fcd93f 2621 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2622 if (fCloseSE.Length())
84fcd93f 2623 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2624 if (fFriendChainName.Length())
84fcd93f 2625 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2626 if (fPackages && fPackages->GetEntries()) {
16a4353c 2627 TIter next(fPackages);
2628 TObject *obj;
2629 TString list;
2630 while ((obj=next())) list += obj->GetName();
84fcd93f 2631 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2632 }
2633}
2634
c57f56b7 2635//______________________________________________________________________________
2636void AliAnalysisAlien::SetDefaults()
2637{
2638// Set default values for everything. What cannot be filled will be left empty.
2639 if (fGridJDL) delete fGridJDL;
2640 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2641 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2642 fPrice = 1;
2643 fTTL = 30000;
2644 fSplitMaxInputFileNumber = 100;
2645 fMaxInitFailed = 0;
2646 fMasterResubmitThreshold = 0;
bb885a9e 2647 fNtestFiles = 10;
149d288c 2648 fNreplicas = 2;
a8739e8a 2649 fRunRange[0] = 0;
2650 fRunRange[1] = 0;
8e3bd06a 2651 fRunPrefix = "%d";
319593fb 2652 fNrunsPerMaster = 1;
16a4353c 2653 fMaxMergeFiles = 100;
c57f56b7 2654 fRunNumbers = "";
2655 fExecutable = "analysis.sh";
5aa94dc3 2656 fExecutableCommand = "root -b -q -x";
c57f56b7 2657 fArguments = "";
631c0b05 2658 fExecutableArgs = "";
c57f56b7 2659 fAnalysisMacro = "myAnalysis.C";
2660 fAnalysisSource = "";
2661 fAdditionalLibs = "";
2662 fSplitMode = "se";
2663 fAPIVersion = "";
2664 fROOTVersion = "";
2665 fAliROOTVersion = "";
2666 fUser = ""; // Your alien user name
2667 fGridWorkingDir = "";
2668 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2669 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2670 fFriendChainName = "";
c57f56b7 2671 fGridOutputDir = "output";
b83f84f0 2672 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2673 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2674 fInputFormat = "xml-single";
2675 fJDLName = "analysis.jdl";
c6cb3634 2676 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2677 fMergeExcludes = "";
0f389141 2678 fMergeViaJDL = 0;
348be253 2679 SetUseCopy(kTRUE);
2680 SetCheckCopy(kTRUE);
149d288c 2681 SetDefaultOutputs(kTRUE);
e1c22e21 2682 fOverwriteMode = 1;
c57f56b7 2683}
2684
a41b2882 2685//______________________________________________________________________________
2686void AliAnalysisAlien::SetFriendChainName(const char *name, const char *libnames)
2687{
2688 // Set file name for the chain of friends and optionally additional libs to be loaded.
2689 // Libs should be separated by blancs.
2690 fFriendChainName = name;
06f67114 2691 fFriendChainName.ReplaceAll(",", " ");
2692 fFriendChainName.Strip();
2693 fFriendChainName.ReplaceAll(" ", " ");
2694
a41b2882 2695 fFriendLibs = libnames;
1a3f255c 2696 if (fFriendLibs.Length()) {
2dffffd6 2697 if(!fFriendLibs.Contains(".so") &&
2698 !fFriendLibs.Contains(".dylib"))
1a3f255c 2699 Fatal("SetFriendChainName()", "You should provide explicit library names (with extension)");
2700 fFriendLibs.ReplaceAll(",", " ");
2701 fFriendLibs.Strip();
2702 fFriendLibs.ReplaceAll(" ", " ");
a41b2882 2703 }
2704}
2705
874de7ea 2706//______________________________________________________________________________
2707void AliAnalysisAlien::SetRootVersionForProof(const char *version)
2708{
2709// Obsolete method. Use SetROOTVersion instead
2710 Warning("SetRootVersionForProof", "Obsolete. Use SetROOTVersion instead");
95e16f01 2711 if (fROOTVersion.IsNull()) SetROOTVersion(version);
2712 else Error("SetRootVersionForProof", "ROOT version already set to %s", fROOTVersion.Data());
874de7ea 2713}
2714
0f389141 2715//______________________________________________________________________________
b3e07543 2716Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2717{
b3e07543 2718// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2719 // First check if the result is already in the output directory.
2720 if (FileExists(Form("%s/%s",aliendir,filename))) {
2721 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2722 return kFALSE;
2723 }
b3e07543 2724 // Now check the last stage done.
2725 Int_t stage = 0;
2726 while (1) {
2727 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2728 stage++;
2729 }
2730 // Next stage of merging
2731 stage++;
2732 TString pattern = "*root_archive.zip";
2733 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2734 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2735 if (res) delete res;
2736 // Write standard output to file
2737 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2738 // Count the number of files inside
2739 ifstream ifile;
2740 ifile.open(Form("Stage_%d.xml",stage));
2741 if (!ifile.good()) {
2742 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2743 return kFALSE;
b3e07543 2744 }
2745 TString line;
2746 Int_t nfiles = 0;
2747 while (!ifile.eof()) {
2748 ifile >> line;
2749 if (line.Contains("/event")) nfiles++;
7c2cd90a 2750 }
b3e07543 2751 ifile.close();
2752 if (!nfiles) {
2753 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2754 return kFALSE;
2755 } else {
2756 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2757 }
b3e07543 2758 // Copy the file in the output directory
2759 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2760// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2761 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2762 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2763 // Check if this is the last stage to be done.
2764 Bool_t laststage = (nfiles<nperchunk);
2765 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
0ad3ea1f 2766 Int_t jobId = 0;
b3e07543 2767 if (laststage) {
2768 printf("### Submiting final merging stage %d\n", stage);
2769 TString finalJDL = jdl;
2770 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2771 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
0ad3ea1f 2772 jobId = SubmitSingleJob(query);
b3e07543 2773 } else {
2774 printf("### Submiting merging stage %d\n", stage);
03335324 2775 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
0ad3ea1f 2776 jobId = SubmitSingleJob(query);
7c2cd90a 2777 }
0ad3ea1f 2778 if (!jobId) return kFALSE;
2779
2780 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
2781 fGridJobIDs.Append(Form("%d", jobId));
2782 if (!fGridStages.IsNull()) fGridStages.Append(" ");
2783 fGridStages.Append(Form("%s_merge_stage%d",
2784 laststage ? "final" : "partial", stage));
2785
b3e07543 2786 return kTRUE;
2787}
7c2cd90a 2788
105631b5 2789//______________________________________________________________________________
2790AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2791{
2792// Loat the analysis manager from a file.
2793 TFile *file = TFile::Open(fname);
2794 if (!file) {
2795 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2796 return 0;
2797 }
2798 TIter nextkey(file->GetListOfKeys());
2799 AliAnalysisManager *mgr = 0;
2800 TKey *key;
2801 while ((key=(TKey*)nextkey())) {
2802 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2803 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2804 }
2805 if (!mgr)
2806 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2807 return mgr;
2808}
2809
7c2cd90a 2810//______________________________________________________________________________
2811Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2812{
2813// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2814 if (!gGrid) return 0;
2815 printf("=> %s ------> ",query);
2816 TGridResult *res = gGrid->Command(query);
2817 if (!res) return 0;
2818 TString jobId = res->GetKey(0,"jobId");
2819 delete res;
2820 if (jobId.IsNull()) {
2821 printf("submission failed. Reason:\n");
2822 gGrid->Stdout();
2823 gGrid->Stderr();
2824 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2825 return 0;
2826 }
0ad3ea1f 2827 Int_t ijobId = jobId.Atoi();
2828 printf(" Job id: '%s' (%d)\n", jobId.Data(), ijobId);
2829 return ijobId;
7c2cd90a 2830}
2831
ec5acdce 2832//______________________________________________________________________________
2833Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2834{
2835// Merges a collection of output files using concatenation.
d66aa2a9 2836 TString scoll(collection);
2837 if (!scoll.Contains(".xml")) return kFALSE;
ec5acdce 2838 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2839 if (!coll) {
2840 ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2841 return kFALSE;
2842 }
2843 // Iterate grid collection
2844 TString outtmp;
2845 Bool_t merged = kFALSE;
2846 Int_t ifile = 0;
2847 while (coll->Next()) {
2848 TString fname = gSystem->DirName(coll->GetTURL());
2849 fname += "/";
2850 fname += output;
2851 outtmp = Form("%d_%s", ifile, output);
2852 if (!TFile::Cp(fname, outtmp)) {
2853 ::Error("MergeInfo", "Could not copy %s", fname.Data());
2854 continue;
2855 }
2856 ifile++;
2857 if (ifile<2) {
2858 gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2859 continue;
2860 }
2861 gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2862 gSystem->Exec("cp tempmerged lastmerged");
2863 }
2864 if (ifile) {
2865 gSystem->Exec(Form("cp lastmerged %s", output));
2866 gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2867 merged = kTRUE;
2868 }
2869 return merged;
2870}
2871
7c2cd90a 2872//______________________________________________________________________________
b3e07543 2873Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2874{
b3e07543 2875// Merge given output files from basedir. Basedir can be an alien output directory
2876// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2877// files in a group (ignored for xml input). Merging can be done in stages:
2878// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2879// stage=1 : works with an xml of all root_archive.zip in the output directory
2880// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2881 TString outputFile = output;
0f389141 2882 TString command;
a2f5fc01 2883 TString outputChunk;
2884 TString previousChunk = "";
b3e07543 2885 TObjArray *listoffiles = new TObjArray();
2886// listoffiles->SetOwner();
a2f5fc01 2887 Int_t countChunk = 0;
2888 Int_t countZero = nmaxmerge;
0f389141 2889 Bool_t merged = kTRUE;
d66aa2a9 2890 Bool_t isGrid = kTRUE;
a2f5fc01 2891 Int_t index = outputFile.Index("@");
2892 if (index > 0) outputFile.Remove(index);
7c2cd90a 2893 TString inputFile = outputFile;
b3e07543 2894 TString sbasedir = basedir;
2895 if (sbasedir.Contains(".xml")) {
2896 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2897 nmaxmerge = 9999999;
2898 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2899 if (!coll) {
2900 ::Error("MergeOutput", "Input XML collection empty.");
2901 return kFALSE;
2902 }
2903 // Iterate grid collection
2904 while (coll->Next()) {
2905 TString fname = gSystem->DirName(coll->GetTURL());
2906 fname += "/";
2907 fname += inputFile;
2908 listoffiles->Add(new TNamed(fname.Data(),""));
2909 }
ffbe06b9 2910 } else if (sbasedir.Contains(".txt")) {
d66aa2a9 2911 // The file having the .txt extension is expected to contain a list of
2912 // folders where the output files will be looked. For alien folders,
2913 // the full folder LFN is expected (starting with alien://)
ffbe06b9 2914 // Assume lfn's on each line
2915 TString line;
2916 ifstream in;
2917 in.open(sbasedir);
d66aa2a9 2918 if (in.fail()) {
2919 ::Error("MergeOutput", "File %s cannot be opened. Merging stopped." ,sbasedir.Data());
2920 return kTRUE;
2921 }
ffbe06b9 2922 Int_t nfiles = 0;
2923 while (in.good()) {
2924 in >> line;
2925 if (line.IsNull() || line.BeginsWith("#")) continue;
d66aa2a9 2926 line.Strip();
2927 if (!line.Contains("alien:")) isGrid = kFALSE;
2928 line += "/";
2929 line += outputFile;
ffbe06b9 2930 nfiles++;
2931 listoffiles->Add(new TNamed(line.Data(),""));
2932 }
d66aa2a9 2933 in.close();
ffbe06b9 2934 if (!nfiles) {
2935 ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data());
2936 delete listoffiles;
2937 return kFALSE;
2938 }
b3e07543 2939 } else {
2940 command = Form("find %s/ *%s", basedir, inputFile.Data());
2941 printf("command: %s\n", command.Data());
2942 TGridResult *res = gGrid->Command(command);
2943 if (!res) {
2944 ::Error("MergeOutput","No result for the find command\n");
2945 delete listoffiles;
2946 return kFALSE;
2947 }
2948 TIter nextmap(res);
2949 TMap *map = 0;
2950 while ((map=(TMap*)nextmap())) {
2951 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2952 if (!objs || !objs->GetString().Length()) {
2953 // Nothing found - skip this output
2954 delete res;
2955 delete listoffiles;
2956 return kFALSE;
2957 }
2958 listoffiles->Add(new TNamed(objs->GetName(),""));
2959 }
2960 delete res;
2961 }
2962 if (!listoffiles->GetEntries()) {
7c2cd90a 2963 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2964 delete listoffiles;
0f389141 2965 return kFALSE;
2966 }
2967
2968 TFileMerger *fm = 0;
b3e07543 2969 TIter next0(listoffiles);
2970 TObjArray *listoffilestmp = new TObjArray();
2971 listoffilestmp->SetOwner();
2972 TObject *nextfile;
2973 TString snextfile;
2974 // Keep only the files at upper level
2975 Int_t countChar = 0;
2976 while ((nextfile=next0())) {
2977 snextfile = nextfile->GetName();
2978 Int_t crtCount = snextfile.CountChar('/');
2979 if (nextfile == listoffiles->First()) countChar = crtCount;
2980 if (crtCount < countChar) countChar = crtCount;
2981 }
2982 next0.Reset();
2983 while ((nextfile=next0())) {
2984 snextfile = nextfile->GetName();
2985 Int_t crtCount = snextfile.CountChar('/');
2986 if (crtCount > countChar) {
2987 delete nextfile;
2988 continue;
2989 }
2990 listoffilestmp->Add(nextfile);
2991 }
2992 delete listoffiles;
2993 listoffiles = listoffilestmp; // Now contains 'good' files
2994 listoffiles->Print();
2995 TIter next(listoffiles);
7c2cd90a 2996 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2997 outputChunk = outputFile;
2998 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2999 // Check for existent temporary merge files
e1c22e21 3000 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 3001 // Preserve old merging functionality for stage 0.
3002 if (stage==0) {
3003 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
3004 while (1) {
3005 // Skip as many input files as in a chunk
2ea55496 3006 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 3007 nextfile = next();
3008 if (!nextfile) {
2ea55496 3009 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 3010 delete listoffiles;
2ea55496 3011 return kFALSE;
3012 }
b3e07543 3013 snextfile = nextfile->GetName();
7c2cd90a 3014 }
3015 outputChunk = outputFile;
3016 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
3017 countChunk++;
3018 if (gSystem->AccessPathName(outputChunk)) continue;
3019 // Merged file with chunks up to <countChunk> found
3020 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
3021 previousChunk = outputChunk;
3022 break;
3023 }
3024 }
3025 countZero = nmaxmerge;
3026
b3e07543 3027 while ((nextfile=next())) {
3028 snextfile = nextfile->GetName();
2ea55496 3029 // Loop 'find' results and get next LFN
7c2cd90a 3030 if (countZero == nmaxmerge) {
3031 // First file in chunk - create file merger and add previous chunk if any.
d66aa2a9 3032 fm = new TFileMerger(isGrid);
7c2cd90a 3033 fm->SetFastMethod(kTRUE);
3034 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
3035 outputChunk = outputFile;
3036 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
3037 }
3038 // If last file found, put merged results in the output file
b3e07543 3039 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 3040 // Add file to be merged and decrement chunk counter.
b3e07543 3041 fm->AddFile(snextfile);
7c2cd90a 3042 countZero--;
b3e07543 3043 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 3044 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3045 // Nothing found - skip this output
3046 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 3047 merged = kFALSE;
3048 break;
7c2cd90a 3049 }
3050 fm->OutputFile(outputChunk);
3051 // Merge the outputs, then go to next chunk
3052 if (!fm->Merge()) {
3053 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 3054 merged = kFALSE;
3055 break;
7c2cd90a 3056 } else {
3057 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
3058 gSystem->Unlink(previousChunk);
3059 }
b3e07543 3060 if (nextfile == listoffiles->Last()) break;
7c2cd90a 3061 countChunk++;
3062 countZero = nmaxmerge;
3063 previousChunk = outputChunk;
0f389141 3064 }
0f389141 3065 }
b3e07543 3066 delete listoffiles;
3067 delete fm;
7c2cd90a 3068 return merged;
3069 }
3070 // Merging stage different than 0.
3071 // Move to the begining of the requested chunk.
d66aa2a9 3072 fm = new TFileMerger(isGrid);
7c2cd90a 3073 fm->SetFastMethod(kTRUE);
b3e07543 3074 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
3075 delete listoffiles;
7c2cd90a 3076 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3077 // Nothing found - skip this output
3078 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
3079 delete fm;
3080 return kFALSE;
3081 }
b3e07543 3082 fm->OutputFile(outputFile);
7c2cd90a 3083 // Merge the outputs
3084 if (!fm->Merge()) {
3085 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
3086 delete fm;
3087 return kFALSE;
3088 } else {
b3e07543 3089 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 3090 }
7c2cd90a 3091 delete fm;
3092 return kTRUE;
0f389141 3093}
3094
c57f56b7 3095//______________________________________________________________________________
3096Bool_t AliAnalysisAlien::MergeOutputs()
3097{
3098// Merge analysis outputs existing in the AliEn space.
3099 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
3100 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3101 if (!Connect()) {
3102 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
3103 return kFALSE;
0f389141 3104 }
c9e8f7fd 3105 if (fMergeViaJDL) {
3106 if (!TestBit(AliAnalysisGrid::kMerge)) {
3107 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
3108 return kFALSE;
3109 }
3110 if (fProductionMode) {
3111 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
3112 return kFALSE;
3113 }
0f389141 3114 Info("MergeOutputs", "Submitting merging JDL");
a03be957 3115 if (!SubmitMerging()) return kFALSE;
0f389141 3116 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 3117 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 3118 return kFALSE;
c57f56b7 3119 }
3120 // Get the output path
dc030a24 3121 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 3122 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 3123 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
3124 return kFALSE;
3125 }
3126 if (!fOutputFiles.Length()) {
3127 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3128 return kFALSE;
0f389141 3129 }
3130 // Check if fast read option was requested
e1c22e21 3131 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
3132 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 3133 if (fFastReadOption) {
d3339be3 3134 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
3135 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 3136 gEnv->SetValue("XNet.ConnectTimeout",50);
3137 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 3138 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 3139 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 3140 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 3141 }
e8b839ab 3142 // Make sure we change the temporary directory
3143 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 3144 // Set temporary compilation directory to current one
3145 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 3146 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3147 TIter next(list);
3148 TObjString *str;
a2f5fc01 3149 TString outputFile;
c57f56b7 3150 Bool_t merged = kTRUE;
3151 while((str=(TObjString*)next())) {
a2f5fc01 3152 outputFile = str->GetString();
3153 Int_t index = outputFile.Index("@");
3154 if (index > 0) outputFile.Remove(index);
3155 TString outputChunk = outputFile;
3156 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 3157 // Skip already merged outputs
a2f5fc01 3158 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 3159 if (fOverwriteMode) {
a2f5fc01 3160 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
3161 gSystem->Unlink(outputFile);
3162 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3163 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3164 outputChunk.Data());
3165 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3166 }
3167 } else {
a2f5fc01 3168 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 3169 continue;
3170 }
3171 } else {
a2f5fc01 3172 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3173 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3174 outputChunk.Data());
3175 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3176 }
3177 }
37d21c01 3178 if (fMergeExcludes.Contains(outputFile.Data()) ||
3179 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 3180 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 3181 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 3182 if (!merged) {
3183 Error("MergeOutputs", "Terminate() will NOT be executed");
09d5920f 3184 delete list;
3185 return kFALSE;
ff07ec61 3186 }
3187 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
3188 if (fileOpened) fileOpened->Close();
c57f56b7 3189 }
09d5920f 3190 delete list;
0f389141 3191 return kTRUE;
c57f56b7 3192}
3193
bb885a9e 3194//______________________________________________________________________________
3195void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
3196{
3197// Use the output files connected to output containers from the analysis manager
3198// rather than the files defined by SetOutputFiles
3199 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 3200 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 3201 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
3202}
3203
149d288c 3204//______________________________________________________________________________
3205void AliAnalysisAlien::SetOutputFiles(const char *list)
3206{
3207// Manually set the output files list.
3208// Removes duplicates. Not allowed if default outputs are not disabled.
3209 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3210 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
3211 return;
3212 }
3213 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
3214 fOutputFiles = "";
3215 TString slist = list;
3216 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
3217 TObjArray *arr = slist.Tokenize(" ");
3218 TObjString *os;
3219 TIter next(arr);
3220 TString sout;
3221 while ((os=(TObjString*)next())) {
3222 sout = os->GetString();
3223 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
3224 if (fOutputFiles.Contains(sout)) continue;
3225 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
3226 fOutputFiles += sout;
3227 }
3228 delete arr;
f790bc1b 3229}
149d288c 3230
3231//______________________________________________________________________________
3232void AliAnalysisAlien::SetOutputArchive(const char *list)
3233{
3234// Manually set the output archive list. Free text - you are on your own...
3235// Not allowed if default outputs are not disabled.
3236 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3237 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
3238 return;
3239 }
3240 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
3241 fOutputArchive = list;
3242}
3243
28ce6c9c 3244//______________________________________________________________________________
3245void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
3246{
3247// Set some PROOF special parameter.
3248 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3249 if (pair) {
3250 TObject *old = pair->Key();
3251 TObject *val = pair->Value();
3252 fProofParam.Remove(old);
3253 delete old;
3254 delete val;
3255 }
3256 fProofParam.Add(new TObjString(pname), new TObjString(value));
3257}
3258
3259//______________________________________________________________________________
3260const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
3261{
3262// Returns a special PROOF parameter.
3263 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3264 if (!pair) return 0;
3265 return pair->Value()->GetName();
3266}
3267
5513444a 3268//______________________________________________________________________________
3269Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 3270{
3271// Start remote grid analysis.
3bdcb562 3272 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3273 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
3274 if (!mgr || !mgr->IsInitialized()) {
3275 Error("StartAnalysis", "You need an initialized analysis manager for this");
3276 return kFALSE;
3277 }
3278 // Are we in PROOF mode ?
3279 if (mgr->IsProofMode()) {
baa34407 3280 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3281 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3282 if (fProofCluster.IsNull()) {
3283 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3284 return kFALSE;
3285 }
3286 if (fProofDataSet.IsNull() && !testMode) {
3287 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3288 return kFALSE;
3289 }
3290 // Set the needed environment
3291 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3292 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3293 if (fProofReset && !testMode) {
3294 if (fProofReset==1) {
3295 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3296 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3297 } else {
3298 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3299 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3300 }
3301 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3302 return kFALSE;
3303 }
baa34407 3304
3305 if (!testMode) {
3306 // Check if there is an old active session
3307 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3308 if (nsessions) {
3309 Error("StartAnalysis","You have to reset your old session first\n");
3310 return kFALSE;
3311 }
53911ca6 3312 }
3bdcb562 3313 // Do we need to change the ROOT version ? The success of this cannot be checked.
874de7ea 3314 if (!fROOTVersion.IsNull() && !testMode) {
cbccef47 3315 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"VO_ALICE@ROOT::%s\");",
874de7ea 3316 fProofCluster.Data(), fROOTVersion.Data()));
3bdcb562 3317 }
3318 // Connect to PROOF and check the status
3319 Long_t proof = 0;
27734f0e 3320 TString sworkers;
3321 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3322 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3323 if (!testMode) {
27734f0e 3324 if (!sworkers.IsNull())
3325 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3326 else
3327 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3328 } else {
3329 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3330 if (!proof) {
3331 Error("StartAnalysis", "Could not start PROOF in test mode");
3332 return kFALSE;
3333 }
3334 }
3335 if (!proof) {
3336 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3337 return kFALSE;
3338 }
27734f0e 3339 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3340 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3341 // Set proof special parameters if any
3342 TIter nextpp(&fProofParam);
3343 TObject *proofparam;
3344 while ((proofparam=nextpp())) {
3345 TString svalue = GetProofParameter(proofparam->GetName());
3346 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3347 }
3bdcb562 3348 // Is dataset existing ?
3349 if (!testMode) {
3350 TString dataset = fProofDataSet;
3351 Int_t index = dataset.Index("#");
3352 if (index>=0) dataset.Remove(index);
830acc4c 3353// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3354// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3355// return kFALSE;
3356// }
3357// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3358 }
3359 // Is ClearPackages() needed ?
3360 if (TestSpecialBit(kClearPackages)) {
3361 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3362 gROOT->ProcessLine("gProof->ClearPackages();");
3363 }
3364 // Is a given aliroot mode requested ?
3365 TList optionsList;
1ac846e9 3366 TString parLibs;
830acc4c 3367 if (!fAliRootMode.IsNull()) {
3bdcb562 3368 TString alirootMode = fAliRootMode;
3369 if (alirootMode == "default") alirootMode = "";
3370 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3371 optionsList.SetOwner();
3372 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3373 // Check the additional libs to be loaded
3374 TString extraLibs;
1ac846e9 3375 Bool_t parMode = kFALSE;
dc1ffbcb 3376 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
2dffffd6 3377 // Parse the extra libs for .so or .dylib
3bdcb562 3378 if (fAdditionalLibs.Length()) {
a41b2882 3379 TString additionalLibs = fAdditionalLibs;
3380 additionalLibs.Strip();
3381 if (additionalLibs.Length() && fFriendLibs.Length())
3382 additionalLibs += " ";
3383 additionalLibs += fFriendLibs;
3384 TObjArray *list = additionalLibs.Tokenize(" ");
3bdcb562 3385 TIter next(list);
3386 TObjString *str;
1ac846e9 3387 while((str=(TObjString*)next())) {
2dffffd6 3388 if (str->GetString().Contains(".so") ||
3389 str->GetString().Contains(".dylib") ) {
1ac846e9 3390 if (parMode) {
3391 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3392 break;
3393 }
3394 TString stmp = str->GetName();
3395 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3396 stmp.ReplaceAll(".so","");
2dffffd6 3397 stmp.ReplaceAll(".dylib","");
1ac846e9 3398 if (!extraLibs.IsNull()) extraLibs += ":";
3399 extraLibs += stmp;
3400 continue;
3401 }
3402 if (str->GetString().Contains(".par")) {
3403 // The first par file found in the list will not allow any further .so
3404 parMode = kTRUE;
3405 if (!parLibs.IsNull()) parLibs += ":";
3406 parLibs += str->GetName();
3407 continue;
3408 }
3bdcb562 3409 }
3410 if (list) delete list;
3411 }
a41b2882 3412 if (!extraLibs.IsNull()) {
3413 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3414 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3415 }
830acc4c 3416 // Check extra includes
3417 if (!fIncludePath.IsNull()) {
3418 TString includePath = fIncludePath;
3419 includePath.ReplaceAll(" ",":");
dc030a24 3420 includePath.ReplaceAll("$ALICE_ROOT/","");
3421 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3422 includePath.ReplaceAll("-I","");
dc030a24 3423 includePath.Remove(TString::kTrailing, ':');
830acc4c 3424 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3425 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3426 }
27734f0e 3427 // Check if connection to grid is requested
3428 if (TestSpecialBit(kProofConnectGrid))
3429 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3430 // Enable AliRoot par
3431 if (testMode) {
3432 // Enable proof lite package
3433 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3434 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3435 TNamed *obj = (TNamed*)optionsList.At(i);
3436 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3437 }
3438 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3439 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3440 Info("StartAnalysis", "AliRootProofLite enabled");
3441 } else {
3442 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3443 return kFALSE;
3444 }
3445 } else {
dc030a24 3446 if ( ! fAliROOTVersion.IsNull() ) {
3447 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3448 fAliROOTVersion.Data(), &optionsList))) {
3449 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3450 return kFALSE;
3451 }
3452 }
27734f0e 3453 }
1ac846e9 3454 // Enable first par files from fAdditionalLibs
3455 if (!parLibs.IsNull()) {
3456 TObjArray *list = parLibs.Tokenize(":");
3457 TIter next(list);
3458 TObjString *package;
3459 while((package=(TObjString*)next())) {
40501bb7 3460 TString spkg = package->GetName();
3461 spkg.ReplaceAll(".par", "");
3462 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3463 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3464 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3465 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3466 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3467 return kFALSE;
3468 }
3469 } else {
3470 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3471 return kFALSE;
3472 }
3473 }
3474 if (list) delete list;
3475 }
3bdcb562 3476 } else {
2dffffd6 3477 if ((fAdditionalLibs.Contains(".so") || fAdditionalLibs.Contains(".dylib")) &&
3478 !testMode) {
3bdcb562 3479 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3480 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3481 return kFALSE;
3482 }
3483 }
3484 // Enable par files if requested
3485 if (fPackages && fPackages->GetEntries()) {
3486 TIter next(fPackages);
3487 TObject *package;
3488 while ((package=next())) {
1ac846e9 3489 // Skip packages already enabled
3490 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3491 TString spkg = package->GetName();
3492 spkg.ReplaceAll(".par", "");
3493 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3494 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3495 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3496 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3497 return kFALSE;
3498 }
3499 } else {
3500 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3501 return kFALSE;
3502 }
3503 }
3504 }
3505 // Do we need to load analysis source files ?
3506 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3507 if (fAnalysisSource.Length()) {
3508 TObjArray *list = fAnalysisSource.Tokenize(" ");
3509 TIter next(list);
3510 TObjString *str;
3511 while((str=(TObjString*)next())) {
3512 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3513 }
3bdcb562 3514 if (list) delete list;
3515 }
830acc4c 3516 if (testMode) {
3517 // Register dataset to proof lite.
3518 if (fFileForTestMode.IsNull()) {
3519 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3520 return kFALSE;
3521 }
3522 if (gSystem->AccessPathName(fFileForTestMode)) {
3523 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3524 return kFALSE;
3525 }
3526 TFileCollection *coll = new TFileCollection();
3527 coll->AddFromFile(fFileForTestMode);
8ddad121 3528 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3529 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3530 }
3bdcb562 3531 return kTRUE;
3532 }
c57f56b7 3533
43da816a 3534 // Check if output files have to be taken from the analysis manager
3535 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3536 // Add output files and AOD files
3537 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3538 // Add extra files registered to the analysis manager
f790bc1b 3539 TString extra = GetListOfFiles("ext");
3540 if (!extra.IsNull()) {
149d288c 3541 extra.ReplaceAll(".root", "*.root");
f790bc1b 3542 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3543 fOutputFiles += extra;
84fcd93f 3544 }
149d288c 3545 // Compose the output archive.
b83f84f0 3546 fOutputArchive = "log_archive.zip:std*@disk=1 ";
ec5acdce 3547 if (mgr->IsCollectThroughput())
95e16f01 3548 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",fOutputFiles.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 3549 else
3550 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3551 }
f7b1cbc2 3552// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3553 if (TestBit(AliAnalysisGrid::kOffline)) {
3554 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3555 \n there nor any job run. You can revise the JDL and analysis \
3556 \n macro then run the same in \"submit\" mode.");
3557 } else if (TestBit(AliAnalysisGrid::kTest)) {
3558 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3559 \n dataset.");
3560 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3561 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3562 \n space and job submitted.");
3563 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3564 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3565 if (fMergeViaJDL) CheckInputData();
5513444a 3566 return kTRUE;
c57f56b7 3567 } else {
3568 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3569 }
3570
348be253 3571 Print();
c57f56b7 3572 if (!Connect()) {
3573 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3574 return kFALSE;
16a4353c 3575 }
d3b18c4c 3576 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3577 if (!CheckInputData()) {
3578 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3579 return kFALSE;
c57f56b7 3580 }
d3339be3 3581 if (!CreateDataset(fDataPattern)) {
3582 TString serror;
3583 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3584 if (fRunNumbers.Length()) serror = "run numbers";
3585 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3586 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3587 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3588 return kFALSE;
3589 }
21ca8e59 3590 WriteAnalysisFile();
c57f56b7 3591 WriteAnalysisMacro();
3592 WriteExecutable();
3593 WriteValidationScript();
0f389141 3594 if (fMergeViaJDL) {
3595 WriteMergingMacro();
3596 WriteMergeExecutable();
3597 WriteValidationScript(kTRUE);
3598 }
5513444a 3599 if (!CreateJDL()) return kFALSE;
3600 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3601 if (testMode) {
c57f56b7 3602 // Locally testing the analysis
3603 Info("StartAnalysis", "\n_______________________________________________________________________ \
3604 \n Running analysis script in a daughter shell as on a worker node \
3605 \n_______________________________________________________________________");
149d288c 3606 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3607 TIter next(list);
3608 TObjString *str;
a2f5fc01 3609 TString outputFile;
c57f56b7 3610 while((str=(TObjString*)next())) {
a2f5fc01 3611 outputFile = str->GetString();
3612 Int_t index = outputFile.Index("@");
3613 if (index > 0) outputFile.Remove(index);
3614 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3615 }
3616 delete list;
3617 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3618 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3619// gSystem->Exec("cat stdout");
5513444a 3620 return kFALSE;
c57f56b7 3621 }
5513444a 3622 // Check if submitting is managed by LPM manager
a3e84053 3623 if (fProductionMode) {
7ac61826 3624 //TString prodfile = fJDLName;
3625 //prodfile.ReplaceAll(".jdl", ".prod");
3626 //WriteProductionFile(prodfile);
5513444a 3627 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3628 return kFALSE;
3629 }
a8739e8a 3630 // Submit AliEn job(s)
d2a409b2 3631 gGrid->Cd(fGridOutputDir);
a8739e8a 3632 TGridResult *res;
c57f56b7 3633 TString jobID = "";
0ad3ea1f 3634 fGridJobIDs = "";
3635 fGridStages = "";
d2a409b2 3636 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3637 // Submit a given xml or a set of runs
a8739e8a 3638 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3639 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3640 if (res) {
3641 const char *cjobId = res->GetKey(0,"jobId");
3642 if (!cjobId) {
a03be957 3643 gGrid->Stdout();
3644 gGrid->Stderr();
a8739e8a 3645 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3646 return kFALSE;
a8739e8a 3647 } else {
3648 Info("StartAnalysis", "\n_______________________________________________________________________ \
3649 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3650 \n_______________________________________________________________________",
3651 fJDLName.Data(), cjobId);
3652 jobID = cjobId;
0ad3ea1f 3653 if (jobID.Atoi()) {
3654 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3655 fGridJobIDs.Append(jobID);
3656 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3657 fGridStages.Append("full");
3658 }
a8739e8a 3659 }
3660 delete res;
a03be957 3661 } else {
3662 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3663 return kFALSE;
a8739e8a 3664 }
3665 } else {
d2a409b2 3666 // Submit for a range of enumeration of runs.
a03be957 3667 if (!Submit()) return kFALSE;
0ad3ea1f 3668 jobID = fGridJobIDs;
c57f56b7 3669 }
a8739e8a 3670
d88e4159 3671 if (fDropToShell) {
3672 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3673 \n You may exit at any time and terminate the job later using the option <terminate> \
3674 \n ##################################################################################", jobID.Data());
3675 gSystem->Exec("aliensh");
3676 } else {
3677 Info("StartAnalysis", "\n#### SUBMITTED JOB %s TO ALIEN QUEUE #### \
3678 \n Remember to terminate the job later using the option <terminate> \
3679 \n ##################################################################################", jobID.Data());
3680 }
5513444a 3681 return kTRUE;
c57f56b7 3682}
3683
f790bc1b 3684//______________________________________________________________________________
3685const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3686{
3687// Get a comma-separated list of output files of the requested type.
3688// Type can be (case unsensitive):
3689// aod - list of aod files (std, extensions and filters)
3690// out - list of output files connected to containers (but not aod's or extras)
3691// ext - list of extra files registered to the manager
3692// ter - list of files produced in terminate
3693 static TString files;
3694 files = "";
3695 TString stype = type;
3696 stype.ToLower();
3697 TString aodfiles, extra;
3698 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3699 if (!mgr) {
3700 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3701 return files.Data();
3702 }
3703 if (mgr->GetOutputEventHandler()) {
6be131bd 3704 aodfiles = "";
3705 if (mgr->GetOutputEventHandler()->GetFillAOD())
3706 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
f6eaa73b 3707 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs(kTRUE);
6be131bd 3708 if (!extraaod.IsNull() && mgr->GetOutputEventHandler()->GetFillExtension()) {
0c0825e4 3709 if (!aodfiles.IsNull()) aodfiles += ",";
f790bc1b 3710 aodfiles += extraaod;
3711 }
3712 }
3713 if (stype.Contains("aod")) {
3714 files = aodfiles;
3715 if (stype == "aod") return files.Data();
3716 }
3717 // Add output files that are not in the list of AOD files
3718 TString outputfiles = "";
3719 TIter next(mgr->GetOutputs());
3720 AliAnalysisDataContainer *output;
3721 const char *filename = 0;
3722 while ((output=(AliAnalysisDataContainer*)next())) {
3723 filename = output->GetFileName();
3724 if (!(strcmp(filename, "default"))) continue;
3725 if (outputfiles.Contains(filename)) continue;
3726 if (aodfiles.Contains(filename)) continue;
0c0825e4 3727 if (!outputfiles.IsNull() && strlen(filename)) outputfiles += ",";
f790bc1b 3728 outputfiles += filename;
3729 }
3730 if (stype.Contains("out")) {
0c0825e4 3731 if (!files.IsNull() && !outputfiles.IsNull()) files += ",";
f790bc1b 3732 files += outputfiles;
3733 if (stype == "out") return files.Data();
3734 }
3735 // Add extra files registered to the analysis manager
3736 TString sextra;
3737 extra = mgr->GetExtraFiles();
3738 if (!extra.IsNull()) {
3739 extra.Strip();
3740 extra.ReplaceAll(" ", ",");
3741 TObjArray *fextra = extra.Tokenize(",");
3742 TIter nextx(fextra);
3743 TObject *obj;
3744 while ((obj=nextx())) {
3745 if (aodfiles.Contains(obj->GetName())) continue;
3746 if (outputfiles.Contains(obj->GetName())) continue;
3747 if (sextra.Contains(obj->GetName())) continue;
3748 if (!sextra.IsNull()) sextra += ",";
3749 sextra += obj->GetName();
3750 }
3751 delete fextra;
3752 if (stype.Contains("ext")) {
0c0825e4 3753 if (!files.IsNull() && !sextra.IsNull()) files += ",";
f790bc1b 3754 files += sextra;
3755 }
3756 }
3757 if (stype == "ext") return files.Data();
3758 TString termfiles;
3759 if (!fTerminateFiles.IsNull()) {
3760 fTerminateFiles.Strip();
3761 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3762 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3763 TIter nextx(fextra);
3764 TObject *obj;
3765 while ((obj=nextx())) {
3766 if (aodfiles.Contains(obj->GetName())) continue;
3767 if (outputfiles.Contains(obj->GetName())) continue;
3768 if (termfiles.Contains(obj->GetName())) continue;
3769 if (sextra.Contains(obj->GetName())) continue;
3770 if (!termfiles.IsNull()) termfiles += ",";
3771 termfiles += obj->GetName();
3772 }
3773 delete fextra;
3774 }
3775 if (stype.Contains("ter")) {
3776 if (!files.IsNull() && !termfiles.IsNull()) {
3777 files += ",";
3778 files += termfiles;
3779 }
3780 }
3781 return files.Data();
3782}
3783
d2a409b2 3784//______________________________________________________________________________
a03be957 3785Bool_t AliAnalysisAlien::Submit()
d2a409b2 3786{
3787// Submit all master jobs.
3788 Int_t nmasterjobs = fInputFiles->GetEntries();
3789 Long_t tshoot = gSystem->Now();
a03be957 3790 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3791 while (fNsubmitted < nmasterjobs) {
3792 Long_t now = gSystem->Now();
3793 if ((now-tshoot)>30000) {
3794 tshoot = now;
a03be957 3795 if (!SubmitNext()) return kFALSE;
d2a409b2 3796 }
3797 }
a03be957 3798 return kTRUE;
d2a409b2 3799}
3800
0f389141 3801//______________________________________________________________________________
a03be957 3802Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3803{
3804// Submit all merging jobs.
dc030a24 3805 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3806 gGrid->Cd(fGridOutputDir);
3807 TString mergeJDLName = fExecutable;
3808 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3809 if (!fInputFiles) {
3810 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3811 return kFALSE;
3812 }
0f389141 3813 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3814 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3815 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3816 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3817 if (fOutputToRunNo) {
3818 // The output directory is the run number
3819 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3820 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3821 } else {
17b4e9a4 3822 if (!fRunNumbers.Length() && !fRunRange[0]) {
3823 // The output directory is the grid outdir
3824 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3825 runOutDir = fGridOutputDir;
3826 } else {
3827 // The output directory is the master number in 3 digits format
3828 printf("### Submitting merging job for master <%03d>\n", i);
3829 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3830 }
7c2cd90a 3831 }
3832 // Check now the number of merging stages.
2b222f7c 3833 TObjArray *list = fOutputFiles.Tokenize(",");
3834 TIter next(list);
3835 TObjString *str;
3836 TString outputFile;
3837 while((str=(TObjString*)next())) {
3838 outputFile = str->GetString();
3839 Int_t index = outputFile.Index("@");
3840 if (index > 0) outputFile.Remove(index);
37d21c01 3841 if (!fMergeExcludes.Contains(outputFile) &&
3842 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3843 }
3844 delete list;
b3e07543 3845 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3846 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3847 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3848 }
a03be957 3849 if (!ntosubmit) return kTRUE;
d88e4159 3850 if (fDropToShell) {
3851 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3852 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3853 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3854 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3855 \n ################################################################################################################");
3856 gSystem->Exec("aliensh");
3857 } else {
3858 Info("StartAnalysis", "\n #### STARTED MERGING JOBS FOR YOU #### \
3859 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL') \
3860 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3861 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3862 \n ################################################################################################################");
3863 }
a03be957 3864 return kTRUE;
0f389141 3865}
3866
d2a409b2 3867//______________________________________________________________________________
a03be957 3868Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3869{
08d5b699 3870// Submit next bunch of master jobs if the queue is free. The first master job is
3871// submitted right away, while the next will not be unless the previous was split.
3872// The plugin will not submit new master jobs if there are more that 500 jobs in
3873// waiting phase.
d2a409b2 3874 static Bool_t iscalled = kFALSE;
3875 static Int_t firstmaster = 0;
3876 static Int_t lastmaster = 0;
3877 static Int_t npermaster = 0;
a03be957 3878 if (iscalled) return kTRUE;
d2a409b2 3879 iscalled = kTRUE;
3880 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3881 Int_t ntosubmit = 0;
3882 TGridResult *res;
3883 TString jobID = "";
3bdcb562 3884 Int_t nmasterjobs = fInputFiles->GetEntries();
3885 if (!fNsubmitted) {
3886 ntosubmit = 1;
3887 if (!IsUseSubmitPolicy()) {
90d50a8c 3888 if (nmasterjobs>5)
3bdcb562 3889 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3890 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3891 ntosubmit = nmasterjobs;
3892 }
3893 } else {
d2a409b2 3894 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3895 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3896 // If last master not split, just return
a03be957 3897 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3898 // No more than 100 waiting jobs
08d5b699 3899 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3900 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3901 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3902 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3903 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3904 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3905 }
d2a409b2 3906 for (Int_t i=0; i<ntosubmit; i++) {
3907 // Submit for a range of enumeration of runs.
a03be957 3908 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3909 TString query;
cd11251e 3910 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3911 runOutDir.ReplaceAll(".xml", "");
3912 if (fOutputToRunNo)
3913 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3914 else
3915 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3916 printf("********* %s\n",query.Data());
d2a409b2 3917 res = gGrid->Command(query);
3918 if (res) {
98ca124f 3919 TString cjobId1 = res->GetKey(0,"jobId");
3920 if (!cjobId1.Length()) {
d2a409b2 3921 iscalled = kFALSE;
a03be957 3922 gGrid->Stdout();
3923 gGrid->Stderr();
3924 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3925 return kFALSE;
d2a409b2 3926 } else {
3927 Info("StartAnalysis", "\n_______________________________________________________________________ \
3928 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3929 \n_______________________________________________________________________",
98ca124f 3930 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
0ad3ea1f 3931 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3932 fGridJobIDs.Append(cjobId1);
3933 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3934 fGridStages.Append("full");
d2a409b2 3935 jobID += cjobId1;
3936 jobID += " ";
98ca124f 3937 lastmaster = cjobId1.Atoi();
d2a409b2 3938 if (!firstmaster) firstmaster = lastmaster;
3939 fNsubmitted++;
3940 }
3941 delete res;
a03be957 3942 } else {
3943 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3944 return kFALSE;
d2a409b2 3945 }
3946 }
3947 iscalled = kFALSE;
a03be957 3948 return kTRUE;
d2a409b2 3949}
3950
c57f56b7 3951//______________________________________________________________________________
3952void AliAnalysisAlien::WriteAnalysisFile()
3953{
f10e8481 3954// Write current analysis manager into the file <analysisFile>
3955 TString analysisFile = fExecutable;
3956 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3957 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3958 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3959 if (!mgr || !mgr->IsInitialized()) {
3960 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3961 return;
3962 }
3963 // Check analysis type
3964 TObject *handler;
3965 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3966 handler = (TObject*)mgr->GetInputEventHandler();
3967 if (handler) {
65eb22f8 3968 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3969 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3970 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3971 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3972 } else {
3973 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3974 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3975 }
c57f56b7 3976 }
3977 TDirectory *cdir = gDirectory;
f10e8481 3978 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3979 if (file) {
e16a394c 3980 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3981 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3982 // Unless merging makes no sense
3983 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3984 mgr->Write();
3985 delete file;
fe2d7fc2 3986 // Enable termination for local jobs
3987 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3988 }
3989 if (cdir) cdir->cd();
f10e8481 3990 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3991 }
3992 Bool_t copy = kTRUE;
d3b18c4c 3993 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3994 if (copy) {
3995 CdWork();
3996 TString workdir = gGrid->GetHomeDirectory();
3997 workdir += fGridWorkingDir;
d3b18c4c 3998 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3999 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 4000 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
4001 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 4002 }
4003}
4004
4005//______________________________________________________________________________
4006void AliAnalysisAlien::WriteAnalysisMacro()
4007{
4008// Write the analysis macro that will steer the analysis in grid mode.
4009 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4010 ofstream out;
4011 out.open(fAnalysisMacro.Data(), ios::out);
4012 if (!out.good()) {
4013 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4014 return;
4015 }
5b273635 4016 Bool_t hasSTEERBase = kFALSE;
4017 Bool_t hasESD = kFALSE;
4018 Bool_t hasAOD = kFALSE;
4019 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4020 Bool_t hasOADB = kFALSE;
5b273635 4021 Bool_t hasANALYSISalice = kFALSE;
4022 Bool_t hasCORRFW = kFALSE;
c57f56b7 4023 TString func = fAnalysisMacro;
4024 TString type = "ESD";
4025 TString comment = "// Analysis using ";
715da79c 4026 if (fMCLoop) {
4027 type = "MCGEN";
4028 comment += "MCGEN";
4029 } else {
4030 if (IsUseMCchain()) {
4031 type = "MC";
4032 comment += "MC";
4033 } else {
4034 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
4035 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
4036 type = "AOD";
4037 comment += "AOD";
4038 }
4039 }
4040 }
0df6ccf2 4041 if (type!="AOD" && fFriendChainName!="") {
4042 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
4043 return;
4044 }
c57f56b7 4045 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
4046 else comment += " data";
4047 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
4048 func.ReplaceAll(".C", "");
4049 out << "void " << func.Data() << "()" << endl;
4050 out << "{" << endl;
4051 out << comment.Data() << endl;
4052 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 4053 out << " TStopwatch timer;" << endl;
4054 out << " timer.Start();" << endl << endl;
13f0c00b 4055 // Change temp directory to current one
fdee57f1 4056 if (!IsLocalTest()) {
4057 out << "// connect to AliEn and make the chain" << endl;
4058 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
4059 }
13f0c00b 4060 out << "// Set temporary merging directory to current one" << endl;
4061 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4062 out << "// Set temporary compilation directory to current one" << endl;
4063 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 4064 // Reset existing include path
4065 out << "// Reset existing include path and add current directory first in the search" << endl;
4066 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 4067 if (!fExecutableCommand.Contains("aliroot")) {
4068 out << "// load base root libraries" << endl;
4069 out << " gSystem->Load(\"libTree\");" << endl;
4070 out << " gSystem->Load(\"libGeom\");" << endl;
4071 out << " gSystem->Load(\"libVMC\");" << endl;
4072 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4073 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4074 }
d5c6455a 4075 if (fAdditionalRootLibs.Length()) {
47a4137d 4076 // in principle libtree /lib geom libvmc etc. can go into this list, too
4077 out << "// Add aditional libraries" << endl;
4078 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4079 TIter next(list);
4080 TObjString *str;
4081 while((str=(TObjString*)next())) {
2dffffd6 4082 if (str->GetString().Contains(".so") || str->GetString().Contains(".dylib"))
47a4137d 4083 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 4084 }
47a4137d 4085 if (list) delete list;
d5c6455a 4086 }
57377eb5 4087 out << "// Load analysis framework libraries" << endl;
652561d5 4088 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 4089 if (!fPackages) {
f790bc1b 4090 if (!fExecutableCommand.Contains("aliroot")) {
4091 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4092 out << " gSystem->Load(\"libESD\");" << endl;
4093 out << " gSystem->Load(\"libAOD\");" << endl;
4094 }
4e5c5506 4095 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4096 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 4097 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4098 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 4099 } else {
4e5c5506 4100 TIter next(fPackages);
4101 TObject *obj;
57377eb5 4102 TString pkgname;
57377eb5 4103 while ((obj=next())) {
4104 pkgname = obj->GetName();
4478e6f1 4105 if (pkgname == "STEERBase" ||
4106 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4107 if (pkgname == "ESD" ||
4108 pkgname == "ESD.par") hasESD = kTRUE;
4109 if (pkgname == "AOD" ||
4110 pkgname == "AOD.par") hasAOD = kTRUE;
4111 if (pkgname == "ANALYSIS" ||
4112 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4113 if (pkgname == "OADB" ||
4114 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 4115 if (pkgname == "ANALYSISalice" ||
4116 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4117 if (pkgname == "CORRFW" ||
4118 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 4119 }
4120 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 4121 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4122 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 4123 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4124 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 4125 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4126 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 4127 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4128 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4129 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4130 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 4131 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4132 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 4133 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4134 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 4135 out << "// Compile other par packages" << endl;
4136 next.Reset();
fcc9bb6f 4137 while ((obj=next())) {
4138 pkgname = obj->GetName();
4478e6f1 4139 if (pkgname == "STEERBase" ||
4140 pkgname == "STEERBase.par" ||
4141 pkgname == "ESD" ||
4142 pkgname == "ESD.par" ||
4143 pkgname == "AOD" ||
4144 pkgname == "AOD.par" ||
4145 pkgname == "ANALYSIS" ||
4146 pkgname == "ANALYSIS.par" ||
7ea40ac8 4147 pkgname == "OADB" ||
4148 pkgname == "OADB.par" ||
4478e6f1 4149 pkgname == "ANALYSISalice" ||
4150 pkgname == "ANALYSISalice.par" ||
4151 pkgname == "CORRFW" ||
4152 pkgname == "CORRFW.par") continue;
5b273635 4153 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 4154 }
4e5c5506 4155 }
b3e07543 4156 out << "// include path" << endl;
4157 // Get the include path from the interpreter and remove entries pointing to AliRoot
4158 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4159 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4160 out << " TIter nextpath(listpaths);" << endl;
4161 out << " TObjString *pname;" << endl;
4162 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4163 out << " TString current = pname->GetName();" << endl;
4164 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4165 out << " gSystem->AddIncludePath(current);" << endl;
4166 out << " }" << endl;
4167 out << " if (listpaths) delete listpaths;" << endl;
4168 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4169 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4170 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
715da79c 4171 if (fMCLoop && !fGeneratorLibs.IsNull()) {
4172 out << "// MC generator libraries" << endl;
4173 TObjArray *list = fGeneratorLibs.Tokenize(" ");
4174 TIter next(list);
4175 TObjString *str;
4176 while((str=(TObjString*)next())) {
4177 out << " gSystem->Load(\"" << str->GetName() << "\");" << endl;
4178 }
4179 delete list;
4180 }
6da75e0b 4181 if (fAdditionalLibs.Length()) {
4182 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4183 TString additionalLibs = fAdditionalLibs;
4184 additionalLibs.Strip();
4185 if (additionalLibs.Length() && fFriendLibs.Length())
4186 additionalLibs += " ";
4187 additionalLibs += fFriendLibs;
4188 TObjArray *list = additionalLibs.Tokenize(" ");
6da75e0b 4189 TIter next(list);
4190 TObjString *str;
4191 while((str=(TObjString*)next())) {
2dffffd6 4192 if (str->GetString().Contains(".so") || str->GetString().Contains(".dylib"))
6da75e0b 4193 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 4194 if (str->GetString().Contains(".par"))
4195 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 4196 }
715da79c 4197 delete list;
6da75e0b 4198 }
4199 out << endl;
c57f56b7 4200 out << "// analysis source to be compiled at runtime (if any)" << endl;
4201 if (fAnalysisSource.Length()) {
4202 TObjArray *list = fAnalysisSource.Tokenize(" ");
4203 TIter next(list);
4204 TObjString *str;
4205 while((str=(TObjString*)next())) {
4206 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4207 }
4208 if (list) delete list;
4209 }
4210 out << endl;
b3e07543 4211// out << " printf(\"Currently load libraries:\\n\");" << endl;
4212// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 4213 if (fFastReadOption) {
d3339be3 4214 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
4215 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 4216 out << "// fast xrootd reading enabled" << endl;
4217 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4218 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4219 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4220 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4221 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4222 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 4223 }
c57f56b7 4224 out << "// read the analysis manager from file" << endl;
f10e8481 4225 TString analysisFile = fExecutable;
4226 analysisFile.ReplaceAll(".sh", ".root");
105631b5 4227 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4228 << analysisFile << "\");" << endl;
4229 out << " if (!mgr) return;" << endl;
21ca8e59 4230 if (IsLocalTest()) {
4231 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
4232 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 4233 if (fFileForTestMode.IsNull())
4234 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
4235 else
4236 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 4237 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4b5d9e21 4238 if (!fFriendChainName.IsNull())
b9f1efa5 4239 out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\",\"" << fFriendLibs << "\");" << endl;
024dfa52 4240 if (IsUseMCchain())
4241 out << " plugin->SetUseMCchain();" << endl;
715da79c 4242 if (fMCLoop)
4243 out << " plugin->SetMCLoop(kTRUE);" << endl;
21ca8e59 4244 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 4245 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 4246 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 4247 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
4248 } else {
7a366e0b 4249 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 4250 out << " mgr->SetNSysInfo(100);" << endl;
4251 }
21ca8e59 4252 }
c57f56b7 4253 out << " mgr->PrintStatus();" << endl;
52b6a92b 4254 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 4255 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 4256 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 4257 } else {
9a8288ff 4258 if (TestBit(AliAnalysisGrid::kTest))
4259 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4260 else
4261 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 4262 }
4263 }
21ca8e59 4264 if (!IsLocalTest()) {
715da79c 4265 if (fMCLoop) {
4266 out << " mgr->SetCacheSize(0);" << endl;
4267 out << " mgr->EventLoop(" << fNMCevents << ");" << endl;
4268 } else {
4269 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
4270 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
4271 }
21ca8e59 4272 } else {
715da79c 4273 if (fMCLoop) {
4274 out << " mgr->SetCacheSize(0);" << endl;
4275 out << " mgr->EventLoop(" << fNMCevents << ");" << endl;
7435f77f
AG		 4276 } else {
4277 out << " mgr->StartAnalysis(\"localfile\");" << endl;
4278 }
242accb2 4279 }
f7498086 4280 out << " timer.Stop();" << endl;
4281 out << " timer.Print();" << endl;
c57f56b7 4282 out << "}" << endl << endl;
715da79c 4283 if (!IsLocalTest() && !fMCLoop) {
fcc9bb6f 4284 out <<"//________________________________________________________________________________" << endl;
c57f56b7 4285 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
4286 out << "{" << endl;
4287 out << "// Create a chain using url's from xml file" << endl;
242accb2 4288 out << " TString filename;" << endl;
4289 out << " Int_t run = 0;" << endl;
c85cfc0f 4290 if (IsUseMCchain()) {
4291 out << " TString treename = \"TE\";" << endl;
4292 } else {
780ec36d 4293 if (!fTreeName.IsNull()) {
4294 out << " TString treename = \"" << fTreeName << "\";" << endl;
4295 } else {
4296 out << " TString treename = type;" << endl;
4297 out << " treename.ToLower();" << endl;
4298 out << " treename += \"Tree\";" << endl;
4299 }
c85cfc0f 4300 }
e02fee64 4301 out << " printf(\"***************************************\\n\");" << endl;
4302 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
4303 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 4304 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
4305 out << " if (!coll) {" << endl;
4306 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
4307 out << " return NULL;" << endl;
4308 out << " }" << endl;
242accb2 4309 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 4310 out << " TChain *chain = new TChain(treename);" << endl;
06f67114 4311 if(!fFriendChainName.IsNull()) {
4312 out << " TList *friends = new TList();" << endl;
4313 out << " TIter nextfriend(friends);" << endl;
4314 out << " TChain *cfriend = 0;" << endl;
4315 TObjArray *list = fFriendChainName.Tokenize(" ");
4316 TIter next(list);
4317 TObjString *str;
4318 while((str=(TObjString*)next())) {
4319 out << " cfriend = new TChain(treename, \"" << str->GetName() << "\");" << endl;
4320 out << " friends->Add(cfriend);" << endl;
4321 out << " chain->AddFriend(cfriend);" << endl;
4322 }
4323 delete list;
4324// out << " TChain *chainFriend = new TChain(treename);" << endl;
0df6ccf2 4325 }
c57f56b7 4326 out << " coll->Reset();" << endl;
0df6ccf2 4327 out << " while (coll->Next()) {" << endl;
242accb2 4328 out << " filename = coll->GetTURL("");" << endl;
4329 out << " if (mgr) {" << endl;
4330 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
4331 out << " if (nrun && nrun != run) {" << endl;
4332 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
4333 out << " mgr->SetRunFromPath(nrun);" << endl;
4334 out << " run = nrun;" << endl;
4335 out << " }" << endl;
4336 out << " }" << endl;
4337 out << " chain->Add(filename);" << endl;
06f67114 4338 if(!fFriendChainName.IsNull()) {
4339 out << " TString bpath=coll->GetTURL(\"\");" << endl;
4340 out << " if (bpath.Index(\"#\") > -1) bpath.Remove(bpath.Index(\"#\"));" << endl;
4341 out << " bpath = gSystem->DirName(bpath);" << endl;
4342 out << " bpath += \"/\";" << endl;
4343 out << " TString fileFriend;" << endl;
4344 out << " nextfriend.Reset();" << endl;
4345 out << " while ((cfriend=(TChain*)nextfriend())) {" << endl;
4346 out << " fileFriend = bpath;" << endl;
4347 out << " fileFriend += cfriend->GetTitle();" << endl;
4348 out << " TFile *file = TFile::Open(fileFriend);" << endl;
4349 out << " if (file) {" << endl;
4350 out << " file->Close();" << endl;
4351 out << " cfriend->Add(fileFriend.Data());" << endl;
4352 out << " } else {" << endl;
4353 out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
4354 out << " return 0;" << endl;
4355 out << " }" << endl;
4b5d9e21 4356 out << " }" << endl;
0df6ccf2 4357 }
4358 out << " }" << endl;
c57f56b7 4359 out << " if (!chain->GetNtrees()) {" << endl;
4360 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
4361 out << " return NULL;" << endl;
4362 out << " }" << endl;
4363 out << " return chain;" << endl;
fcc9bb6f 4364 out << "}" << endl << endl;
c57f56b7 4365 }
5b273635 4366 if (hasANALYSISalice) {
4367 out <<"//________________________________________________________________________________" << endl;
4368 out << "Bool_t SetupPar(const char *package) {" << endl;
4369 out << "// Compile the package and set it up." << endl;
4370 out << " TString pkgdir = package;" << endl;
4371 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4372 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4373 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4374 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4375 out << " // Check for BUILD.sh and execute" << endl;
4376 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4377 out << " printf(\"*******************************\\n\");" << endl;
4378 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4379 out << " printf(\"*******************************\\n\");" << endl;
4380 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4381 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4382 out << " gSystem->ChangeDirectory(cdir);" << endl;
4383 out << " return kFALSE;" << endl;
4384 out << " }" << endl;
4385 out << " } else {" << endl;
4386 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4387 out << " gSystem->ChangeDirectory(cdir);" << endl;
4388 out << " return kFALSE;" << endl;
4389 out << " }" << endl;
4390 out << " // Check for SETUP.C and execute" << endl;
4391 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4392 out << " printf(\"*******************************\\n\");" << endl;
4393 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4394 out << " printf(\"*******************************\\n\");" << endl;
4395 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4396 out << " } else {" << endl;
4397 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4398 out << " gSystem->ChangeDirectory(cdir);" << endl;
4399 out << " return kFALSE;" << endl;
4400 out << " }" << endl;
4401 out << " // Restore original workdir" << endl;
4402 out << " gSystem->ChangeDirectory(cdir);" << endl;
4403 out << " return kTRUE;" << endl;
4404 out << "}" << endl;
4405 }
c57f56b7 4406 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4407 }
4408 Bool_t copy = kTRUE;
d3b18c4c 4409 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4410 if (copy) {
4411 CdWork();
4412 TString workdir = gGrid->GetHomeDirectory();
4413 workdir += fGridWorkingDir;
4414 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4415 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4416// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4417 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4418 Form("alien://%s/%s", workdir.Data(),
4419 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4420 }
4421}
4422
0f389141 4423//______________________________________________________________________________
4424void AliAnalysisAlien::WriteMergingMacro()
4425{
4426// Write a macro to merge the outputs per master job.
4427 if (!fMergeViaJDL) return;
4428 if (!fOutputFiles.Length()) {
4429 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4430 return;
4431 }
ec5acdce 4432 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
0f389141 4433 TString mergingMacro = fExecutable;
4434 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4435 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4436 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4437 ofstream out;
4438 out.open(mergingMacro.Data(), ios::out);
4439 if (!out.good()) {
4440 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4441 return;
4442 }
5b273635 4443 Bool_t hasSTEERBase = kFALSE;
4444 Bool_t hasESD = kFALSE;
4445 Bool_t hasAOD = kFALSE;
4446 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4447 Bool_t hasOADB = kFALSE;
5b273635 4448 Bool_t hasANALYSISalice = kFALSE;
4449 Bool_t hasCORRFW = kFALSE;
0f389141 4450 TString func = mergingMacro;
4451 TString comment;
4452 func.ReplaceAll(".C", "");
f47d5cb4 4453 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4454 out << "{" << endl;
4455 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4456 out << " TStopwatch timer;" << endl;
4457 out << " timer.Start();" << endl << endl;
b3e07543 4458 // Reset existing include path
4459 out << "// Reset existing include path and add current directory first in the search" << endl;
4460 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4461 if (!fExecutableCommand.Contains("aliroot")) {
4462 out << "// load base root libraries" << endl;
4463 out << " gSystem->Load(\"libTree\");" << endl;
4464 out << " gSystem->Load(\"libGeom\");" << endl;
4465 out << " gSystem->Load(\"libVMC\");" << endl;
4466 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4467 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4468 }
0f389141 4469 if (fAdditionalRootLibs.Length()) {
4470 // in principle libtree /lib geom libvmc etc. can go into this list, too
4471 out << "// Add aditional libraries" << endl;
4472 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4473 TIter next(list);
4474 TObjString *str;
4475 while((str=(TObjString*)next())) {
2dffffd6 4476 if (str->GetString().Contains(".so") || str->GetString().Contains(".dylib"))
0f389141 4477 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4478 }
4479 if (list) delete list;
4480 }
0f389141 4481 out << "// Load analysis framework libraries" << endl;
4482 if (!fPackages) {
7c2cd90a 4483 if (!fExecutableCommand.Contains("aliroot")) {
4484 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4485 out << " gSystem->Load(\"libESD\");" << endl;
4486 out << " gSystem->Load(\"libAOD\");" << endl;
4487 }
0f389141 4488 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4489 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4490 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4491 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4492 } else {
4493 TIter next(fPackages);
4494 TObject *obj;
4495 TString pkgname;
5b273635 4496 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4497 while ((obj=next())) {
4498 pkgname = obj->GetName();
4499 if (pkgname == "STEERBase" ||
4500 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4501 if (pkgname == "ESD" ||
4502 pkgname == "ESD.par") hasESD = kTRUE;
4503 if (pkgname == "AOD" ||
4504 pkgname == "AOD.par") hasAOD = kTRUE;
4505 if (pkgname == "ANALYSIS" ||
4506 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4507 if (pkgname == "OADB" ||
4508 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4509 if (pkgname == "ANALYSISalice" ||
4510 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4511 if (pkgname == "CORRFW" ||
4512 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4513 }
5b273635 4514 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4515 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4516 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4517 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4518 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4519 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4520 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4521 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4522 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4523 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4524 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4525 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4526 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4527 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4528 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4529 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4530 out << "// Compile other par packages" << endl;
4531 next.Reset();
4532 while ((obj=next())) {
4533 pkgname = obj->GetName();
4534 if (pkgname == "STEERBase" ||
4535 pkgname == "STEERBase.par" ||
4536 pkgname == "ESD" ||
4537 pkgname == "ESD.par" ||
4538 pkgname == "AOD" ||
4539 pkgname == "AOD.par" ||
4540 pkgname == "ANALYSIS" ||
4541 pkgname == "ANALYSIS.par" ||
7ea40ac8 4542 pkgname == "OADB" ||
4543 pkgname == "OADB.par" ||
0f389141 4544 pkgname == "ANALYSISalice" ||
4545 pkgname == "ANALYSISalice.par" ||
4546 pkgname == "CORRFW" ||
4547 pkgname == "CORRFW.par") continue;
5b273635 4548 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4549 }
4550 }
b3e07543 4551 out << "// include path" << endl;
4552 // Get the include path from the interpreter and remove entries pointing to AliRoot
4553 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4554 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4555 out << " TIter nextpath(listpaths);" << endl;
4556 out << " TObjString *pname;" << endl;
4557 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4558 out << " TString current = pname->GetName();" << endl;
4559 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4560 out << " gSystem->AddIncludePath(current);" << endl;
4561 out << " }" << endl;
4562 out << " if (listpaths) delete listpaths;" << endl;
4563 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4564 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4565 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0d239a91
AG		 4566 if (fMCLoop && !fGeneratorLibs.IsNull()) {
4567 out << "// MC generator libraries" << endl;
4568 TObjArray *list = fGeneratorLibs.Tokenize(" ");
4569 TIter next(list);
4570 TObjString *str;
4571 while((str=(TObjString*)next())) {
4572 out << " gSystem->Load(\"" << str->GetName() << "\");" << endl;
4573 }
4574 delete list;
4575 }
0f389141 4576 if (fAdditionalLibs.Length()) {
4577 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4578 TString additionalLibs = fAdditionalLibs;
4579 additionalLibs.Strip();
4580 if (additionalLibs.Length() && fFriendLibs.Length())
4581 additionalLibs += " ";
4582 additionalLibs += fFriendLibs;
4583 TObjArray *list = additionalLibs.Tokenize(" ");
0f389141 4584 TIter next(list);
4585 TObjString *str;
4586 while((str=(TObjString*)next())) {
2dffffd6 4587 if (str->GetString().Contains(".so") || str->GetString().Contains(".dylib"))
0f389141 4588 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4589 }
4590 if (list) delete list;
4591 }
4592 out << endl;
4593 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4594 if (fAnalysisSource.Length()) {
4595 TObjArray *list = fAnalysisSource.Tokenize(" ");
4596 TIter next(list);
4597 TObjString *str;
4598 while((str=(TObjString*)next())) {
4599 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4600 }
4601 if (list) delete list;
4602 }
149d288c 4603 out << endl;
4604
0f389141 4605 if (fFastReadOption) {
4606 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4607 out << "// fast xrootd reading enabled" << endl;
4608 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4609 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4610 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4611 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4612 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4613 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4614 }
4615 // Change temp directory to current one
fdee57f1 4616 out << "// Connect to AliEn" << endl;
4617 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
e8b839ab 4618 out << "// Set temporary merging directory to current one" << endl;
4619 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4620 out << "// Set temporary compilation directory to current one" << endl;
4621 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
7c2cd90a 4622 out << " TString outputDir = dir;" << endl;
d87f72d5 4623 if (IsMergeAOD())
4624 out << " TString outputFiles = \"" << GetListOfFiles("outaod") << "\";" << endl;
4625 else
4626 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4627 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4628 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4629 out << " TIter *iter = new TIter(list);" << endl;
4630 out << " TObjString *str;" << endl;
a2f5fc01 4631 out << " TString outputFile;" << endl;
0f389141 4632 out << " Bool_t merged = kTRUE;" << endl;
ec5acdce 4633 TString analysisFile = fExecutable;
4634 analysisFile.ReplaceAll(".sh", ".root");
4635 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4636 << analysisFile << "\");" << endl;
4637 out << " if (!mgr) {" << endl;
4638 out << " printf(\"ERROR: Analysis manager could not be extracted from file \");" << endl;
4639 out << " return;" << endl;
4640 out << " }" << endl;
d66aa2a9 4641 if (IsLocalTest()) {
0c0825e4 4642 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4643 out << " printf(\"Testing merging...\\n\");" << endl;
0c0825e4 4644 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4645 }
0f389141 4646 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4647 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4648 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4649 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4650 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4651 out << " // Skip already merged outputs" << endl;
a2f5fc01 4652 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
d66aa2a9 4653 out << " printf(\"Output file <%s> found. Not merging again.\\n\",outputFile.Data());" << endl;
0f389141 4654 out << " continue;" << endl;
4655 out << " }" << endl;
a2f5fc01 4656 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4657 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4658 out << " if (!merged) {" << endl;
a2f5fc01 4659 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4660 out << " return;" << endl;
0f389141 4661 out << " }" << endl;
4662 out << " }" << endl;
d66aa2a9 4663 if (mgr && mgr->IsCollectThroughput() && !IsLocalTest()) {
ec5acdce 4664 out << " TString infolog = \"" << mgr->GetFileInfoLog() << "\";" << endl;
4665 out << " AliAnalysisAlien::MergeInfo(infolog, outputDir);" << endl;
4666 }
7c2cd90a 4667 out << " // all outputs merged, validate" << endl;
4668 out << " ofstream out;" << endl;
4669 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4670 out << " out.close();" << endl;
4671 out << " // read the analysis manager from file" << endl;
d66aa2a9 4672 if (IsLocalTest()) {
0c0825e4 4673 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4674 out << " printf(\"Testing Terminate()...\\n\");" << endl;
0c0825e4 4675 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4676 } else {
4677 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
4678 }
5c4250fc 4679 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4680 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4681 out << " mgr->PrintStatus();" << endl;
ec5acdce 4682 if (mgr) {
4683 if (mgr->GetDebugLevel()>3) {
f866cba5 4684 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4685 } else {
9a8288ff 4686 if (TestBit(AliAnalysisGrid::kTest))
4687 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4688 else
4689 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4690 }
4691 }
9a8288ff 4692 out << " TTree *tree = NULL;" << endl;
4693 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4694 out << "}" << endl << endl;
5b273635 4695 if (hasANALYSISalice) {
4696 out <<"//________________________________________________________________________________" << endl;
4697 out << "Bool_t SetupPar(const char *package) {" << endl;
4698 out << "// Compile the package and set it up." << endl;
4699 out << " TString pkgdir = package;" << endl;
4700 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4701 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4702 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4703 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4704 out << " // Check for BUILD.sh and execute" << endl;
4705 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4706 out << " printf(\"*******************************\\n\");" << endl;
4707 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4708 out << " printf(\"*******************************\\n\");" << endl;
4709 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4710 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4711 out << " gSystem->ChangeDirectory(cdir);" << endl;
4712 out << " return kFALSE;" << endl;
4713 out << " }" << endl;
4714 out << " } else {" << endl;
4715 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4716 out << " gSystem->ChangeDirectory(cdir);" << endl;
4717 out << " return kFALSE;" << endl;
4718 out << " }" << endl;
4719 out << " // Check for SETUP.C and execute" << endl;
4720 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4721 out << " printf(\"*******************************\\n\");" << endl;
4722 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4723 out << " printf(\"*******************************\\n\");" << endl;
4724 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4725 out << " } else {" << endl;
4726 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4727 out << " gSystem->ChangeDirectory(cdir);" << endl;
4728 out << " return kFALSE;" << endl;
4729 out << " }" << endl;
4730 out << " // Restore original workdir" << endl;
4731 out << " gSystem->ChangeDirectory(cdir);" << endl;
4732 out << " return kTRUE;" << endl;
4733 out << "}" << endl;
4734 }
0f389141 4735 }
4736 Bool_t copy = kTRUE;
d3b18c4c 4737 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4738 if (copy) {
4739 CdWork();
4740 TString workdir = gGrid->GetHomeDirectory();
4741 workdir += fGridWorkingDir;
4742 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4743 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4744// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4745 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4746 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4747 }
4748}
4749
4750//______________________________________________________________________________
4751Bool_t AliAnalysisAlien::SetupPar(const char *package)
4752{
205b201f 4753// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4754// Note that for loading the compiled library. The current directory should have precedence in
4755// LD_LIBRARY_PATH
4756 TString pkgdir = package;
4757 pkgdir.ReplaceAll(".par","");
b50a41c3 4758 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4759 TString cdir = gSystem->WorkingDirectory();
4760 gSystem->ChangeDirectory(pkgdir);
4761 // Check for BUILD.sh and execute
4762 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4763 printf("**************************************************\n");
4764 printf("*** Building PAR archive %s\n", package);
4765 printf("**************************************************\n");
4766 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4767 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4768 gSystem->ChangeDirectory(cdir);
4769 return kFALSE;
4770 }
4771 } else {
4772 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4773 gSystem->ChangeDirectory(cdir);
4774 return kFALSE;
4775 }
4776 // Check for SETUP.C and execute
4777 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4778 printf("**************************************************\n");
4779 printf("*** Setup PAR archive %s\n", package);
4780 printf("**************************************************\n");
4781 gROOT->Macro("PROOF-INF/SETUP.C");
4782 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4783 } else {
4784 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4785 gSystem->ChangeDirectory(cdir);
4786 return kFALSE;
4787 }
4788 // Restore original workdir
4789 gSystem->ChangeDirectory(cdir);
4790 return kTRUE;
4791}
4792
c57f56b7 4793//______________________________________________________________________________
4794void AliAnalysisAlien::WriteExecutable()
4795{
4796// Generate the alien executable script.
5aa94dc3 4797 // Patch executable with -x to catch error code
4798 if (fExecutableCommand.Contains("root") &&
4799 fExecutableCommand.Contains("-q") &&
4800 !fExecutableCommand.Contains("-x")) fExecutableCommand += " -x";
c57f56b7 4801 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4802 ofstream out;
4803 out.open(fExecutable.Data(), ios::out);
4804 if (out.bad()) {
5513444a 4805 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4806 return;
4807 }
4808 out << "#!/bin/bash" << endl;
2a4d5166 4809 // Make sure we can properly compile par files
4810 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4811 out << "echo \"=========================================\"" << endl;
4812 out << "echo \"############## PATH : ##############\"" << endl;
4813 out << "echo $PATH" << endl;
4814 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4815 out << "echo $LD_LIBRARY_PATH" << endl;
4816 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4817 out << "echo $ROOTSYS" << endl;
4818 out << "echo \"############## which root : ##############\"" << endl;
4819 out << "which root" << endl;
4820 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4821 out << "echo $ALICE_ROOT" << endl;
4822 out << "echo \"############## which aliroot : ##############\"" << endl;
4823 out << "which aliroot" << endl;
9c5ddadc 4824 out << "echo \"############## system limits : ##############\"" << endl;
4825 out << "ulimit -a" << endl;
4826 out << "echo \"############## memory : ##############\"" << endl;
4827 out << "free -m" << endl;
c57f56b7 4828 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4829 out << fExecutableCommand << " ";
370c9880 4830 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4831 out << "RET=$?" << endl;
4832 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4833 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4834 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4835 out << " let sig=\"$RET - 128\""<<endl;
4836 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4837 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4838 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4839 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4840 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4841 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4842 out << " fi"<<endl;
4843 out << " exit $RET"<< endl;
4844 out << "fi" << endl << endl ;
4845 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4846 out << "echo \"############## memory after: ##############\"" << endl;
4847 out << "free -m" << endl;
c57f56b7 4848 }
4849 Bool_t copy = kTRUE;
d3b18c4c 4850 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4851 if (copy) {
4852 CdWork();
4853 TString workdir = gGrid->GetHomeDirectory();
4854 workdir += fGridWorkingDir;
6da4839e 4855 TString executable = TString::Format("%s/%s", workdir.Data(), fExecutable.Data());
c57f56b7 4856 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4857 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4858 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4859 executable.Data())) Fatal("","Terminating");
c57f56b7 4860 }
4861}
4862
0f389141 4863//______________________________________________________________________________
4864void AliAnalysisAlien::WriteMergeExecutable()
4865{
4866// Generate the alien executable script for the merging job.
4867 if (!fMergeViaJDL) return;
4868 TString mergeExec = fExecutable;
4869 mergeExec.ReplaceAll(".sh", "_merge.sh");
4870 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4871 ofstream out;
4872 out.open(mergeExec.Data(), ios::out);
4873 if (out.bad()) {
4874 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4875 return;
4876 }
4877 out << "#!/bin/bash" << endl;
2a4d5166 4878 // Make sure we can properly compile par files
4879 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4880 out << "echo \"=========================================\"" << endl;
4881 out << "echo \"############## PATH : ##############\"" << endl;
4882 out << "echo $PATH" << endl;
4883 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4884 out << "echo $LD_LIBRARY_PATH" << endl;
4885 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4886 out << "echo $ROOTSYS" << endl;
4887 out << "echo \"############## which root : ##############\"" << endl;
4888 out << "which root" << endl;
4889 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4890 out << "echo $ALICE_ROOT" << endl;
4891 out << "echo \"############## which aliroot : ##############\"" << endl;
4892 out << "which aliroot" << endl;
4893 out << "echo \"############## system limits : ##############\"" << endl;
4894 out << "ulimit -a" << endl;
4895 out << "echo \"############## memory : ##############\"" << endl;
4896 out << "free -m" << endl;
4897 out << "echo \"=========================================\"" << endl << endl;
0f389141 4898 TString mergeMacro = fExecutable;
4899 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4900 if (IsOneStageMerging())
4901 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4902 else
f47d5cb4 4903 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4904 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4905 out << "RET=$?" << endl;
4906 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4907 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4908 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4909 out << " let sig=\"$RET - 128\""<<endl;
4910 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4911 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4912 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4913 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4914 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4915 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4916 out << " fi"<<endl;
4917 out << " exit $RET"<< endl;
4918 out << "fi" << endl << endl ;
0f389141 4919 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4920 out << "echo \"############## memory after: ##############\"" << endl;
4921 out << "free -m" << endl;
0f389141 4922 }
4923 Bool_t copy = kTRUE;
d3b18c4c 4924 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4925 if (copy) {
4926 CdWork();
4927 TString workdir = gGrid->GetHomeDirectory();
0f389141 4928 workdir += fGridWorkingDir;
6da4839e 4929 TString executable = TString::Format("%s/%s", workdir.Data(), mergeExec.Data());
0f389141 4930 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4931 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6da4839e
AG		 4932 if (!copyLocal2Alien("WriteMergeExecutable",mergeExec.Data(),
4933 executable.Data())) Fatal("","Terminating");
0f389141 4934 }
4935}
4936
c57f56b7 4937//______________________________________________________________________________
5513444a 4938void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4939{
4940// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4941// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4942// Next lines: full_path_to_dataset XXX (XXX is a string)
4943// To submit, one has to: submit jdl XXX for all lines
4944 ofstream out;
4945 out.open(filename, ios::out);
4946 if (out.bad()) {
4947 Error("WriteProductionFile", "Bad file name: %s", filename);
4948 return;
4949 }
d3b18c4c 4950 TString workdir;
4951 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4952 workdir = gGrid->GetHomeDirectory();
5513444a 4953 workdir += fGridWorkingDir;
f5e8c702 4954 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4955 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4956 out << locjdl << " " << njobspermaster << endl;
5513444a 4957 Int_t nmasterjobs = fInputFiles->GetEntries();
4958 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4959 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4960 runOutDir.ReplaceAll(".xml", "");
4961 if (fOutputToRunNo)
4962 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4963 else
4964 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4965 }
d3b18c4c 4966 if (gGrid) {
4967 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4968 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4969// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4970 if (!copyLocal2Alien("WriteProductionFile", filename,
4971 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4972 }
5513444a 4973}
4974
4975//______________________________________________________________________________
0f389141 4976void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4977{
4978// Generate the alien validation script.
4979 // Generate the validation script
4980 TObjString *os;
d3b18c4c 4981 if (fValidationScript.IsNull()) {
4982 fValidationScript = fExecutable;
4983 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4984 }
4985 TString validationScript = fValidationScript;
4986 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4987 if (!Connect()) {
4988 Error("WriteValidationScript", "Alien connection required");
4989 return;
4990 }
d3b18c4c 4991 if (!fTerminateFiles.IsNull()) {
4992 fTerminateFiles.Strip();
4993 fTerminateFiles.ReplaceAll(" ",",");
4994 }
a2f5fc01 4995 TString outStream = "";
4996 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4997 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4998 ofstream out;
0d5d317c 4999 out.open(validationScript, ios::out);
c57f56b7 5000 out << "#!/bin/bash" << endl;
5001 out << "##################################################" << endl;
5002 out << "validateout=`dirname $0`" << endl;
5003 out << "validatetime=`date`" << endl;
5004 out << "validated=\"0\";" << endl;
5005 out << "error=0" << endl;
5006 out << "if [ -z $validateout ]" << endl;
5007 out << "then" << endl;
5008 out << " validateout=\".\"" << endl;
5009 out << "fi" << endl << endl;
5010 out << "cd $validateout;" << endl;
5011 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 5012 out << "echo \"*******************************************************\"" << outStream << endl;
5013 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 5014 out << "" << endl;
a2f5fc01 5015 out << "echo \"* Time: $validatetime \"" << outStream << endl;
5016 out << "echo \"* Dir: $validateout\"" << outStream << endl;
5017 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
5018 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5019 out << "ls -la ./" << outStream << endl;
5020 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 5021 out << "##################################################" << endl;
ebec370a 5022 out << "" << endl;
5023
5024 out << "if [ ! -f stderr ] ; then" << endl;
5025 out << " error=1" << endl;
a2f5fc01 5026 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
5027 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5028 out << "fi" << endl;
5029
b34c9f51 5030 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
5031 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
5032 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
5033 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
5034 out << "" << endl;
5035
ebec370a 5036 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
5037 out << " error=1" << endl;
a2f5fc01 5038 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
5039 out << " echo \"$parArch\" " << outStream << endl;
5040 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5041 out << "fi" << endl;
5042
5043 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
5044 out << " error=1" << endl;
a2f5fc01 5045 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
5046 out << " echo \"$segViol\" " << outStream << endl;
5047 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5048 out << "fi" << endl;
5049
5050 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
5051 out << " error=1" << endl;
a2f5fc01 5052 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
5053 out << " echo \"$segFault\" " << outStream << endl;
5054 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5055 out << "fi" << endl;
5056
b34c9f51 5057 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
5058 out << " error=1" << endl;
a2f5fc01 5059 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
5060 out << " echo \"$glibcErr\" " << outStream << endl;
5061 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 5062 out << "fi" << endl;
5063
ebec370a 5064 // Part dedicated to the specific analyses running into the train
5065
d3b18c4c 5066 TString outputFiles = fOutputFiles;
5067 if (merge && !fTerminateFiles.IsNull()) {
0c0825e4 5068 if (!outputFiles.IsNull()) outputFiles += ",";
d3b18c4c 5069 outputFiles += fTerminateFiles;
5070 }
5071 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 5072 TIter next1(arr);
a2f5fc01 5073 TString outputFile;
bbd88db8 5074 while (!merge && (os=(TObjString*)next1())) {
5075 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 5076 outputFile = os->GetString();
5077 Int_t index = outputFile.Index("@");
5078 if (index > 0) outputFile.Remove(index);
bbd88db8 5079 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 5080 if (outputFile.Contains("*")) continue;
5081 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 5082 out << " error=1" << endl;
7c2cd90a 5083 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
5084 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 5085 out << "fi" << endl;
5086 }
5087 delete arr;
7c2cd90a 5088 out << "if ! [ -f outputs_valid ] ; then" << endl;
5089 out << " error=1" << endl;
5090 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
5091 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
5092 out << "fi" << endl;
923e2ca5 5093
c57f56b7 5094 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 5095 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 5096 if (!IsKeepLogs()) {
5097 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 5098 outStream = "";
149d288c 5099 out << " rm -f std*" << endl;
5100 }
c57f56b7 5101 out << "fi" << endl;
5102
a2f5fc01 5103 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5104 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 5105 out << "cd -" << endl;
5106 out << "exit $error" << endl;
5107 }
5108 Bool_t copy = kTRUE;
d3b18c4c 5109 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 5110 if (copy) {
5111 CdWork();
5112 TString workdir = gGrid->GetHomeDirectory();
5113 workdir += fGridWorkingDir;
d3b18c4c 5114 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 5115 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 5116// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
5117 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
5118 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 5119 }
5120}