]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
delete constrained tracks for all filters
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
715da79c 32#include "TInterpreter.h"
c57f56b7 33#include "TFile.h"
830acc4c 34#include "TFileCollection.h"
3bdcb562 35#include "TChain.h"
c57f56b7 36#include "TObjString.h"
37#include "TObjArray.h"
77f34eae 38#include "TMacro.h"
c57f56b7 39#include "TGrid.h"
40#include "TGridResult.h"
41#include "TGridCollection.h"
42#include "TGridJDL.h"
d2a409b2 43#include "TGridJobStatusList.h"
44#include "TGridJobStatus.h"
c57f56b7 45#include "TFileMerger.h"
46#include "AliAnalysisManager.h"
77f34eae 47#include "AliAnalysisTaskCfg.h"
bb885a9e 48#include "AliVEventHandler.h"
49#include "AliAnalysisDataContainer.h"
65eb22f8 50#include "AliMultiInputEventHandler.h"
c57f56b7 51
c82bb898 52using std::ofstream;
53using std::ifstream;
54using std::ios;
55using std::endl;
c57f56b7 56ClassImp(AliAnalysisAlien)
6c395669 57#if 0
58;
59#endif
c57f56b7 60
6c395669 61namespace {
62 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
63 {
64 TString sl(Form("file:%s", loc));
65 TString sr(Form("alien://%s", rem));
66 Bool_t ret = TFile::Cp(sl, sr);
67 if (!ret) {
68 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
69 }
70 return ret;
71 }
72}
73
c57f56b7 74//______________________________________________________________________________
75AliAnalysisAlien::AliAnalysisAlien()
76 :AliAnalysisGrid(),
77 fGridJDL(NULL),
0f389141 78 fMergingJDL(NULL),
c57f56b7 79 fPrice(0),
80 fTTL(0),
81 fSplitMaxInputFileNumber(0),
82 fMaxInitFailed(0),
83 fMasterResubmitThreshold(0),
bb885a9e 84 fNtestFiles(0),
319593fb 85 fNrunsPerMaster(0),
16a4353c 86 fMaxMergeFiles(0),
b3e07543 87 fMaxMergeStages(0),
d2a409b2 88 fNsubmitted(0),
a3e84053 89 fProductionMode(0),
cd11251e 90 fOutputToRunNo(0),
0f389141 91 fMergeViaJDL(0),
92 fFastReadOption(0),
e1c22e21 93 fOverwriteMode(1),
149d288c 94 fNreplicas(2),
3bdcb562 95 fNproofWorkers(0),
27734f0e 96 fNproofWorkersPerSlave(0),
3bdcb562 97 fProofReset(0),
715da79c 98 fNMCevents(0),
99 fNMCjobs(0),
c57f56b7 100 fRunNumbers(),
101 fExecutable(),
0a1c1f7f 102 fExecutableCommand(),
c57f56b7 103 fArguments(),
631c0b05 104 fExecutableArgs(),
c57f56b7 105 fAnalysisMacro(),
106 fAnalysisSource(),
d3b18c4c 107 fValidationScript(),
d5c6455a 108 fAdditionalRootLibs(),
c57f56b7 109 fAdditionalLibs(),
715da79c 110 fGeneratorLibs(),
c57f56b7 111 fSplitMode(),
112 fAPIVersion(),
113 fROOTVersion(),
114 fAliROOTVersion(),
648174cf 115 fExternalPackages(),
c57f56b7 116 fUser(),
117 fGridWorkingDir(),
118 fGridDataDir(),
119 fDataPattern(),
120 fGridOutputDir(),
121 fOutputArchive(),
122 fOutputFiles(),
123 fInputFormat(),
e7c71df0 124 fDatasetName(),
c57f56b7 125 fJDLName(),
d3b18c4c 126 fTerminateFiles(),
bb885a9e 127 fMergeExcludes(),
37d21c01 128 fRegisterExcludes(),
f965131e 129 fIncludePath(),
bb885a9e 130 fCloseSE(),
0df6ccf2 131 fFriendChainName(),
c6cb3634 132 fJobTag(),
648174cf 133 fOutputSingle(),
5fce53f4 134 fRunPrefix(),
3bdcb562 135 fProofCluster(),
136 fProofDataSet(),
137 fFileForTestMode(),
3bdcb562 138 fAliRootMode(),
d88e4159 139 fProofProcessOpt(),
f47d5cb4 140 fMergeDirName(),
4e5c5506 141 fInputFiles(0),
28ce6c9c 142 fPackages(0),
77f34eae 143 fModules(0),
d88e4159 144 fProofParam(),
a41b2882 145 fDropToShell(true),
715da79c 146 fMCLoop(false),
a41b2882 147 fGridJobIDs(""),
148 fGridStages(""),
780ec36d 149 fFriendLibs(""),
150 fTreeName()
c57f56b7 151{
152// Dummy ctor.
153 SetDefaults();
154}
155
156//______________________________________________________________________________
157AliAnalysisAlien::AliAnalysisAlien(const char *name)
158 :AliAnalysisGrid(name),
159 fGridJDL(NULL),
0f389141 160 fMergingJDL(NULL),
c57f56b7 161 fPrice(0),
162 fTTL(0),
163 fSplitMaxInputFileNumber(0),
164 fMaxInitFailed(0),
165 fMasterResubmitThreshold(0),
bb885a9e 166 fNtestFiles(0),
319593fb 167 fNrunsPerMaster(0),
16a4353c 168 fMaxMergeFiles(0),
b3e07543 169 fMaxMergeStages(0),
d2a409b2 170 fNsubmitted(0),
a3e84053 171 fProductionMode(0),
cd11251e 172 fOutputToRunNo(0),
0f389141 173 fMergeViaJDL(0),
174 fFastReadOption(0),
e1c22e21 175 fOverwriteMode(1),
149d288c 176 fNreplicas(2),
3bdcb562 177 fNproofWorkers(0),
27734f0e 178 fNproofWorkersPerSlave(0),
3bdcb562 179 fProofReset(0),
715da79c 180 fNMCevents(0),
181 fNMCjobs(0),
c57f56b7 182 fRunNumbers(),
183 fExecutable(),
0a1c1f7f 184 fExecutableCommand(),
c57f56b7 185 fArguments(),
631c0b05 186 fExecutableArgs(),
c57f56b7 187 fAnalysisMacro(),
188 fAnalysisSource(),
d3b18c4c 189 fValidationScript(),
d5c6455a 190 fAdditionalRootLibs(),
c57f56b7 191 fAdditionalLibs(),
715da79c 192 fGeneratorLibs(),
c57f56b7 193 fSplitMode(),
194 fAPIVersion(),
195 fROOTVersion(),
196 fAliROOTVersion(),
648174cf 197 fExternalPackages(),
c57f56b7 198 fUser(),
199 fGridWorkingDir(),
200 fGridDataDir(),
201 fDataPattern(),
202 fGridOutputDir(),
203 fOutputArchive(),
204 fOutputFiles(),
205 fInputFormat(),
e7c71df0 206 fDatasetName(),
c57f56b7 207 fJDLName(),
d3b18c4c 208 fTerminateFiles(),
bb885a9e 209 fMergeExcludes(),
37d21c01 210 fRegisterExcludes(),
f965131e 211 fIncludePath(),
bb885a9e 212 fCloseSE(),
0df6ccf2 213 fFriendChainName(),
c6cb3634 214 fJobTag(),
648174cf 215 fOutputSingle(),
5fce53f4 216 fRunPrefix(),
3bdcb562 217 fProofCluster(),
218 fProofDataSet(),
219 fFileForTestMode(),
3bdcb562 220 fAliRootMode(),
d88e4159 221 fProofProcessOpt(),
f47d5cb4 222 fMergeDirName(),
4e5c5506 223 fInputFiles(0),
28ce6c9c 224 fPackages(0),
77f34eae 225 fModules(0),
d88e4159 226 fProofParam(),
0ad3ea1f 227 fDropToShell(true),
715da79c 228 fMCLoop(false),
0ad3ea1f 229 fGridJobIDs(""),
a41b2882 230 fGridStages(""),
780ec36d 231 fFriendLibs(""),
232 fTreeName()
c57f56b7 233{
234// Default ctor.
235 SetDefaults();
236}
237
238//______________________________________________________________________________
239AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
240 :AliAnalysisGrid(other),
241 fGridJDL(NULL),
0f389141 242 fMergingJDL(NULL),
c57f56b7 243 fPrice(other.fPrice),
244 fTTL(other.fTTL),
245 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
246 fMaxInitFailed(other.fMaxInitFailed),
247 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 248 fNtestFiles(other.fNtestFiles),
319593fb 249 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 250 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 251 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 252 fNsubmitted(other.fNsubmitted),
a3e84053 253 fProductionMode(other.fProductionMode),
cd11251e 254 fOutputToRunNo(other.fOutputToRunNo),
0f389141 255 fMergeViaJDL(other.fMergeViaJDL),
256 fFastReadOption(other.fFastReadOption),
257 fOverwriteMode(other.fOverwriteMode),
149d288c 258 fNreplicas(other.fNreplicas),
3bdcb562 259 fNproofWorkers(other.fNproofWorkers),
27734f0e 260 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 261 fProofReset(other.fProofReset),
715da79c 262 fNMCevents(other.fNMCevents),
263 fNMCjobs(other.fNMCjobs),
c57f56b7 264 fRunNumbers(other.fRunNumbers),
265 fExecutable(other.fExecutable),
0a1c1f7f 266 fExecutableCommand(other.fExecutableCommand),
c57f56b7 267 fArguments(other.fArguments),
631c0b05 268 fExecutableArgs(other.fExecutableArgs),
c57f56b7 269 fAnalysisMacro(other.fAnalysisMacro),
270 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 271 fValidationScript(other.fValidationScript),
d5c6455a 272 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 273 fAdditionalLibs(other.fAdditionalLibs),
715da79c 274 fGeneratorLibs(other.fGeneratorLibs),
c57f56b7 275 fSplitMode(other.fSplitMode),
276 fAPIVersion(other.fAPIVersion),
277 fROOTVersion(other.fROOTVersion),
278 fAliROOTVersion(other.fAliROOTVersion),
648174cf 279 fExternalPackages(other.fExternalPackages),
c57f56b7 280 fUser(other.fUser),
281 fGridWorkingDir(other.fGridWorkingDir),
282 fGridDataDir(other.fGridDataDir),
283 fDataPattern(other.fDataPattern),
284 fGridOutputDir(other.fGridOutputDir),
285 fOutputArchive(other.fOutputArchive),
286 fOutputFiles(other.fOutputFiles),
287 fInputFormat(other.fInputFormat),
e7c71df0 288 fDatasetName(other.fDatasetName),
c57f56b7 289 fJDLName(other.fJDLName),
d3b18c4c 290 fTerminateFiles(other.fTerminateFiles),
bb885a9e 291 fMergeExcludes(other.fMergeExcludes),
37d21c01 292 fRegisterExcludes(other.fRegisterExcludes),
f965131e 293 fIncludePath(other.fIncludePath),
bb885a9e 294 fCloseSE(other.fCloseSE),
0df6ccf2 295 fFriendChainName(other.fFriendChainName),
c6cb3634 296 fJobTag(other.fJobTag),
648174cf 297 fOutputSingle(other.fOutputSingle),
5fce53f4 298 fRunPrefix(other.fRunPrefix),
3bdcb562 299 fProofCluster(other.fProofCluster),
300 fProofDataSet(other.fProofDataSet),
301 fFileForTestMode(other.fFileForTestMode),
3bdcb562 302 fAliRootMode(other.fAliRootMode),
d88e4159 303 fProofProcessOpt(other.fProofProcessOpt),
f47d5cb4 304 fMergeDirName(other.fMergeDirName),
4e5c5506 305 fInputFiles(0),
28ce6c9c 306 fPackages(0),
77f34eae 307 fModules(0),
d88e4159 308 fProofParam(),
0ad3ea1f 309 fDropToShell(other.fDropToShell),
715da79c 310 fMCLoop(other.fMCLoop),
0ad3ea1f 311 fGridJobIDs(other.fGridJobIDs),
a41b2882 312 fGridStages(other.fGridStages),
780ec36d 313 fFriendLibs(other.fFriendLibs),
314 fTreeName(other.fTreeName)
c57f56b7 315{
316// Copy ctor.
317 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 318 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 319 fRunRange[0] = other.fRunRange[0];
320 fRunRange[1] = other.fRunRange[1];
c57f56b7 321 if (other.fInputFiles) {
322 fInputFiles = new TObjArray();
323 TIter next(other.fInputFiles);
324 TObject *obj;
325 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
326 fInputFiles->SetOwner();
327 }
4e5c5506 328 if (other.fPackages) {
329 fPackages = new TObjArray();
330 TIter next(other.fPackages);
331 TObject *obj;
332 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
333 fPackages->SetOwner();
334 }
77f34eae 335 if (other.fModules) {
336 fModules = new TObjArray();
337 fModules->SetOwner();
338 TIter next(other.fModules);
339 AliAnalysisTaskCfg *mod, *crt;
340 while ((crt=(AliAnalysisTaskCfg*)next())) {
341 mod = new AliAnalysisTaskCfg(*crt);
342 fModules->Add(mod);
343 }
344 }
c57f56b7 345}
346
347//______________________________________________________________________________
348AliAnalysisAlien::~AliAnalysisAlien()
349{
350// Destructor.
77f34eae 351 delete fGridJDL;
352 delete fMergingJDL;
353 delete fInputFiles;
354 delete fPackages;
355 delete fModules;
28ce6c9c 356 fProofParam.DeleteAll();
c57f56b7 357}
358
359//______________________________________________________________________________
360AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
361{
362// Assignment.
363 if (this != &other) {
364 AliAnalysisGrid::operator=(other);
365 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 366 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 367 fPrice = other.fPrice;
368 fTTL = other.fTTL;
369 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
370 fMaxInitFailed = other.fMaxInitFailed;
371 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 372 fNtestFiles = other.fNtestFiles;
a3e84053 373 fNrunsPerMaster = other.fNrunsPerMaster;
374 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 375 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 376 fNsubmitted = other.fNsubmitted;
377 fProductionMode = other.fProductionMode;
cd11251e 378 fOutputToRunNo = other.fOutputToRunNo;
0f389141 379 fMergeViaJDL = other.fMergeViaJDL;
380 fFastReadOption = other.fFastReadOption;
381 fOverwriteMode = other.fOverwriteMode;
149d288c 382 fNreplicas = other.fNreplicas;
3bdcb562 383 fNproofWorkers = other.fNproofWorkers;
27734f0e 384 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 385 fProofReset = other.fProofReset;
715da79c 386 fNMCevents = other.fNMCevents;
387 fNMCjobs = other.fNMCjobs;
c57f56b7 388 fRunNumbers = other.fRunNumbers;
389 fExecutable = other.fExecutable;
0a1c1f7f 390 fExecutableCommand = other.fExecutableCommand;
c57f56b7 391 fArguments = other.fArguments;
631c0b05 392 fExecutableArgs = other.fExecutableArgs;
c57f56b7 393 fAnalysisMacro = other.fAnalysisMacro;
394 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 395 fValidationScript = other.fValidationScript;
d5c6455a 396 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 397 fAdditionalLibs = other.fAdditionalLibs;
715da79c 398 fGeneratorLibs = other.fGeneratorLibs;
c57f56b7 399 fSplitMode = other.fSplitMode;
400 fAPIVersion = other.fAPIVersion;
401 fROOTVersion = other.fROOTVersion;
402 fAliROOTVersion = other.fAliROOTVersion;
648174cf 403 fExternalPackages = other.fExternalPackages;
c57f56b7 404 fUser = other.fUser;
405 fGridWorkingDir = other.fGridWorkingDir;
406 fGridDataDir = other.fGridDataDir;
407 fDataPattern = other.fDataPattern;
408 fGridOutputDir = other.fGridOutputDir;
409 fOutputArchive = other.fOutputArchive;
410 fOutputFiles = other.fOutputFiles;
411 fInputFormat = other.fInputFormat;
e7c71df0 412 fDatasetName = other.fDatasetName;
c57f56b7 413 fJDLName = other.fJDLName;
d3b18c4c 414 fTerminateFiles = other.fTerminateFiles;
bb885a9e 415 fMergeExcludes = other.fMergeExcludes;
37d21c01 416 fRegisterExcludes = other.fRegisterExcludes;
f965131e 417 fIncludePath = other.fIncludePath;
bb885a9e 418 fCloseSE = other.fCloseSE;
0df6ccf2 419 fFriendChainName = other.fFriendChainName;
c6cb3634 420 fJobTag = other.fJobTag;
648174cf 421 fOutputSingle = other.fOutputSingle;
5fce53f4 422 fRunPrefix = other.fRunPrefix;
3bdcb562 423 fProofCluster = other.fProofCluster;
424 fProofDataSet = other.fProofDataSet;
425 fFileForTestMode = other.fFileForTestMode;
3bdcb562 426 fAliRootMode = other.fAliRootMode;
d88e4159 427 fProofProcessOpt = other.fProofProcessOpt;
f47d5cb4 428 fMergeDirName = other.fMergeDirName;
d88e4159 429 fDropToShell = other.fDropToShell;
715da79c 430 fMCLoop = other.fMCLoop;
0ad3ea1f 431 fGridJobIDs = other.fGridJobIDs;
432 fGridStages = other.fGridStages;
a41b2882 433 fFriendLibs = other.fFriendLibs;
780ec36d 434 fTreeName = other.fTreeName;
c57f56b7 435 if (other.fInputFiles) {
436 fInputFiles = new TObjArray();
437 TIter next(other.fInputFiles);
438 TObject *obj;
439 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
440 fInputFiles->SetOwner();
441 }
4e5c5506 442 if (other.fPackages) {
443 fPackages = new TObjArray();
444 TIter next(other.fPackages);
445 TObject *obj;
446 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
447 fPackages->SetOwner();
448 }
77f34eae 449 if (other.fModules) {
450 fModules = new TObjArray();
451 fModules->SetOwner();
452 TIter next(other.fModules);
453 AliAnalysisTaskCfg *mod, *crt;
454 while ((crt=(AliAnalysisTaskCfg*)next())) {
455 mod = new AliAnalysisTaskCfg(*crt);
456 fModules->Add(mod);
457 }
458 }
c57f56b7 459 }
460 return *this;
461}
462
37d21c01 463//______________________________________________________________________________
464void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
465{
466// Add a single additional library to be loaded. Extension must be present.
467 TString lib(name);
468 if (!lib.Contains(".")) {
469 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
470 return;
471 }
472 if (fAdditionalLibs.Contains(name)) {
473 Warning("AddAdditionalLibrary", "Library %s already added.", name);
474 return;
475 }
476 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
477 fAdditionalLibs += lib;
478}
479
77f34eae 480//______________________________________________________________________________
481void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
482{
483// Adding a module. Checks if already existing. Becomes owned by this.
484 if (!module) return;
485 if (GetModule(module->GetName())) {
486 Error("AddModule", "A module having the same name %s already added", module->GetName());
487 return;
488 }
489 if (!fModules) {
490 fModules = new TObjArray();
491 fModules->SetOwner();
492 }
493 fModules->Add(module);
494}
495
496//______________________________________________________________________________
497void AliAnalysisAlien::AddModules(TObjArray *list)
498{
499// Adding a list of modules. Checks if already existing. Becomes owned by this.
500 TIter next(list);
501 AliAnalysisTaskCfg *module;
502 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
503}
504
505//______________________________________________________________________________
506Bool_t AliAnalysisAlien::CheckDependencies()
507{
508// Check if all dependencies are satisfied. Reorder modules if needed.
509 Int_t nmodules = GetNmodules();
510 if (!nmodules) {
511 Warning("CheckDependencies", "No modules added yet to check their dependencies");
512 return kTRUE;
513 }
514 AliAnalysisTaskCfg *mod = 0;
515 AliAnalysisTaskCfg *dep = 0;
516 TString depname;
517 Int_t i, j, k;
518 for (i=0; i<nmodules; i++) {
519 mod = (AliAnalysisTaskCfg*) fModules->At(i);
520 Int_t ndeps = mod->GetNdeps();
521 Int_t istart = i;
522 for (j=0; j<ndeps; j++) {
523 depname = mod->GetDependency(j);
524 dep = GetModule(depname);
525 if (!dep) {
526 Error("CheckDependencies","Dependency %s not added for module %s",
527 depname.Data(), mod->GetName());
528 return kFALSE;
529 }
530 if (dep->NeedsDependency(mod->GetName())) {
531 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
532 mod->GetName(), dep->GetName());
533 return kFALSE;
534 }
535 Int_t idep = fModules->IndexOf(dep);
536 // The dependency task must come first
537 if (idep>i) {
538 // Remove at idep and move all objects below up one slot
539 // down to index i included.
540 fModules->RemoveAt(idep);
3e40fd4c 541 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 542 fModules->AddAt(dep, i++);
543 }
544 //Redo from istart if dependencies were inserted
545 if (i>istart) i=istart-1;
546 }
547 }
548 return kTRUE;
549}
550
3e40fd4c 551//______________________________________________________________________________
552AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
553{
554// Create the analysis manager and optionally execute the macro in filename.
555 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
715da79c 556 if (mgr) {
557 mgr->SetMCLoop(fMCLoop);
558 return mgr;
559 }
3e40fd4c 560 mgr = new AliAnalysisManager(name);
561 mgr->SetGridHandler((AliAnalysisGrid*)this);
715da79c 562 mgr->SetMCLoop(fMCLoop);
3e40fd4c 563 if (strlen(filename)) {
564 TString line = gSystem->ExpandPathName(filename);
565 line.Prepend(".x ");
566 gROOT->ProcessLine(line.Data());
567 }
568 return mgr;
569}
570
77f34eae 571//______________________________________________________________________________
572Int_t AliAnalysisAlien::GetNmodules() const
573{
574// Get number of modules.
575 if (!fModules) return 0;
576 return fModules->GetEntries();
577}
578
579//______________________________________________________________________________
580AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
581{
582// Get a module by name.
583 if (!fModules) return 0;
584 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
585}
586
587//______________________________________________________________________________
588Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
589{
590// Load a given module.
591 if (mod->IsLoaded()) return kTRUE;
192ddca2 592 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
593 if (!mgr) {
594 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
595 return kFALSE;
596 }
77f34eae 597 Int_t ndeps = mod->GetNdeps();
598 TString depname;
599 for (Int_t j=0; j<ndeps; j++) {
600 depname = mod->GetDependency(j);
601 AliAnalysisTaskCfg *dep = GetModule(depname);
602 if (!dep) {
603 Error("LoadModule","Dependency %s not existing for module %s",
604 depname.Data(), mod->GetName());
605 return kFALSE;
606 }
607 if (!LoadModule(dep)) {
608 Error("LoadModule","Dependency %s for module %s could not be loaded",
609 depname.Data(), mod->GetName());
610 return kFALSE;
611 }
612 }
613 // Load libraries for the module
614 if (!mod->CheckLoadLibraries()) {
615 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
616 return kFALSE;
617 }
192ddca2 618 // Check if a custom file name was requested
619 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
620
621 // Check if a custom terminate file name was requested
622 if (strlen(mod->GetTerminateFileName())) {
623 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
624 fTerminateFiles += mod->GetTerminateFileName();
625 }
626
77f34eae 627 // Execute the macro
628 if (mod->ExecuteMacro()<0) {
629 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
630 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
631 return kFALSE;
632 }
633 // Configure dependencies
634 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
635 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
636 mod->GetConfigMacro()->GetTitle(), mod->GetName());
637 return kFALSE;
638 }
5e8c7c22 639 // Adjust extra libraries
640 Int_t nlibs = mod->GetNlibs();
641 TString lib;
642 for (Int_t i=0; i<nlibs; i++) {
643 lib = mod->GetLibrary(i);
5e8c7c22 644 lib = Form("lib%s.so", lib.Data());
c9d6bd12 645 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 646 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
647 fAdditionalLibs += lib;
648 }
77f34eae 649 return kTRUE;
650}
651
139fe6ba 652//______________________________________________________________________________
653Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
654{
655// Generate the full train.
656 fAdditionalLibs = "";
657 if (!LoadModules()) return kFALSE;
658 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
659 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 660 mgr->RunLocalInit();
139fe6ba 661 mgr->PrintStatus();
662 Int_t productionMode = fProductionMode;
663 SetProductionMode();
664 TString macro = fAnalysisMacro;
665 TString executable = fExecutable;
666 TString validation = fValidationScript;
667 TString execCommand = fExecutableCommand;
668 SetAnalysisMacro(Form("%s.C", name));
669 SetExecutable(Form("%s.sh", name));
a8e31fb0 670// SetExecutableCommand("aliroot -b -q ");
139fe6ba 671 SetValidationScript(Form("%s_validation.sh", name));
672 StartAnalysis();
673 SetProductionMode(productionMode);
674 fAnalysisMacro = macro;
675 fExecutable = executable;
676 fExecutableCommand = execCommand;
677 fValidationScript = validation;
678 return kTRUE;
679}
680
77f34eae 681//______________________________________________________________________________
21ca8e59 682Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 683{
684// Generate test macros for a single module or for the full train.
5e8c7c22 685 fAdditionalLibs = "";
77f34eae 686 if (strlen(modname)) {
21ca8e59 687 if (!CheckDependencies()) return kFALSE;
688 AliAnalysisTaskCfg *mod = GetModule(modname);
689 if (!mod) {
690 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
691 return kFALSE;
692 }
693 if (!LoadModule(mod)) return kFALSE;
a41b2882 694 if (!LoadFriendLibs()) return kFALSE;
21ca8e59 695 } else if (!LoadModules()) return kFALSE;
696 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
697 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 698 mgr->RunLocalInit();
21ca8e59 699 mgr->PrintStatus();
700 SetLocalTest(kTRUE);
701 Int_t productionMode = fProductionMode;
702 SetProductionMode();
703 TString macro = fAnalysisMacro;
704 TString executable = fExecutable;
705 TString validation = fValidationScript;
706 TString execCommand = fExecutableCommand;
707 SetAnalysisMacro(Form("%s.C", name));
708 SetExecutable(Form("%s.sh", name));
d66aa2a9 709 fOutputFiles = GetListOfFiles("outaod");
710 // Add extra files registered to the analysis manager
711 TString extra = GetListOfFiles("ext");
712 if (!extra.IsNull()) {
713 extra.ReplaceAll(".root", "*.root");
714 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
715 fOutputFiles += extra;
716 }
a8e31fb0 717// SetExecutableCommand("aliroot -b -q ");
21ca8e59 718 SetValidationScript(Form("%s_validation.sh", name));
719 WriteAnalysisFile();
720 WriteAnalysisMacro();
721 WriteExecutable();
722 WriteValidationScript();
ffbe06b9 723 WriteMergingMacro();
724 WriteMergeExecutable();
725 WriteValidationScript(kTRUE);
21ca8e59 726 SetLocalTest(kFALSE);
727 SetProductionMode(productionMode);
728 fAnalysisMacro = macro;
729 fExecutable = executable;
730 fExecutableCommand = execCommand;
731 fValidationScript = validation;
77f34eae 732 return kTRUE;
733}
734
735//______________________________________________________________________________
736Bool_t AliAnalysisAlien::LoadModules()
737{
738// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 739 fAdditionalLibs = "";
21ca8e59 740 Int_t nmodules = GetNmodules();
741 if (!nmodules) {
742 Warning("LoadModules", "No module to be loaded");
743 return kTRUE;
744 }
3e40fd4c 745 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
746 if (!mgr) {
747 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
748 return kFALSE;
749 }
77f34eae 750 if (!CheckDependencies()) return kFALSE;
21ca8e59 751 nmodules = GetNmodules();
77f34eae 752 AliAnalysisTaskCfg *mod;
21ca8e59 753 for (Int_t imod=0; imod<nmodules; imod++) {
754 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 755 if (!LoadModule(mod)) return kFALSE;
756 }
a41b2882 757 // Load additional friend libraries
758 return LoadFriendLibs();
77f34eae 759}
760
a41b2882 761//______________________________________________________________________________
762Bool_t AliAnalysisAlien::LoadFriendLibs() const
763{
764// Load libraries required for reading friends.
765 if (fFriendLibs.Length()) {
766 TObjArray *list = 0;
767 TString lib;
768 if (fFriendLibs.Contains(",")) list = fFriendLibs.Tokenize(",");
769 else list = fFriendLibs.Tokenize(" ");
770 for (Int_t ilib=0; ilib<list->GetEntriesFast(); ilib++) {
771 lib = list->At(ilib)->GetName();
772 lib.ReplaceAll(".so","");
773 lib.ReplaceAll(" ","");
774 if (lib.BeginsWith("lib")) lib.Remove(0, 3);
775 lib.Prepend("lib");
776 Int_t loaded = strlen(gSystem->GetLibraries(lib,"",kFALSE));
777 if (!loaded) loaded = gSystem->Load(lib);
778 if (loaded < 0) {
779 Error("LoadModules", "Cannot load library for friends %s", lib.Data());
780 return kFALSE;
781 }
782 }
783 delete list;
784 }
785 return kTRUE;
786}
787
74013535 788//______________________________________________________________________________
789void AliAnalysisAlien::SetRunPrefix(const char *prefix)
790{
791// Set the run number format. Can be a prefix or a format like "%09d"
792 fRunPrefix = prefix;
793 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
794}
795
f965131e 796//______________________________________________________________________________
797void AliAnalysisAlien::AddIncludePath(const char *path)
798{
799// Add include path in the remote analysis macro.
800 TString p(path);
801 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
802 else fIncludePath += Form("-I%s ", path);
803}
804
c57f56b7 805//______________________________________________________________________________
806void AliAnalysisAlien::AddRunNumber(Int_t run)
807{
808// Add a run number to the list of runs to be processed.
809 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 810 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 811}
812
98690895 813//______________________________________________________________________________
814void AliAnalysisAlien::AddRunList(const char* runList)
815{
816// Add several runs into the list of runs; they are expected to be separated by a blank character.
817 TString sList = runList;
818 TObjArray *list = sList.Tokenize(" ");
819 Int_t n = list->GetEntries();
820 for (Int_t i = 0; i < n; i++) {
821 TObjString *os = (TObjString*)list->At(i);
822 AddRunNumber(os->GetString().Atoi());
823 }
824 delete list;
825}
826
ee75cfc3 827//______________________________________________________________________________
828void AliAnalysisAlien::AddRunNumber(const char* run)
829{
830// Add a run number to the list of runs to be processed.
7aa5d539 831 TString runs = run;
832 TObjString *os;
833 TObjArray *arr = runs.Tokenize(" ");
834 TIter next(arr);
835 TString prefix;
836 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
837 while ((os=(TObjString*)next())){
838 if (fRunNumbers.Length()) fRunNumbers += " ";
839 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
840 }
841 delete arr;
ee75cfc3 842}
843
c57f56b7 844//______________________________________________________________________________
845void AliAnalysisAlien::AddDataFile(const char *lfn)
846{
847// Adds a data file to the input to be analysed. The file should be a valid LFN
848// or point to an existing file in the alien workdir.
849 if (!fInputFiles) fInputFiles = new TObjArray();
850 fInputFiles->Add(new TObjString(lfn));
851}
648174cf 852
853//______________________________________________________________________________
854void AliAnalysisAlien::AddExternalPackage(const char *package)
855{
856// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
857 if (fExternalPackages) fExternalPackages += " ";
858 fExternalPackages += package;
859}
860
c57f56b7 861//______________________________________________________________________________
862Bool_t AliAnalysisAlien::Connect()
863{
864// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
865 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 866 if (fProductionMode) return kTRUE;
c57f56b7 867 if (!gGrid) {
868 Info("Connect", "Trying to connect to AliEn ...");
869 TGrid::Connect("alien://");
870 }
871 if (!gGrid || !gGrid->IsConnected()) {
872 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
873 return kFALSE;
874 }
875 fUser = gGrid->GetUser();
876 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
877 return kTRUE;
878}
879
880//______________________________________________________________________________
881void AliAnalysisAlien::CdWork()
882{
883// Check validity of alien workspace. Create directory if possible.
884 if (!Connect()) {
885 Error("CdWork", "Alien connection required");
886 return;
887 }
888 TString homedir = gGrid->GetHomeDirectory();
889 TString workdir = homedir + fGridWorkingDir;
923e2ca5 890 if (DirectoryExists(workdir)) {
891 gGrid->Cd(workdir);
892 return;
893 }
894 // Work directory not existing - create it
895 gGrid->Cd(homedir);
b93f8109 896 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 897 gGrid->Cd(fGridWorkingDir);
d3b18c4c 898 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 899 } else {
d3b18c4c 900 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 901 workdir.Data(), homedir.Data());
902 fGridWorkingDir = "";
903 }
c57f56b7 904}
905
348be253 906//______________________________________________________________________________
907Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
908{
909// Check if file copying is possible.
d3b18c4c 910 if (fProductionMode) return kTRUE;
b5ae4fe0 911 TString salienpath(alienpath);
912 if (salienpath.Contains(" ")) {
913 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
914 return kFALSE;
915 }
348be253 916 if (!Connect()) {
917 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
918 return kFALSE;
919 }
d3339be3 920 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
921 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 922 // Check if alien_CLOSE_SE is defined
923 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
924 if (!closeSE.IsNull()) {
925 Info("CheckFileCopy", "Your current close storage is pointing to: \
926 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
927 } else {
928 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
929 }
930 // Check if grid directory exists.
931 if (!DirectoryExists(alienpath)) {
932 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
933 return kFALSE;
934 }
b5ae4fe0 935 TString stest = "plugin_test_copy";
936 TFile f(stest, "RECREATE");
348be253 937 // User may not have write permissions to current directory
938 if (f.IsZombie()) {
939 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
940 gSystem->WorkingDirectory());
941 return kFALSE;
942 }
943 f.Close();
b5ae4fe0 944 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
945 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 946 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 947 \n# 1. Make sure you have write permissions there. If this is the case: \
948 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
949 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
950 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
951 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 952 gSystem->Unlink(stest.Data());
348be253 953 return kFALSE;
954 }
b5ae4fe0 955 gSystem->Unlink(stest.Data());
956 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 957 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 958 return kTRUE;
959}
960
c57f56b7 961//______________________________________________________________________________
962Bool_t AliAnalysisAlien::CheckInputData()
963{
964// Check validity of input data. If necessary, create xml files.
d3b18c4c 965 if (fProductionMode) return kTRUE;
a8739e8a 966 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 967 if (!fGridDataDir.Length()) {
968 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
969 return kFALSE;
970 }
9f020952 971 if (fMergeViaJDL) {
972 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
973 return kFALSE;
974 }
d2a409b2 975 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 976 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
977 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 978 return kTRUE;
c57f56b7 979 }
980 // Process declared files
a2f5fc01 981 Bool_t isCollection = kFALSE;
982 Bool_t isXml = kFALSE;
983 Bool_t useTags = kFALSE;
c57f56b7 984 Bool_t checked = kFALSE;
d3b18c4c 985 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 986 TString file;
987 TString workdir = gGrid->GetHomeDirectory();
988 workdir += fGridWorkingDir;
989 if (fInputFiles) {
990 TObjString *objstr;
991 TIter next(fInputFiles);
992 while ((objstr=(TObjString*)next())) {
993 file = workdir;
994 file += "/";
995 file += objstr->GetString();
996 // Store full lfn path
997 if (FileExists(file)) objstr->SetString(file);
998 else {
999 file = objstr->GetName();
1000 if (!FileExists(objstr->GetName())) {
1001 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
1002 objstr->GetName(), workdir.Data());
1003 return kFALSE;
1004 }
1005 }
1006 Bool_t iscoll, isxml, usetags;
1007 CheckDataType(file, iscoll, isxml, usetags);
1008 if (!checked) {
1009 checked = kTRUE;
a2f5fc01 1010 isCollection = iscoll;
1011 isXml = isxml;
1012 useTags = usetags;
1013 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1014 } else {
a2f5fc01 1015 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 1016 Error("CheckInputData", "Some conflict was found in the types of inputs");
1017 return kFALSE;
1018 }
1019 }
1020 }
1021 }
1022 // Process requested run numbers
a8739e8a 1023 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 1024 // Check validity of alien data directory
1025 if (!fGridDataDir.Length()) {
1026 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
1027 return kFALSE;
1028 }
923e2ca5 1029 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 1030 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
1031 return kFALSE;
1032 }
a2f5fc01 1033 if (isCollection) {
c57f56b7 1034 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
1035 return kFALSE;
1036 }
1037
a2f5fc01 1038 if (checked && !isXml) {
c57f56b7 1039 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
1040 return kFALSE;
1041 }
1042 // Check validity of run number(s)
1043 TObjArray *arr;
1044 TObjString *os;
74013535 1045 TString format;
319593fb 1046 Int_t nruns = 0;
904f9f5f 1047 TString schunk, schunk2;
c57f56b7 1048 TString path;
1049 if (!checked) {
1050 checked = kTRUE;
a2f5fc01 1051 useTags = fDataPattern.Contains("tag");
1052 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1053 }
a2f5fc01 1054 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 1055 Error("CheckInputData", "Cannot mix input files using/not using tags");
1056 return kFALSE;
1057 }
1058 if (fRunNumbers.Length()) {
a8739e8a 1059 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 1060 arr = fRunNumbers.Tokenize(" ");
1061 TIter next(arr);
1062 while ((os=(TObjString*)next())) {
1063 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1064 if (!DirectoryExists(path)) {
1065 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 1066 continue;
c57f56b7 1067 }
1068 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
1069 TString msg = "\n##### file: ";
1070 msg += path;
1071 msg += " type: xml_collection;";
a2f5fc01 1072 if (useTags) msg += " using_tags: Yes";
c57f56b7 1073 else msg += " using_tags: No";
23329835 1074 Info("CheckDataType", "%s", msg.Data());
319593fb 1075 if (fNrunsPerMaster<2) {
d2a409b2 1076 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 1077 } else {
1078 nruns++;
1079 if (((nruns-1)%fNrunsPerMaster) == 0) {
1080 schunk = os->GetString();
1081 }
1082 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1083 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1084 AddDataFile(schunk);
319593fb 1085 }
c57f56b7 1086 }
1087 delete arr;
a8739e8a 1088 } else {
1089 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1090 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1091 format = Form("%%s/%s ", fRunPrefix.Data());
1092 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1093 if (!DirectoryExists(path)) {
a8739e8a 1094 continue;
1095 }
74013535 1096 format = Form("%%s/%s.xml", fRunPrefix.Data());
1097 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1098 TString msg = "\n##### file: ";
1099 msg += path;
1100 msg += " type: xml_collection;";
a2f5fc01 1101 if (useTags) msg += " using_tags: Yes";
a8739e8a 1102 else msg += " using_tags: No";
23329835 1103 Info("CheckDataType", "%s", msg.Data());
319593fb 1104 if (fNrunsPerMaster<2) {
74013535 1105 format = Form("%s.xml", fRunPrefix.Data());
1106 AddDataFile(Form(format.Data(),irun));
319593fb 1107 } else {
1108 nruns++;
1109 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1110 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1111 }
74013535 1112 format = Form("_%s.xml", fRunPrefix.Data());
1113 schunk2 = Form(format.Data(), irun);
319593fb 1114 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1115 schunk += schunk2;
d2a409b2 1116 AddDataFile(schunk);
319593fb 1117 }
a8739e8a 1118 }
904f9f5f 1119 if (!fInputFiles) {
1120 schunk += schunk2;
1121 AddDataFile(schunk);
1122 }
c57f56b7 1123 }
1124 return kTRUE;
1125}
1126
21ca8e59 1127//______________________________________________________________________________
4a8b667c 1128Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1129{
1130// Copy data from the given grid directory according a pattern and make a local
1131// dataset.
2ba4c287 1132// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1133 if (!Connect()) {
1134 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
4a8b667c 1135 return 0;
21ca8e59 1136 }
1137 if (!DirectoryExists(griddir)) {
1138 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
4a8b667c 1139 return 0;
21ca8e59 1140 }
1141 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1142 printf("Running command: %s\n", command.Data());
1143 TGridResult *res = gGrid->Command(command);
1144 Int_t nfound = res->GetEntries();
1145 if (!nfound) {
1146 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
4a8b667c 1147 return 0;
21ca8e59 1148 }
1149 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1150
1151 // archives
1152 TObjArray* additionalArchives = 0;
1153 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1154 additionalArchives = TString(archivefile).Tokenize(";");
1155 archivefile = additionalArchives->At(0)->GetName();
1156 additionalArchives->RemoveAt(0);
1157 additionalArchives->Compress();
1158 }
1159
21ca8e59 1160 // Copy files locally
1161 ofstream out;
1162 out.open(output, ios::out);
1163 TMap *map;
1164 TString turl, dirname, filename, temp;
1165 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1166 gSystem->MakeDirectory(outputdir);
1167 gSystem->ChangeDirectory(outputdir);
4a8b667c 1168 Int_t ncopied = 0;
21ca8e59 1169 for (Int_t i=0; i<nfound; i++) {
1170 map = (TMap*)res->At(i);
1171 turl = map->GetValue("turl")->GetName();
1172 filename = gSystem->BaseName(turl.Data());
1173 dirname = gSystem->DirName(turl.Data());
1174 dirname = gSystem->BaseName(dirname.Data());
1175 gSystem->MakeDirectory(dirname);
43b807f6 1176
1177 TString source(turl);
1178 TString targetFileName(filename);
1179
1180 if (strlen(archivefile) > 0) {
4a8b667c 1181// TODO here the archive in which the file resides should be determined
1182// however whereis returns only a guid, and guid2lfn does not work
1183// Therefore we use the one provided as argument for now
1184 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1185 targetFileName = archivefile;
43b807f6 1186 }
1187 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
4a8b667c 1188 Bool_t success = kTRUE;
1189 if (additionalArchives) {
1190 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1191 TString target;
1192 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1193 gSystem->MakeDirectory(gSystem->DirName(target));
1194 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1195 }
1196 }
2ba4c287 1197
4a8b667c 1198 if (success) {
1199 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1200 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1201 ncopied++;
1202 }
21ca8e59 1203 }
1204 }
1205 gSystem->ChangeDirectory(cdir);
1206 delete res;
4a8b667c 1207 delete additionalArchives;
1208 return ncopied;
21ca8e59 1209}
1210
c57f56b7 1211//______________________________________________________________________________
1212Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1213{
1214// Create dataset for the grid data directory + run number.
3af861ff 1215 const Int_t gMaxEntries = 15000;
d3b18c4c 1216 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1217 if (!Connect()) {
1218 Error("CreateDataset", "Cannot create dataset with no grid connection");
1219 return kFALSE;
1220 }
1221
1222 // Cd workspace
d3b18c4c 1223 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1224 TString workdir = gGrid->GetHomeDirectory();
1225 workdir += fGridWorkingDir;
1226
1227 // Compose the 'find' command arguments
74013535 1228 TString format;
c57f56b7 1229 TString command;
780ec36d 1230 TString delimiter = pattern;
1231 delimiter.Strip();
1232 if (delimiter.Contains(" ")) delimiter = "";
1233 else delimiter = " ";
c57f56b7 1234 TString options = "-x collection ";
bb885a9e 1235 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1236 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1237 TString conditions = "";
3af861ff 1238 Int_t nstart = 0;
1239 Int_t ncount = 0;
1240 Int_t stage = 0;
c57f56b7 1241 TString file;
1242 TString path;
319593fb 1243 Int_t nruns = 0;
904f9f5f 1244 TString schunk, schunk2;
ab254fd1 1245 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1246 if (!fRunNumbers.Length() && !fRunRange[0]) {
1247 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1248 // Make a single data collection from data directory.
1249 path = fGridDataDir;
923e2ca5 1250 if (!DirectoryExists(path)) {
d2a409b2 1251 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1252 return kFALSE;
780ec36d 1253 }
923e2ca5 1254// CdWork();
d2a409b2 1255 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1256 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1257 while (1) {
1258 ncount = 0;
1259 stage++;
1260 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1261 command = "find ";
1262 command += Form("%s -o %d ",options.Data(), nstart);
1263 command += path;
780ec36d 1264 command += delimiter;
3af861ff 1265 command += pattern;
1266 command += conditions;
1267 printf("command: %s\n", command.Data());
1268 TGridResult *res = gGrid->Command(command);
1269 if (res) delete res;
1270 // Write standard output to file
1271 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1272 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1273 Bool_t nullFile = kFALSE;
1274 if (!hasGrep) {
1275 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1276 } else {
1277 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1278 if (nullFile) {
1279 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1280 gSystem->Exec("rm -f __tmp*");
1281 return kFALSE;
1282 }
1283 TString line;
1284 ifstream in;
1285 in.open("__tmp__");
1286 in >> line;
1287 in.close();
1288 gSystem->Exec("rm -f __tmp__");
1289 ncount = line.Atoi();
1290 }
1291 }
1292 if (ncount == gMaxEntries) {
1293 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1294 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1295 if (!cbase) cbase = cadd;
1296 else {
1297 cbase->Add(cadd);
1298 delete cadd;
defd7a3a 1299 }
3af861ff 1300 nstart += ncount;
1301 } else {
1302 if (cbase) {
1303 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1304 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1305 cbase->Add(cadd);
1306 delete cadd;
1307 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1308 delete cbase; cbase = 0;
1309 } else {
1310 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1311 }
1312 gSystem->Exec("rm -f __tmp*");
1313 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1314 break;
1315 }
0f389141 1316 }
1317 Bool_t fileExists = FileExists(file);
1318 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1319 // Copy xml file to alien space
0f389141 1320 if (fileExists) gGrid->Rm(file);
d2a409b2 1321 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1322 if (!FileExists(file)) {
1323 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1324 return kFALSE;
1325 }
1326 // Update list of files to be processed.
1327 }
1328 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1329 return kTRUE;
1330 }
c57f56b7 1331 // Several runs
a2f5fc01 1332 Bool_t nullResult = kTRUE;
a8739e8a 1333 if (fRunNumbers.Length()) {
1334 TObjArray *arr = fRunNumbers.Tokenize(" ");
1335 TObjString *os;
1336 TIter next(arr);
1337 while ((os=(TObjString*)next())) {
3af861ff 1338 nstart = 0;
1339 stage = 0;
0dab9d6d 1340 path = Form("%s/%s", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1341 if (!DirectoryExists(path)) continue;
1342// CdWork();
a8739e8a 1343 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1344 else file = Form("%s.xml", os->GetString().Data());
319593fb 1345 // If local collection file does not exist, create it via 'find' command.
3af861ff 1346 while (1) {
1347 ncount = 0;
1348 stage++;
1349 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1350 command = "find ";
1351 command += Form("%s -o %d ",options.Data(), nstart);
1352 command += path;
780ec36d 1353 command += delimiter;
3af861ff 1354 command += pattern;
1355 command += conditions;
1356 TGridResult *res = gGrid->Command(command);
1357 if (res) delete res;
1358 // Write standard output to file
1359 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1360 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1361 Bool_t nullFile = kFALSE;
1362 if (!hasGrep) {
1363 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1364 } else {
1365 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1366 if (nullFile) {
1367 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1368 gSystem->Exec("rm -f __tmp*");
1369 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1370 break;
3af861ff 1371 }
1372 TString line;
1373 ifstream in;
1374 in.open("__tmp__");
1375 in >> line;
1376 in.close();
1377 gSystem->Exec("rm -f __tmp__");
1378 ncount = line.Atoi();
1379 }
1380 nullResult = kFALSE;
1381 }
1382 if (ncount == gMaxEntries) {
1383 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1384 if (fNrunsPerMaster > 1) {
1385 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1386 file.Data(),gMaxEntries);
1387 return kFALSE;
1388 }
1389 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1390 if (!cbase) cbase = cadd;
1391 else {
1392 cbase->Add(cadd);
1393 delete cadd;
defd7a3a 1394 }
3af861ff 1395 nstart += ncount;
1396 } else {
c89a84bc 1397 if (cbase && fNrunsPerMaster<2) {
3af861ff 1398 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1399 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1400 cbase->Add(cadd);
1401 delete cadd;
1402 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1403 delete cbase; cbase = 0;
1404 } else {
1405 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1406 }
1407 gSystem->Exec("rm -f __tmp*");
1408 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1409 break;
d3339be3 1410 }
3af861ff 1411 }
a8739e8a 1412 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1413 // Check if there is one run per master job.
1414 if (fNrunsPerMaster<2) {
1415 if (FileExists(file)) {
0f389141 1416 if (fOverwriteMode) gGrid->Rm(file);
1417 else {
1418 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1419 continue;
1420 }
319593fb 1421 }
1422 // Copy xml file to alien space
1423 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1424 if (!FileExists(file)) {
1425 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1426 delete arr;
1427 return kFALSE;
1428 }
1429 } else {
1430 nruns++;
1431 if (((nruns-1)%fNrunsPerMaster) == 0) {
1432 schunk = os->GetString();
1433 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1434 } else {
1435 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1436 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1437 cbase->Add(cadd);
1438 delete cadd;
1439 }
1440 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1441 continue;
1442 }
1443 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1444 if (FileExists(schunk)) {
1445 if (fOverwriteMode) gGrid->Rm(file);
1446 else {
1447 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1448 continue;
1449 }
319593fb 1450 }
84fcd93f 1451 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1452 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1453 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1454 if (!FileExists(schunk)) {
1455 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1456 delete arr;
1457 return kFALSE;
1458 }
d3339be3 1459 }
a8739e8a 1460 }
1461 delete arr;
a2f5fc01 1462 if (nullResult) {
d3339be3 1463 Error("CreateDataset", "No valid dataset corresponding to the query!");
1464 return kFALSE;
1465 }
a8739e8a 1466 } else {
1467 // Process a full run range.
1468 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
0dab9d6d 1469 format = Form("%%s/%s", fRunPrefix.Data());
3af861ff 1470 nstart = 0;
1471 stage = 0;
74013535 1472 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1473 if (!DirectoryExists(path)) continue;
1474// CdWork();
74013535 1475 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1476 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1477 else file = Form(format.Data(), irun);
0f389141 1478 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1479 if (fOverwriteMode) gGrid->Rm(file);
1480 else {
1481 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1482 continue;
1483 }
a8739e8a 1484 }
319593fb 1485 // If local collection file does not exist, create it via 'find' command.
3af861ff 1486 while (1) {
1487 ncount = 0;
1488 stage++;
1489 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1490 command = "find ";
1491 command += Form("%s -o %d ",options.Data(), nstart);
1492 command += path;
780ec36d 1493 command += delimiter;
3af861ff 1494 command += pattern;
1495 command += conditions;
1496 TGridResult *res = gGrid->Command(command);
1497 if (res) delete res;
1498 // Write standard output to file
1499 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1500 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1501 Bool_t nullFile = kFALSE;
1502 if (!hasGrep) {
1503 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1504 } else {
1505 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1506 if (nullFile) {
1507 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1508 gSystem->Exec("rm -f __tmp*");
1604c9ed 1509 break;
3af861ff 1510 }
1511 TString line;
1512 ifstream in;
1513 in.open("__tmp__");
1514 in >> line;
1515 in.close();
1516 gSystem->Exec("rm -f __tmp__");
1517 ncount = line.Atoi();
1518 }
1519 nullResult = kFALSE;
1520 }
1521 if (ncount == gMaxEntries) {
1522 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1523 if (fNrunsPerMaster > 1) {
1524 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1525 file.Data(),gMaxEntries);
1526 return kFALSE;
1527 }
1528 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1529 if (!cbase) cbase = cadd;
1530 else {
1531 cbase->Add(cadd);
1532 delete cadd;
defd7a3a 1533 }
3af861ff 1534 nstart += ncount;
1535 } else {
c89a84bc 1536 if (cbase && fNrunsPerMaster<2) {
3af861ff 1537 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1538 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1539 cbase->Add(cadd);
1540 delete cadd;
1541 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1542 delete cbase; cbase = 0;
1543 } else {
1544 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1545 }
1546 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1547 break;
d3339be3 1548 }
319593fb 1549 }
a8739e8a 1550 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1551 // Check if there is one run per master job.
1552 if (fNrunsPerMaster<2) {
1553 if (FileExists(file)) {
0f389141 1554 if (fOverwriteMode) gGrid->Rm(file);
1555 else {
1556 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1557 continue;
1558 }
319593fb 1559 }
1560 // Copy xml file to alien space
1561 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1562 if (!FileExists(file)) {
1563 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1564 return kFALSE;
1565 }
1566 } else {
1567 nruns++;
95e5b448 1568 // Check if the collection for the chunk exist locally.
1569 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1570 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1571 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1572 else continue;
1573 }
84fcd93f 1574 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1575 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1576 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1577 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1578 } else {
1579 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1580 cbase->Add(cadd);
1581 delete cadd;
1582 }
74013535 1583 format = Form("%%s_%s.xml", fRunPrefix.Data());
1584 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1585 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1586 continue;
1587 }
904f9f5f 1588 schunk = schunk2;
319593fb 1589 if (FileExists(schunk)) {
0f389141 1590 if (fOverwriteMode) gGrid->Rm(schunk);
1591 else {
1592 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1593 continue;
1594 }
319593fb 1595 }
84fcd93f 1596 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1597 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1598 if (FileExists(schunk)) {
0f389141 1599 if (fOverwriteMode) gGrid->Rm(schunk);
1600 else {
1601 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1602 continue;
1603 }
95e5b448 1604 }
319593fb 1605 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1606 if (!FileExists(schunk)) {
1607 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1608 return kFALSE;
1609 }
1610 }
c57f56b7 1611 }
a2f5fc01 1612 if (nullResult) {
d3339be3 1613 Error("CreateDataset", "No valid dataset corresponding to the query!");
1614 return kFALSE;
1615 }
a8739e8a 1616 }
c57f56b7 1617 return kTRUE;
1618}
1619
1620//______________________________________________________________________________
1621Bool_t AliAnalysisAlien::CreateJDL()
1622{
1623// Generate a JDL file according to current settings. The name of the file is
1624// specified by fJDLName.
ec5acdce 1625 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
c57f56b7 1626 Bool_t error = kFALSE;
1627 TObjArray *arr = 0;
1628 Bool_t copy = kTRUE;
d3b18c4c 1629 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1630 Bool_t generate = kTRUE;
1631 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1632 if (!Connect()) {
1633 Error("CreateJDL", "Alien connection required");
1634 return kFALSE;
1635 }
1636 // Check validity of alien workspace
d3b18c4c 1637 TString workdir;
1638 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1639 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1640 workdir += fGridWorkingDir;
1641 if (generate) {
1642 TObjString *os;
1e466628 1643 if (!fInputFiles && !fMCLoop) {
c57f56b7 1644 Error("CreateJDL()", "Define some input files for your analysis.");
1645 error = kTRUE;
1646 }
1647 // Compose list of input files
1648 // Check if output files were defined
1649 if (!fOutputFiles.Length()) {
1650 Error("CreateJDL", "You must define at least one output file");
1651 error = kTRUE;
1652 }
1653 // Check if an output directory was defined and valid
1654 if (!fGridOutputDir.Length()) {
1655 Error("CreateJDL", "You must define AliEn output directory");
1656 error = kTRUE;
1657 } else {
d3b18c4c 1658 if (!fProductionMode) {
1659 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1660 if (!DirectoryExists(fGridOutputDir)) {
1661 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1662 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1663 } else {
1664 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1665 // error = kTRUE;
1666 }
67a3b254 1667 } else {
1668 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1669 }
1670 gGrid->Cd(workdir);
1671 }
c57f56b7 1672 }
1673 // Exit if any error up to now
1674 if (error) return kFALSE;
1675 // Set JDL fields
0f389141 1676 if (!fUser.IsNull()) {
1677 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1678 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
6da4839e
AG		 1679 }
1680 TString executable = fExecutable;
1681 if (!executable.BeginsWith("/"))
1682 executable.Prepend(Form("%s/", workdir.Data()));
1683 fGridJDL->SetExecutable(executable, "This is the startup script");
1684 TString mergeExec = executable;
0f389141 1685 mergeExec.ReplaceAll(".sh", "_merge.sh");
1686 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1687 mergeExec.ReplaceAll(".sh", ".C");
9c5cde18 1688 fMergingJDL->AddToInputSandbox(Form("LF:%s", mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1689 if (!fArguments.IsNull())
1690 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1691 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1692 else {
1693 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1694 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1695 }
b3e07543 1696
1f0d1ca2 1697 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1698 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1699 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1700 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1701
0f389141 1702 if (fMaxInitFailed > 0) {
c57f56b7 1703 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1704 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1705 }
6da4839e 1706 if (fSplitMaxInputFileNumber > 0 && !fMCLoop) {
c57f56b7 1707 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1708 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1709 }
d7eb1f39 1710 if (!IsOneStageMerging()) {
f47d5cb4 1711 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1712 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1713 }
1714 if (fSplitMode.Length()) {
c57f56b7 1715 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1716 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1717 }
1718 fMergingJDL->SetValue("Split", "\"se\"");
1719 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1720 if (!fAliROOTVersion.IsNull()) {
1721 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1722 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1723 }
1724 if (!fROOTVersion.IsNull()) {
c57f56b7 1725 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1726 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1727 }
1728 if (!fAPIVersion.IsNull()) {
c57f56b7 1729 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1730 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1731 }
648174cf 1732 if (!fExternalPackages.IsNull()) {
1733 arr = fExternalPackages.Tokenize(" ");
1734 TIter next(arr);
1735 while ((os=(TObjString*)next())) {
1736 TString pkgname = os->GetString();
1737 Int_t index = pkgname.Index("::");
1738 TString pkgversion = pkgname(index+2, pkgname.Length());
1739 pkgname.Remove(index);
1740 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1741 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1742 }
1743 delete arr;
1744 }
6da4839e
AG		 1745 if (!fMCLoop) {
1746 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1747 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
6da4839e 1748 }
d7eb1f39
AG		 1749 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1750 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1751 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1752 TString analysisFile = fExecutable;
1753 analysisFile.ReplaceAll(".sh", ".root");
1754 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1755 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1756 if (fAdditionalLibs.Length()) {
1757 arr = fAdditionalLibs.Tokenize(" ");
1758 TIter next(arr);
1759 while ((os=(TObjString*)next())) {
1760 if (os->GetString().Contains(".so")) continue;
1761 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1762 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1763 }
1764 delete arr;
1765 }
4e5c5506 1766 if (fPackages) {
1767 TIter next(fPackages);
1768 TObject *obj;
0f389141 1769 while ((obj=next())) {
4e5c5506 1770 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1771 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1772 }
4e5c5506 1773 }
95e16f01 1774 const char *comment = "List of output files and archives";
c57f56b7 1775 if (fOutputArchive.Length()) {
e4d9a069 1776 TString outputArchive = fOutputArchive;
1777 if (!fRegisterExcludes.IsNull()) {
1778 arr = fRegisterExcludes.Tokenize(" ");
1779 TIter next1(arr);
1780 while ((os=(TObjString*)next1())) {
1781 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1782 outputArchive.ReplaceAll(os->GetString(),"");
1783 }
1784 delete arr;
1785 }
1786 arr = outputArchive.Tokenize(" ");
c57f56b7 1787 TIter next(arr);
0f389141 1788 Bool_t first = kTRUE;
0f389141 1789 while ((os=(TObjString*)next())) {
0f389141 1790 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1791 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
0f389141 1792 else
95e16f01 1793 fGridJDL->AddToSet("Output", os->GetString());
1794 if (first) fGridJDL->AddToSetDescription("Output", comment);
0f389141 1795 first = kFALSE;
1796 }
c57f56b7 1797 delete arr;
f790bc1b 1798 // Output archive for the merging jdl
f790bc1b 1799 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1800 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1801 // Add normal output files, extra files + terminate files
1802 TString files = GetListOfFiles("outextter");
37d21c01 1803 // Do not register files in fRegisterExcludes
1804 if (!fRegisterExcludes.IsNull()) {
1805 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1806 TIter next1(arr);
1807 while ((os=(TObjString*)next1())) {
1808 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1809 files.ReplaceAll(os->GetString(),"");
1810 }
66d71516 1811 delete arr;
0f389141 1812 }
f790bc1b 1813 files.ReplaceAll(".root", "*.root");
ec5acdce 1814
1815 if (mgr->IsCollectThroughput())
95e16f01 1816 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",files.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 1817 else
1818 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1819 } else {
95f84779 1820 TString files = fOutputArchive;
1821 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1822 outputArchive = files;
f790bc1b 1823 }
0f389141 1824 arr = outputArchive.Tokenize(" ");
705adb3e 1825 TIter next2(arr);
0f389141 1826 first = kTRUE;
705adb3e 1827 while ((os=(TObjString*)next2())) {
0cdf65a8 1828 TString currentfile = os->GetString();
0cdf65a8 1829 if (!currentfile.Contains("@") && fCloseSE.Length())
95e16f01 1830 fMergingJDL->AddToSet("Output", Form("%s@%s",currentfile.Data(), fCloseSE.Data()));
0f389141 1831 else
95e16f01 1832 fMergingJDL->AddToSet("Output", currentfile);
1833 if (first) fMergingJDL->AddToSetDescription("Output", comment);
0f389141 1834 first = kFALSE;
1835 }
1836 delete arr;
c57f56b7 1837 }
149d288c 1838 arr = fOutputFiles.Tokenize(",");
c57f56b7 1839 TIter next(arr);
0f389141 1840 Bool_t first = kTRUE;
43da816a 1841 while ((os=(TObjString*)next())) {
e1eaf596 1842 // Ignore ouputs in jdl that are also in outputarchive
1843 TString sout = os->GetString();
f790bc1b 1844 sout.ReplaceAll("*", "");
1845 sout.ReplaceAll(".root", "");
e1eaf596 1846 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1847 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1848 // Ignore fRegisterExcludes
1849 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1850 if (!first) comment = NULL;
43da816a 1851 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1852 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
43da816a 1853 else
95e16f01 1854 fGridJDL->AddToSet("Output", os->GetString());
1855 if (first) fGridJDL->AddToSetDescription("Output", comment);
f790bc1b 1856 if (fMergeExcludes.Contains(sout)) continue;
1857 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1858 fMergingJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
f790bc1b 1859 else
95e16f01 1860 fMergingJDL->AddToSet("Output", os->GetString());
1861 if (first) fMergingJDL->AddToSetDescription("Output", comment);
1862 first = kFALSE;
43da816a 1863 }
c57f56b7 1864 delete arr;
0f389141 1865 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1866 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1867 TString validationScript = fValidationScript;
0f389141 1868 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1869 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1870 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1871 if (fMasterResubmitThreshold) {
1872 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1873 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1874 }
d2a409b2 1875 // Write a jdl with 2 input parameters: collection name and output dir name.
1876 WriteJDL(copy);
c57f56b7 1877 }
1878 // Copy jdl to grid workspace
a8739e8a 1879 if (copy) {
b5e4aaa7 1880 // Check if an output directory was defined and valid
1881 if (!fGridOutputDir.Length()) {
1882 Error("CreateJDL", "You must define AliEn output directory");
1883 return kFALSE;
1884 } else {
1885 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1886 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1887 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1888 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1889 } else {
1890 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1891 return kFALSE;
1892 }
1893 }
1894 gGrid->Cd(workdir);
1895 }
648174cf 1896 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1897 TString mergeJDLName = fExecutable;
1898 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1899 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1900 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1901 if (fProductionMode) {
648174cf 1902 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1903 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1904 }
648174cf 1905 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1906 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1907 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1908 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1909 Fatal("","Terminating");
1910// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1911 if (fMergeViaJDL) {
1912 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1913// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1914 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1915 Fatal("","Terminating");
0f389141 1916 }
648174cf 1917 }
c57f56b7 1918 if (fAdditionalLibs.Length()) {
e7c71df0 1919 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1920 TObjString *os;
1921 TIter next(arr);
1922 while ((os=(TObjString*)next())) {
c57f56b7 1923 if (os->GetString().Contains(".so")) continue;
4e5c5506 1924 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1925 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1926// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1927 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1928 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1929 Fatal("","Terminating");
c57f56b7 1930 }
1931 delete arr;
1932 }
4e5c5506 1933 if (fPackages) {
1934 TIter next(fPackages);
1935 TObject *obj;
1936 while ((obj=next())) {
fdbbc7be 1937 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1938 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1939// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1940 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1941 Form("%s/%s", workdir.Data(), obj->GetName())))
1942 Fatal("","Terminating");
4e5c5506 1943 }
1944 }
c57f56b7 1945 }
1946 return kTRUE;
1947}
1948
a8739e8a 1949//______________________________________________________________________________
d2a409b2 1950Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1951{
1952// Writes one or more JDL's corresponding to findex. If findex is negative,
1953// all run numbers are considered in one go (jdl). For non-negative indices
1954// they correspond to the indices in the array fInputFiles.
dec33edc 1955 if (!fInputFiles && !fMCLoop) return kFALSE;
d3b18c4c 1956 TObject *os;
1957 TString workdir;
1958 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1959 workdir += fGridWorkingDir;
f47d5cb4 1960 TString stageName = "$2";
1961 if (fProductionMode) stageName = "$4";
1962 if (!fMergeDirName.IsNull()) {
1963 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1964 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1965 } else {
1966 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1967 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1968 }
d3b18c4c 1969 if (fProductionMode) {
a8739e8a 1970 TIter next(fInputFiles);
b3e07543 1971 while ((os=next())) {
d3b18c4c 1972 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1973 }
56754bf6 1974 if (!fOutputToRunNo)
1975 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1976 else
1977 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1978 } else {
1979 if (!fRunNumbers.Length() && !fRunRange[0]) {
1980 // One jdl with no parameters in case input data is specified by name.
1981 TIter next(fInputFiles);
1982 while ((os=next()))
1983 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1984 if (!fOutputSingle.IsNull())
1985 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1986 else {
1987 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1988 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1989 }
1990 } else {
1991 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1992 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1993 if (!fOutputSingle.IsNull()) {
1994 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1995 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1996 } else {
1997 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1998 }
1999 }
a8739e8a 2000 }
2001
a8739e8a 2002 // Generate the JDL as a string
2003 TString sjdl = fGridJDL->Generate();
0f389141 2004 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 2005 // Final merge jdl
2006 if (!fMergeDirName.IsNull()) {
2007 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
2008 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
2009 } else {
2010 fMergingJDL->SetOutputDirectory("$1", "Output directory");
2011 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
2012 }
b3e07543 2013 TString sjdl2 = fMergingJDL->Generate();
2014 Int_t index, index1;
95e16f01 2015 sjdl.ReplaceAll("\",\"", "\",\n \"");
a8739e8a 2016 sjdl.ReplaceAll("(member", "\n (member");
2017 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2018 sjdl.ReplaceAll("{", "{\n ");
2019 sjdl.ReplaceAll("};", "\n};");
2020 sjdl.ReplaceAll("{\n \n", "{\n");
2021 sjdl.ReplaceAll("\n\n", "\n");
2022 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2023 sjdl1.ReplaceAll("\",\"", "\",\n \"");
0f389141 2024 sjdl1.ReplaceAll("(member", "\n (member");
2025 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2026 sjdl1.ReplaceAll("{", "{\n ");
2027 sjdl1.ReplaceAll("};", "\n};");
2028 sjdl1.ReplaceAll("{\n \n", "{\n");
2029 sjdl1.ReplaceAll("\n\n", "\n");
2030 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2031 sjdl2.ReplaceAll("\",\"", "\",\n \"");
b3e07543 2032 sjdl2.ReplaceAll("(member", "\n (member");
2033 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2034 sjdl2.ReplaceAll("{", "{\n ");
2035 sjdl2.ReplaceAll("};", "\n};");
2036 sjdl2.ReplaceAll("{\n \n", "{\n");
2037 sjdl2.ReplaceAll("\n\n", "\n");
2038 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 2039 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 2040 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 2041 index = sjdl.Index("JDLVariables");
2042 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 2043 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2044 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 2045 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 2046 index = fJobTag.Index(":");
2047 if (index < 0) index = fJobTag.Length();
2048 TString jobTag = fJobTag;
f47d5cb4 2049 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 2050 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 2051 if (fProductionMode) {
2052 sjdl1.Prepend("# Generated merging jdl (production mode) \
2053 \n# $1 = full alien path to output directory to be merged \
2054 \n# $2 = train number \
2055 \n# $3 = production (like LHC10b) \
2056 \n# $4 = merging stage \
2057 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2058 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2059 sjdl2.Prepend("# Generated merging jdl \
2060 \n# $1 = full alien path to output directory to be merged \
2061 \n# $2 = train number \
2062 \n# $3 = production (like LHC10b) \
2063 \n# $4 = merging stage \
2064 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2065 } else {
2066 sjdl1.Prepend("# Generated merging jdl \
2067 \n# $1 = full alien path to output directory to be merged \
2068 \n# $2 = merging stage \
03335324 2069 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2070 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2071 sjdl2.Prepend("# Generated merging jdl \
2072 \n# $1 = full alien path to output directory to be merged \
2073 \n# $2 = merging stage \
03335324 2074 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2075 }
0f389141 2076 index = sjdl1.Index("JDLVariables");
2077 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 2078 index = sjdl2.Index("JDLVariables");
2079 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 2080 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2081 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
2082 index = sjdl2.Index("Split =");
2083 if (index>=0) {
2084 index1 = sjdl2.Index("\n", index);
2085 sjdl2.Remove(index, index1-index+1);
2086 }
2087 index = sjdl2.Index("SplitMaxInputFileNumber");
2088 if (index>=0) {
2089 index1 = sjdl2.Index("\n", index);
2090 sjdl2.Remove(index, index1-index+1);
2091 }
2092 index = sjdl2.Index("InputDataCollection");
2093 if (index>=0) {
2094 index1 = sjdl2.Index(";", index);
2095 sjdl2.Remove(index, index1-index+1);
2096 }
2097 index = sjdl2.Index("InputDataListFormat");
2098 if (index>=0) {
2099 index1 = sjdl2.Index("\n", index);
2100 sjdl2.Remove(index, index1-index+1);
2101 }
2102 index = sjdl2.Index("InputDataList");
2103 if (index>=0) {
2104 index1 = sjdl2.Index("\n", index);
2105 sjdl2.Remove(index, index1-index+1);
2106 }
f47d5cb4 2107 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2108 // Write jdl to file
a8739e8a 2109 ofstream out;
d2a409b2 2110 out.open(fJDLName.Data(), ios::out);
a8739e8a 2111 if (out.bad()) {
d3b18c4c 2112 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2113 return kFALSE;
2114 }
2115 out << sjdl << endl;
b3e07543 2116 out.close();
0f389141 2117 TString mergeJDLName = fExecutable;
2118 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2119 if (fMergeViaJDL) {
2120 ofstream out1;
2121 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2122 if (out1.bad()) {
d3b18c4c 2123 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2124 return kFALSE;
2125 }
2126 out1 << sjdl1 << endl;
b3e07543 2127 out1.close();
2128 ofstream out2;
2129 TString finalJDL = mergeJDLName;
2130 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2131 out2.open(finalJDL.Data(), ios::out);
2132 if (out2.bad()) {
2133 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2134 return kFALSE;
2135 }
2136 out2 << sjdl2 << endl;
2137 out2.close();
0f389141 2138 }
a8739e8a 2139
2140 // Copy jdl to grid workspace
2141 if (!copy) {
d3b18c4c 2142 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2143 } else {
d2a409b2 2144 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2145 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2146 TString finalJDL = mergeJDLName;
2147 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2148 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2149 if (fProductionMode) {
b5fe9cba 2150 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2151 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2152 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2153 }
d2a409b2 2154 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2155 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2156 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2157 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2158// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2159 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2160 Fatal("","Terminating");
0f389141 2161 if (fMergeViaJDL) {
b3e07543 2162 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2163// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2164// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2165 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2166 Fatal("","Terminating");
2167 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2168 Fatal("","Terminating");
0f389141 2169 }
a8739e8a 2170 }
2171 return kTRUE;
2172}
2173
c57f56b7 2174//______________________________________________________________________________
5513444a 2175Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2176{
2177// Returns true if file exists.
5513444a 2178 if (!gGrid) return kFALSE;
2ea55496 2179 TString slfn = lfn;
2180 slfn.ReplaceAll("alien://","");
2181 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2182 if (!res) return kFALSE;
2183 TMap *map = dynamic_cast<TMap*>(res->At(0));
2184 if (!map) {
2185 delete res;
2186 return kFALSE;
2187 }
2188 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2189 if (!objs || !objs->GetString().Length()) {
2190 delete res;
2191 return kFALSE;
2192 }
2193 delete res;
2194 return kTRUE;
2195}
2196
923e2ca5 2197//______________________________________________________________________________
2198Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2199{
2200// Returns true if directory exists. Can be also a path.
2201 if (!gGrid) return kFALSE;
2202 // Check if dirname is a path
2203 TString dirstripped = dirname;
2204 dirstripped = dirstripped.Strip();
2205 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2206 TString dir = gSystem->BaseName(dirstripped);
2207 dir += "/";
2208 TString path = gSystem->DirName(dirstripped);
2209 TGridResult *res = gGrid->Ls(path, "-F");
2210 if (!res) return kFALSE;
2211 TIter next(res);
2212 TMap *map;
2213 TObject *obj;
2214 while ((map=dynamic_cast<TMap*>(next()))) {
2215 obj = map->GetValue("name");
2216 if (!obj) break;
2217 if (dir == obj->GetName()) {
2218 delete res;
2219 return kTRUE;
2220 }
2221 }
2222 delete res;
2223 return kFALSE;
2224}
2225
c57f56b7 2226//______________________________________________________________________________
a2f5fc01 2227void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2228{
2229// Check input data type.
a2f5fc01 2230 isCollection = kFALSE;
2231 isXml = kFALSE;
2232 useTags = kFALSE;
c57f56b7 2233 if (!gGrid) {
2234 Error("CheckDataType", "No connection to grid");
2235 return;
2236 }
a2f5fc01 2237 isCollection = IsCollection(lfn);
c57f56b7 2238 TString msg = "\n##### file: ";
2239 msg += lfn;
a2f5fc01 2240 if (isCollection) {
c57f56b7 2241 msg += " type: raw_collection;";
2242 // special treatment for collections
a2f5fc01 2243 isXml = kFALSE;
c57f56b7 2244 // check for tag files in the collection
2245 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2246 if (!res) {
2247 msg += " using_tags: No (unknown)";
23329835 2248 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2249 return;
2250 }
2251 const char* typeStr = res->GetKey(0, "origLFN");
2252 if (!typeStr || !strlen(typeStr)) {
2253 msg += " using_tags: No (unknown)";
23329835 2254 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2255 return;
2256 }
2257 TString file = typeStr;
a2f5fc01 2258 useTags = file.Contains(".tag");
2259 if (useTags) msg += " using_tags: Yes";
c57f56b7 2260 else msg += " using_tags: No";
23329835 2261 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2262 return;
2263 }
2264 TString slfn(lfn);
2265 slfn.ToLower();
a2f5fc01 2266 isXml = slfn.Contains(".xml");
2267 if (isXml) {
c57f56b7 2268 // Open xml collection and check if there are tag files inside
2269 msg += " type: xml_collection;";
2270 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2271 if (!coll) {
2272 msg += " using_tags: No (unknown)";
23329835 2273 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2274 return;
2275 }
2276 TMap *map = coll->Next();
2277 if (!map) {
2278 msg += " using_tags: No (unknown)";
23329835 2279 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2280 return;
2281 }
2282 map = (TMap*)map->GetValue("");
2283 TString file;
2284 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2285 useTags = file.Contains(".tag");
c57f56b7 2286 delete coll;
a2f5fc01 2287 if (useTags) msg += " using_tags: Yes";
c57f56b7 2288 else msg += " using_tags: No";
23329835 2289 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2290 return;
2291 }
a2f5fc01 2292 useTags = slfn.Contains(".tag");
c57f56b7 2293 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2294 else msg += " type: unknown file;";
a2f5fc01 2295 if (useTags) msg += " using_tags: Yes";
c57f56b7 2296 else msg += " using_tags: No";
23329835 2297 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2298}
2299
4e5c5506 2300//______________________________________________________________________________
2301void AliAnalysisAlien::EnablePackage(const char *package)
2302{
2303// Enables a par file supposed to exist in the current directory.
2304 TString pkg(package);
2305 pkg.ReplaceAll(".par", "");
2306 pkg += ".par";
2307 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2308 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2309 return;
2310 }
2311 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2312 Info("EnablePackage", "AliEn plugin will use .par packages");
2313 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2314 if (!fPackages) {
2315 fPackages = new TObjArray();
2316 fPackages->SetOwner();
2317 }
2318 fPackages->Add(new TObjString(pkg));
2319}
2320
3bdcb562 2321//______________________________________________________________________________
2322TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2323{
2324// Make a tree from files having the location specified in fFileForTestMode.
2325// Inspired from JF's CreateESDChain.
2326 if (fFileForTestMode.IsNull()) {
2327 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2328 return NULL;
2329 }
2330 if (gSystem->AccessPathName(fFileForTestMode)) {
2331 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2332 return NULL;
2333 }
2334 // Open the file
2335 ifstream in;
2336 in.open(fFileForTestMode);
2337 Int_t count = 0;
2338 // Read the input list of files and add them to the chain
b505f1f8 2339 TString line;
d6c60197 2340 TString streeName(treeName);
2341 if (IsUseMCchain()) streeName = "TE";
2342 TChain *chain = new TChain(streeName);
06f67114 2343 TList *friends = new TList();
2344 TChain *cfriend = 0;
2345 if (!fFriendChainName.IsNull()) {
2346 TObjArray *list = fFriendChainName.Tokenize(" ");
2347 TIter next(list);
2348 TObjString *str;
2349 while((str=(TObjString*)next())) {
2350 cfriend = new TChain(streeName, str->GetName());
2351 friends->Add(cfriend);
2352 chain->AddFriend(cfriend);
2353 }
2354 delete list;
2355 }
2356 TString bpath;
2357 TIter nextfriend(friends);
b505f1f8 2358 while (in.good())
2359 {
3bdcb562 2360 in >> line;
ffbe06b9 2361 if (line.IsNull() || line.BeginsWith("#")) continue;
3bdcb562 2362 if (count++ == fNtestFiles) break;
2363 TString esdFile(line);
2364 TFile *file = TFile::Open(esdFile);
b505f1f8 2365 if (file && !file->IsZombie()) {
2366 chain->Add(esdFile);
3bdcb562 2367 file->Close();
b505f1f8 2368 if (!fFriendChainName.IsNull()) {
4b5d9e21 2369 if (esdFile.Index("#") > -1)
2370 esdFile.Remove(esdFile.Index("#"));
06f67114 2371 bpath = gSystem->DirName(esdFile);
2372 bpath += "/";
2373 TString fileFriend;
2374 nextfriend.Reset();
2375 while ((cfriend=(TChain*)nextfriend())) {
2376 fileFriend = bpath;
2377 fileFriend += cfriend->GetTitle();
2378 file = TFile::Open(fileFriend);
2379 if (file && !file->IsZombie()) {
2380 file->Close();
2381 cfriend->Add(fileFriend);
2382 } else {
2383 Fatal("GetChainForTestMode", "Cannot open friend file: %s", fileFriend.Data());
2384 return 0;
2385 }
2386 }
4b5d9e21 2387 }
3bdcb562 2388 } else {
2389 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2390 }
b505f1f8 2391 }
2392 in.close();
2393 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2394 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2395 delete chain;
06f67114 2396 friends->Delete();
2397 delete friends;
3bdcb562 2398 return NULL;
b505f1f8 2399 }
b505f1f8 2400 return chain;
3bdcb562 2401}
2402
d2a409b2 2403//______________________________________________________________________________
2404const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2405{
2406// Get job status for all jobs with jobid>jobidstart.
2407 static char mstatus[20];
2408 mstatus[0] = '\0';
2409 nrunning = 0;
2410 nwaiting = 0;
2411 nerror = 0;
2412 ndone = 0;
2413 TGridJobStatusList *list = gGrid->Ps("");
2414 if (!list) return mstatus;
2415 Int_t nentries = list->GetSize();
2416 TGridJobStatus *status;
2417 Int_t pid;
2418 for (Int_t ijob=0; ijob<nentries; ijob++) {
2419 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2420 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2421 if (pid<jobidstart) continue;
2422 if (pid == lastid) {
8ddad121 2423 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2424 }
2425 switch (status->GetStatus()) {
2426 case TGridJobStatus::kWAITING:
2427 nwaiting++; break;
2428 case TGridJobStatus::kRUNNING:
2429 nrunning++; break;
2430 case TGridJobStatus::kABORTED:
2431 case TGridJobStatus::kFAIL:
2432 case TGridJobStatus::kUNKNOWN:
2433 nerror++; break;
2434 case TGridJobStatus::kDONE:
2435 ndone++;
2436 }
2437 }
2438 list->Delete();
2439 delete list;
2440 return mstatus;
2441}
2442
c57f56b7 2443//______________________________________________________________________________
2444Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2445{
2446// Returns true if file is a collection. Functionality duplicated from
2447// TAlien::Type() because we don't want to directly depend on TAlien.
2448 if (!gGrid) {
2449 Error("IsCollection", "No connection to grid");
2450 return kFALSE;
2451 }
2452 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2453 if (!res) return kFALSE;
2454 const char* typeStr = res->GetKey(0, "type");
2455 if (!typeStr || !strlen(typeStr)) return kFALSE;
2456 if (!strcmp(typeStr, "collection")) return kTRUE;
2457 delete res;
2458 return kFALSE;
2459}
2460
fe2d7fc2 2461//______________________________________________________________________________
2462Bool_t AliAnalysisAlien::IsSingleOutput() const
2463{
2464// Check if single-ouput option is on.
2465 return (!fOutputSingle.IsNull());
2466}
715da79c 2467
2468//______________________________________________________________________________
2469Long64_t AliAnalysisAlien::RunMacroAndExtractLibs(const char* macro, const char *args, TString &libs)
2470{
2471// Tries to run the specified macro and return the libraries that it loads.
2472 TString expname;
2473 if (strlen(macro)) expname = gSystem->ExpandPathName(macro);
2474 if (expname.IsNull() || gSystem->AccessPathName(expname)) {
2475 ::Error("RunMacroAndExtractLibs","Cannot find macro %s in current directory", macro);
2476 return -1;
2477 }
2478 TString oldlibs = gSystem->GetLibraries();
2479 TMacro m(expname);
2480 Int_t error = 0;
2481 Long64_t retval = m.Exec(args, &error);
2482 if (error != TInterpreter::kNoError)
2483 {
2484 ::Error("RunMacroAndExtractLibs", "Macro interpretation %s failed", macro);
2485 return -1;
2486 }
2487 libs = gSystem->GetLibraries();
2488 libs.ReplaceAll(oldlibs, "");
2489 libs.Strip(TString::kLeading);
2490 TObjArray *libTokens = libs.Tokenize(" ");
2491 libs = "";
2492 for (Int_t i=0; i<libTokens->GetEntries(); i++) {
2493 if (!libs.IsNull()) libs += " ";
2494 libs += gSystem->BaseName(libTokens->At(i)->GetName());
2495 }
2496 delete libTokens;
2497 return retval;
2498}
2499
16a4353c 2500//______________________________________________________________________________
2501void AliAnalysisAlien::Print(Option_t *) const
2502{
2503// Print current plugin settings.
84fcd93f 2504 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2505 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2506 if (mgr && mgr->IsProofMode()) {
2507 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2508 if (TestBit(AliAnalysisGrid::kTest))
2509 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2510 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2511 if (!fProofDataSet.IsNull())
2512 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2513 if (fProofReset==1)
2514 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2515 if (fProofReset>1)
2516 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
874de7ea 2517 if (!fROOTVersion.IsNull())
2518 printf("= ROOT version requested________________________ %s\n", fROOTVersion.Data());
d3b18c4c 2519 else
2520 printf("= ROOT version requested________________________ default\n");
2521 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2522 if (!fAliRootMode.IsNull())
2523 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2524 if (fNproofWorkers)
2525 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2526 if (fNproofWorkersPerSlave)
2527 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2528 if (TestSpecialBit(kClearPackages))
2529 printf("= ClearPackages requested...\n");
2530 if (fIncludePath.Data())
2531 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2532 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2533 if (fPackages && fPackages->GetEntries()) {
2534 TIter next(fPackages);
2535 TObject *obj;
2536 TString list;
2537 while ((obj=next())) list += obj->GetName();
2538 printf("= Par files to be used: ________________________ %s\n", list.Data());
2539 }
2540 if (TestSpecialBit(kProofConnectGrid))
2541 printf("= Requested PROOF connection to grid\n");
2542 return;
2543 }
e1c22e21 2544 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2545 if (fOverwriteMode) {
2546 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2547 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2548 }
348be253 2549 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
780ec36d 2550 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO:Print");
84fcd93f 2551 printf("= Production mode:______________________________ %d\n", fProductionMode);
2552 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2553 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2554 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2555 if (fUser.Length())
84fcd93f 2556 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2557 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2558 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
780ec36d 2559 TString basedatadir = fGridDataDir;
2560 TString pattern = fDataPattern;
2561 pattern.Strip();
2562 Int_t ind = pattern.Index(" ");
2563 if (ind>=0) {
2564 basedatadir += "/%run%/";
2565 basedatadir += pattern(0, ind);
2566 pattern = pattern(ind+1, pattern.Length());
2567 }
2568 printf("= Data base directory path requested: __________ %s\n", basedatadir.Data());
2569 printf("= Data search pattern: _________________________ %s\n", pattern.Data());
84fcd93f 2570 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2571 if (fRunNumbers.Length())
84fcd93f 2572 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2573 if (fRunRange[0])
74013535 2574 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2575 if (!fRunRange[0] && !fRunNumbers.Length()) {
2576 TIter next(fInputFiles);
2577 TObject *obj;
2578 TString list;
2579 while ((obj=next())) list += obj->GetName();
84fcd93f 2580 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2581 }
2582 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2583 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2584 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2585 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2586 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2587 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2588 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2589 printf("=====================================================================\n");
2590 printf("= Job price: ___________________________________ %d\n", fPrice);
2591 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2592 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2593 if (fMaxInitFailed>0)
84fcd93f 2594 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2595 if (fMasterResubmitThreshold>0)
84fcd93f 2596 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2597 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2598 if (fNrunsPerMaster>0)
84fcd93f 2599 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2600 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2601 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2602 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2603 if (fArguments.Length())
84fcd93f 2604 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2605 if (fExecutableArgs.Length())
2606 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2607 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2608 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2609 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2610 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2611 if (fDatasetName)
84fcd93f 2612 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2613 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2614 if (fIncludePath.Data())
84fcd93f 2615 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2616 if (fCloseSE.Length())
84fcd93f 2617 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2618 if (fFriendChainName.Length())
84fcd93f 2619 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2620 if (fPackages && fPackages->GetEntries()) {
16a4353c 2621 TIter next(fPackages);
2622 TObject *obj;
2623 TString list;
2624 while ((obj=next())) list += obj->GetName();
84fcd93f 2625 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2626 }
2627}
2628
c57f56b7 2629//______________________________________________________________________________
2630void AliAnalysisAlien::SetDefaults()
2631{
2632// Set default values for everything. What cannot be filled will be left empty.
2633 if (fGridJDL) delete fGridJDL;
2634 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2635 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2636 fPrice = 1;
2637 fTTL = 30000;
2638 fSplitMaxInputFileNumber = 100;
2639 fMaxInitFailed = 0;
2640 fMasterResubmitThreshold = 0;
bb885a9e 2641 fNtestFiles = 10;
149d288c 2642 fNreplicas = 2;
a8739e8a 2643 fRunRange[0] = 0;
2644 fRunRange[1] = 0;
8e3bd06a 2645 fRunPrefix = "%d";
319593fb 2646 fNrunsPerMaster = 1;
16a4353c 2647 fMaxMergeFiles = 100;
c57f56b7 2648 fRunNumbers = "";
2649 fExecutable = "analysis.sh";
5aa94dc3 2650 fExecutableCommand = "root -b -q -x";
c57f56b7 2651 fArguments = "";
631c0b05 2652 fExecutableArgs = "";
c57f56b7 2653 fAnalysisMacro = "myAnalysis.C";
2654 fAnalysisSource = "";
2655 fAdditionalLibs = "";
2656 fSplitMode = "se";
2657 fAPIVersion = "";
2658 fROOTVersion = "";
2659 fAliROOTVersion = "";
2660 fUser = ""; // Your alien user name
2661 fGridWorkingDir = "";
2662 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2663 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2664 fFriendChainName = "";
c57f56b7 2665 fGridOutputDir = "output";
b83f84f0 2666 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2667 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2668 fInputFormat = "xml-single";
2669 fJDLName = "analysis.jdl";
c6cb3634 2670 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2671 fMergeExcludes = "";
0f389141 2672 fMergeViaJDL = 0;
348be253 2673 SetUseCopy(kTRUE);
2674 SetCheckCopy(kTRUE);
149d288c 2675 SetDefaultOutputs(kTRUE);
e1c22e21 2676 fOverwriteMode = 1;
c57f56b7 2677}
2678
a41b2882 2679//______________________________________________________________________________
2680void AliAnalysisAlien::SetFriendChainName(const char *name, const char *libnames)
2681{
2682 // Set file name for the chain of friends and optionally additional libs to be loaded.
2683 // Libs should be separated by blancs.
2684 fFriendChainName = name;
06f67114 2685 fFriendChainName.ReplaceAll(",", " ");
2686 fFriendChainName.Strip();
2687 fFriendChainName.ReplaceAll(" ", " ");
2688
a41b2882 2689 fFriendLibs = libnames;
1a3f255c 2690 if (fFriendLibs.Length()) {
2691 if(!fFriendLibs.Contains(".so"))
2692 Fatal("SetFriendChainName()", "You should provide explicit library names (with extension)");
2693 fFriendLibs.ReplaceAll(",", " ");
2694 fFriendLibs.Strip();
2695 fFriendLibs.ReplaceAll(" ", " ");
a41b2882 2696 }
2697}
2698
874de7ea 2699//______________________________________________________________________________
2700void AliAnalysisAlien::SetRootVersionForProof(const char *version)
2701{
2702// Obsolete method. Use SetROOTVersion instead
2703 Warning("SetRootVersionForProof", "Obsolete. Use SetROOTVersion instead");
95e16f01 2704 if (fROOTVersion.IsNull()) SetROOTVersion(version);
2705 else Error("SetRootVersionForProof", "ROOT version already set to %s", fROOTVersion.Data());
874de7ea 2706}
2707
0f389141 2708//______________________________________________________________________________
b3e07543 2709Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2710{
b3e07543 2711// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2712 // First check if the result is already in the output directory.
2713 if (FileExists(Form("%s/%s",aliendir,filename))) {
2714 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2715 return kFALSE;
2716 }
b3e07543 2717 // Now check the last stage done.
2718 Int_t stage = 0;
2719 while (1) {
2720 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2721 stage++;
2722 }
2723 // Next stage of merging
2724 stage++;
2725 TString pattern = "*root_archive.zip";
2726 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2727 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2728 if (res) delete res;
2729 // Write standard output to file
2730 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2731 // Count the number of files inside
2732 ifstream ifile;
2733 ifile.open(Form("Stage_%d.xml",stage));
2734 if (!ifile.good()) {
2735 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2736 return kFALSE;
b3e07543 2737 }
2738 TString line;
2739 Int_t nfiles = 0;
2740 while (!ifile.eof()) {
2741 ifile >> line;
2742 if (line.Contains("/event")) nfiles++;
7c2cd90a 2743 }
b3e07543 2744 ifile.close();
2745 if (!nfiles) {
2746 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2747 return kFALSE;
2748 } else {
2749 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2750 }
b3e07543 2751 // Copy the file in the output directory
2752 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2753// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2754 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2755 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2756 // Check if this is the last stage to be done.
2757 Bool_t laststage = (nfiles<nperchunk);
2758 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
0ad3ea1f 2759 Int_t jobId = 0;
b3e07543 2760 if (laststage) {
2761 printf("### Submiting final merging stage %d\n", stage);
2762 TString finalJDL = jdl;
2763 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2764 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
0ad3ea1f 2765 jobId = SubmitSingleJob(query);
b3e07543 2766 } else {
2767 printf("### Submiting merging stage %d\n", stage);
03335324 2768 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
0ad3ea1f 2769 jobId = SubmitSingleJob(query);
7c2cd90a 2770 }
0ad3ea1f 2771 if (!jobId) return kFALSE;
2772
2773 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
2774 fGridJobIDs.Append(Form("%d", jobId));
2775 if (!fGridStages.IsNull()) fGridStages.Append(" ");
2776 fGridStages.Append(Form("%s_merge_stage%d",
2777 laststage ? "final" : "partial", stage));
2778
b3e07543 2779 return kTRUE;
2780}
7c2cd90a 2781
105631b5 2782//______________________________________________________________________________
2783AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2784{
2785// Loat the analysis manager from a file.
2786 TFile *file = TFile::Open(fname);
2787 if (!file) {
2788 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2789 return 0;
2790 }
2791 TIter nextkey(file->GetListOfKeys());
2792 AliAnalysisManager *mgr = 0;
2793 TKey *key;
2794 while ((key=(TKey*)nextkey())) {
2795 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2796 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2797 }
2798 if (!mgr)
2799 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2800 return mgr;
2801}
2802
7c2cd90a 2803//______________________________________________________________________________
2804Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2805{
2806// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2807 if (!gGrid) return 0;
2808 printf("=> %s ------> ",query);
2809 TGridResult *res = gGrid->Command(query);
2810 if (!res) return 0;
2811 TString jobId = res->GetKey(0,"jobId");
2812 delete res;
2813 if (jobId.IsNull()) {
2814 printf("submission failed. Reason:\n");
2815 gGrid->Stdout();
2816 gGrid->Stderr();
2817 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2818 return 0;
2819 }
0ad3ea1f 2820 Int_t ijobId = jobId.Atoi();
2821 printf(" Job id: '%s' (%d)\n", jobId.Data(), ijobId);
2822 return ijobId;
7c2cd90a 2823}
2824
ec5acdce 2825//______________________________________________________________________________
2826Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2827{
2828// Merges a collection of output files using concatenation.
d66aa2a9 2829 TString scoll(collection);
2830 if (!scoll.Contains(".xml")) return kFALSE;
ec5acdce 2831 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2832 if (!coll) {
2833 ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2834 return kFALSE;
2835 }
2836 // Iterate grid collection
2837 TString outtmp;
2838 Bool_t merged = kFALSE;
2839 Int_t ifile = 0;
2840 while (coll->Next()) {
2841 TString fname = gSystem->DirName(coll->GetTURL());
2842 fname += "/";
2843 fname += output;
2844 outtmp = Form("%d_%s", ifile, output);
2845 if (!TFile::Cp(fname, outtmp)) {
2846 ::Error("MergeInfo", "Could not copy %s", fname.Data());
2847 continue;
2848 }
2849 ifile++;
2850 if (ifile<2) {
2851 gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2852 continue;
2853 }
2854 gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2855 gSystem->Exec("cp tempmerged lastmerged");
2856 }
2857 if (ifile) {
2858 gSystem->Exec(Form("cp lastmerged %s", output));
2859 gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2860 merged = kTRUE;
2861 }
2862 return merged;
2863}
2864
7c2cd90a 2865//______________________________________________________________________________
b3e07543 2866Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2867{
b3e07543 2868// Merge given output files from basedir. Basedir can be an alien output directory
2869// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2870// files in a group (ignored for xml input). Merging can be done in stages:
2871// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2872// stage=1 : works with an xml of all root_archive.zip in the output directory
2873// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2874 TString outputFile = output;
0f389141 2875 TString command;
a2f5fc01 2876 TString outputChunk;
2877 TString previousChunk = "";
b3e07543 2878 TObjArray *listoffiles = new TObjArray();
2879// listoffiles->SetOwner();
a2f5fc01 2880 Int_t countChunk = 0;
2881 Int_t countZero = nmaxmerge;
0f389141 2882 Bool_t merged = kTRUE;
d66aa2a9 2883 Bool_t isGrid = kTRUE;
a2f5fc01 2884 Int_t index = outputFile.Index("@");
2885 if (index > 0) outputFile.Remove(index);
7c2cd90a 2886 TString inputFile = outputFile;
b3e07543 2887 TString sbasedir = basedir;
2888 if (sbasedir.Contains(".xml")) {
2889 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2890 nmaxmerge = 9999999;
2891 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2892 if (!coll) {
2893 ::Error("MergeOutput", "Input XML collection empty.");
2894 return kFALSE;
2895 }
2896 // Iterate grid collection
2897 while (coll->Next()) {
2898 TString fname = gSystem->DirName(coll->GetTURL());
2899 fname += "/";
2900 fname += inputFile;
2901 listoffiles->Add(new TNamed(fname.Data(),""));
2902 }
ffbe06b9 2903 } else if (sbasedir.Contains(".txt")) {
d66aa2a9 2904 // The file having the .txt extension is expected to contain a list of
2905 // folders where the output files will be looked. For alien folders,
2906 // the full folder LFN is expected (starting with alien://)
ffbe06b9 2907 // Assume lfn's on each line
2908 TString line;
2909 ifstream in;
2910 in.open(sbasedir);
d66aa2a9 2911 if (in.fail()) {
2912 ::Error("MergeOutput", "File %s cannot be opened. Merging stopped." ,sbasedir.Data());
2913 return kTRUE;
2914 }
ffbe06b9 2915 Int_t nfiles = 0;
2916 while (in.good()) {
2917 in >> line;
2918 if (line.IsNull() || line.BeginsWith("#")) continue;
d66aa2a9 2919 line.Strip();
2920 if (!line.Contains("alien:")) isGrid = kFALSE;
2921 line += "/";
2922 line += outputFile;
ffbe06b9 2923 nfiles++;
2924 listoffiles->Add(new TNamed(line.Data(),""));
2925 }
d66aa2a9 2926 in.close();
ffbe06b9 2927 if (!nfiles) {
2928 ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data());
2929 delete listoffiles;
2930 return kFALSE;
2931 }
b3e07543 2932 } else {
2933 command = Form("find %s/ *%s", basedir, inputFile.Data());
2934 printf("command: %s\n", command.Data());
2935 TGridResult *res = gGrid->Command(command);
2936 if (!res) {
2937 ::Error("MergeOutput","No result for the find command\n");
2938 delete listoffiles;
2939 return kFALSE;
2940 }
2941 TIter nextmap(res);
2942 TMap *map = 0;
2943 while ((map=(TMap*)nextmap())) {
2944 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2945 if (!objs || !objs->GetString().Length()) {
2946 // Nothing found - skip this output
2947 delete res;
2948 delete listoffiles;
2949 return kFALSE;
2950 }
2951 listoffiles->Add(new TNamed(objs->GetName(),""));
2952 }
2953 delete res;
2954 }
2955 if (!listoffiles->GetEntries()) {
7c2cd90a 2956 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2957 delete listoffiles;
0f389141 2958 return kFALSE;
2959 }
2960
2961 TFileMerger *fm = 0;
b3e07543 2962 TIter next0(listoffiles);
2963 TObjArray *listoffilestmp = new TObjArray();
2964 listoffilestmp->SetOwner();
2965 TObject *nextfile;
2966 TString snextfile;
2967 // Keep only the files at upper level
2968 Int_t countChar = 0;
2969 while ((nextfile=next0())) {
2970 snextfile = nextfile->GetName();
2971 Int_t crtCount = snextfile.CountChar('/');
2972 if (nextfile == listoffiles->First()) countChar = crtCount;
2973 if (crtCount < countChar) countChar = crtCount;
2974 }
2975 next0.Reset();
2976 while ((nextfile=next0())) {
2977 snextfile = nextfile->GetName();
2978 Int_t crtCount = snextfile.CountChar('/');
2979 if (crtCount > countChar) {
2980 delete nextfile;
2981 continue;
2982 }
2983 listoffilestmp->Add(nextfile);
2984 }
2985 delete listoffiles;
2986 listoffiles = listoffilestmp; // Now contains 'good' files
2987 listoffiles->Print();
2988 TIter next(listoffiles);
7c2cd90a 2989 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2990 outputChunk = outputFile;
2991 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2992 // Check for existent temporary merge files
e1c22e21 2993 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2994 // Preserve old merging functionality for stage 0.
2995 if (stage==0) {
2996 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2997 while (1) {
2998 // Skip as many input files as in a chunk
2ea55496 2999 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 3000 nextfile = next();
3001 if (!nextfile) {
2ea55496 3002 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 3003 delete listoffiles;
2ea55496 3004 return kFALSE;
3005 }
b3e07543 3006 snextfile = nextfile->GetName();
7c2cd90a 3007 }
3008 outputChunk = outputFile;
3009 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
3010 countChunk++;
3011 if (gSystem->AccessPathName(outputChunk)) continue;
3012 // Merged file with chunks up to <countChunk> found
3013 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
3014 previousChunk = outputChunk;
3015 break;
3016 }
3017 }
3018 countZero = nmaxmerge;
3019
b3e07543 3020 while ((nextfile=next())) {
3021 snextfile = nextfile->GetName();
2ea55496 3022 // Loop 'find' results and get next LFN
7c2cd90a 3023 if (countZero == nmaxmerge) {
3024 // First file in chunk - create file merger and add previous chunk if any.
d66aa2a9 3025 fm = new TFileMerger(isGrid);
7c2cd90a 3026 fm->SetFastMethod(kTRUE);
3027 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
3028 outputChunk = outputFile;
3029 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
3030 }
3031 // If last file found, put merged results in the output file
b3e07543 3032 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 3033 // Add file to be merged and decrement chunk counter.
b3e07543 3034 fm->AddFile(snextfile);
7c2cd90a 3035 countZero--;
b3e07543 3036 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 3037 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3038 // Nothing found - skip this output
3039 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 3040 merged = kFALSE;
3041 break;
7c2cd90a 3042 }
3043 fm->OutputFile(outputChunk);
3044 // Merge the outputs, then go to next chunk
3045 if (!fm->Merge()) {
3046 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 3047 merged = kFALSE;
3048 break;
7c2cd90a 3049 } else {
3050 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
3051 gSystem->Unlink(previousChunk);
3052 }
b3e07543 3053 if (nextfile == listoffiles->Last()) break;
7c2cd90a 3054 countChunk++;
3055 countZero = nmaxmerge;
3056 previousChunk = outputChunk;
0f389141 3057 }
0f389141 3058 }
b3e07543 3059 delete listoffiles;
3060 delete fm;
7c2cd90a 3061 return merged;
3062 }
3063 // Merging stage different than 0.
3064 // Move to the begining of the requested chunk.
d66aa2a9 3065 fm = new TFileMerger(isGrid);
7c2cd90a 3066 fm->SetFastMethod(kTRUE);
b3e07543 3067 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
3068 delete listoffiles;
7c2cd90a 3069 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3070 // Nothing found - skip this output
3071 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
3072 delete fm;
3073 return kFALSE;
3074 }
b3e07543 3075 fm->OutputFile(outputFile);
7c2cd90a 3076 // Merge the outputs
3077 if (!fm->Merge()) {
3078 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
3079 delete fm;
3080 return kFALSE;
3081 } else {
b3e07543 3082 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 3083 }
7c2cd90a 3084 delete fm;
3085 return kTRUE;
0f389141 3086}
3087
c57f56b7 3088//______________________________________________________________________________
3089Bool_t AliAnalysisAlien::MergeOutputs()
3090{
3091// Merge analysis outputs existing in the AliEn space.
3092 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
3093 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3094 if (!Connect()) {
3095 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
3096 return kFALSE;
0f389141 3097 }
c9e8f7fd 3098 if (fMergeViaJDL) {
3099 if (!TestBit(AliAnalysisGrid::kMerge)) {
3100 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
3101 return kFALSE;
3102 }
3103 if (fProductionMode) {
3104 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
3105 return kFALSE;
3106 }
0f389141 3107 Info("MergeOutputs", "Submitting merging JDL");
a03be957 3108 if (!SubmitMerging()) return kFALSE;
0f389141 3109 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 3110 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 3111 return kFALSE;
c57f56b7 3112 }
3113 // Get the output path
dc030a24 3114 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 3115 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 3116 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
3117 return kFALSE;
3118 }
3119 if (!fOutputFiles.Length()) {
3120 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3121 return kFALSE;
0f389141 3122 }
3123 // Check if fast read option was requested
e1c22e21 3124 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
3125 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 3126 if (fFastReadOption) {
d3339be3 3127 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
3128 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 3129 gEnv->SetValue("XNet.ConnectTimeout",50);
3130 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 3131 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 3132 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 3133 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 3134 }
e8b839ab 3135 // Make sure we change the temporary directory
3136 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 3137 // Set temporary compilation directory to current one
3138 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 3139 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3140 TIter next(list);
3141 TObjString *str;
a2f5fc01 3142 TString outputFile;
c57f56b7 3143 Bool_t merged = kTRUE;
3144 while((str=(TObjString*)next())) {
a2f5fc01 3145 outputFile = str->GetString();
3146 Int_t index = outputFile.Index("@");
3147 if (index > 0) outputFile.Remove(index);
3148 TString outputChunk = outputFile;
3149 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 3150 // Skip already merged outputs
a2f5fc01 3151 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 3152 if (fOverwriteMode) {
a2f5fc01 3153 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
3154 gSystem->Unlink(outputFile);
3155 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3156 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3157 outputChunk.Data());
3158 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3159 }
3160 } else {
a2f5fc01 3161 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 3162 continue;
3163 }
3164 } else {
a2f5fc01 3165 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3166 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3167 outputChunk.Data());
3168 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3169 }
3170 }
37d21c01 3171 if (fMergeExcludes.Contains(outputFile.Data()) ||
3172 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 3173 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 3174 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 3175 if (!merged) {
3176 Error("MergeOutputs", "Terminate() will NOT be executed");
09d5920f 3177 delete list;
3178 return kFALSE;
ff07ec61 3179 }
3180 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
3181 if (fileOpened) fileOpened->Close();
c57f56b7 3182 }
09d5920f 3183 delete list;
0f389141 3184 return kTRUE;
c57f56b7 3185}
3186
bb885a9e 3187//______________________________________________________________________________
3188void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
3189{
3190// Use the output files connected to output containers from the analysis manager
3191// rather than the files defined by SetOutputFiles
3192 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 3193 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 3194 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
3195}
3196
149d288c 3197//______________________________________________________________________________
3198void AliAnalysisAlien::SetOutputFiles(const char *list)
3199{
3200// Manually set the output files list.
3201// Removes duplicates. Not allowed if default outputs are not disabled.
3202 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3203 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
3204 return;
3205 }
3206 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
3207 fOutputFiles = "";
3208 TString slist = list;
3209 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
3210 TObjArray *arr = slist.Tokenize(" ");
3211 TObjString *os;
3212 TIter next(arr);
3213 TString sout;
3214 while ((os=(TObjString*)next())) {
3215 sout = os->GetString();
3216 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
3217 if (fOutputFiles.Contains(sout)) continue;
3218 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
3219 fOutputFiles += sout;
3220 }
3221 delete arr;
f790bc1b 3222}
149d288c 3223
3224//______________________________________________________________________________
3225void AliAnalysisAlien::SetOutputArchive(const char *list)
3226{
3227// Manually set the output archive list. Free text - you are on your own...
3228// Not allowed if default outputs are not disabled.
3229 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3230 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
3231 return;
3232 }
3233 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
3234 fOutputArchive = list;
3235}
3236
3237//______________________________________________________________________________
3238void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
3239{
3240// Setting a prefered output SE is not allowed anymore.
3241 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
3242}
3243
28ce6c9c 3244//______________________________________________________________________________
3245void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
3246{
3247// Set some PROOF special parameter.
3248 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3249 if (pair) {
3250 TObject *old = pair->Key();
3251 TObject *val = pair->Value();
3252 fProofParam.Remove(old);
3253 delete old;
3254 delete val;
3255 }
3256 fProofParam.Add(new TObjString(pname), new TObjString(value));
3257}
3258
3259//______________________________________________________________________________
3260const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
3261{
3262// Returns a special PROOF parameter.
3263 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3264 if (!pair) return 0;
3265 return pair->Value()->GetName();
3266}
3267
5513444a 3268//______________________________________________________________________________
3269Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 3270{
3271// Start remote grid analysis.
3bdcb562 3272 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3273 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
3274 if (!mgr || !mgr->IsInitialized()) {
3275 Error("StartAnalysis", "You need an initialized analysis manager for this");
3276 return kFALSE;
3277 }
3278 // Are we in PROOF mode ?
3279 if (mgr->IsProofMode()) {
baa34407 3280 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3281 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3282 if (fProofCluster.IsNull()) {
3283 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3284 return kFALSE;
3285 }
3286 if (fProofDataSet.IsNull() && !testMode) {
3287 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3288 return kFALSE;
3289 }
3290 // Set the needed environment
3291 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3292 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3293 if (fProofReset && !testMode) {
3294 if (fProofReset==1) {
3295 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3296 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3297 } else {
3298 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3299 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3300 }
3301 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3302 return kFALSE;
3303 }
baa34407 3304
3305 if (!testMode) {
3306 // Check if there is an old active session
3307 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3308 if (nsessions) {
3309 Error("StartAnalysis","You have to reset your old session first\n");
3310 return kFALSE;
3311 }
53911ca6 3312 }
3bdcb562 3313 // Do we need to change the ROOT version ? The success of this cannot be checked.
874de7ea 3314 if (!fROOTVersion.IsNull() && !testMode) {
cbccef47 3315 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"VO_ALICE@ROOT::%s\");",
874de7ea 3316 fProofCluster.Data(), fROOTVersion.Data()));
3bdcb562 3317 }
3318 // Connect to PROOF and check the status
3319 Long_t proof = 0;
27734f0e 3320 TString sworkers;
3321 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3322 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3323 if (!testMode) {
27734f0e 3324 if (!sworkers.IsNull())
3325 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3326 else
3327 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3328 } else {
3329 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3330 if (!proof) {
3331 Error("StartAnalysis", "Could not start PROOF in test mode");
3332 return kFALSE;
3333 }
3334 }
3335 if (!proof) {
3336 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3337 return kFALSE;
3338 }
27734f0e 3339 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3340 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3341 // Set proof special parameters if any
3342 TIter nextpp(&fProofParam);
3343 TObject *proofparam;
3344 while ((proofparam=nextpp())) {
3345 TString svalue = GetProofParameter(proofparam->GetName());
3346 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3347 }
3bdcb562 3348 // Is dataset existing ?
3349 if (!testMode) {
3350 TString dataset = fProofDataSet;
3351 Int_t index = dataset.Index("#");
3352 if (index>=0) dataset.Remove(index);
830acc4c 3353// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3354// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3355// return kFALSE;
3356// }
3357// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3358 }
3359 // Is ClearPackages() needed ?
3360 if (TestSpecialBit(kClearPackages)) {
3361 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3362 gROOT->ProcessLine("gProof->ClearPackages();");
3363 }
3364 // Is a given aliroot mode requested ?
3365 TList optionsList;
1ac846e9 3366 TString parLibs;
830acc4c 3367 if (!fAliRootMode.IsNull()) {
3bdcb562 3368 TString alirootMode = fAliRootMode;
3369 if (alirootMode == "default") alirootMode = "";
3370 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3371 optionsList.SetOwner();
3372 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3373 // Check the additional libs to be loaded
3374 TString extraLibs;
1ac846e9 3375 Bool_t parMode = kFALSE;
dc1ffbcb 3376 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3377 // Parse the extra libs for .so
3378 if (fAdditionalLibs.Length()) {
a41b2882 3379 TString additionalLibs = fAdditionalLibs;
3380 additionalLibs.Strip();
3381 if (additionalLibs.Length() && fFriendLibs.Length())
3382 additionalLibs += " ";
3383 additionalLibs += fFriendLibs;
3384 TObjArray *list = additionalLibs.Tokenize(" ");
3bdcb562 3385 TIter next(list);
3386 TObjString *str;
1ac846e9 3387 while((str=(TObjString*)next())) {
3388 if (str->GetString().Contains(".so")) {
3389 if (parMode) {
3390 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3391 break;
3392 }
3393 TString stmp = str->GetName();
3394 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3395 stmp.ReplaceAll(".so","");
3396 if (!extraLibs.IsNull()) extraLibs += ":";
3397 extraLibs += stmp;
3398 continue;
3399 }
3400 if (str->GetString().Contains(".par")) {
3401 // The first par file found in the list will not allow any further .so
3402 parMode = kTRUE;
3403 if (!parLibs.IsNull()) parLibs += ":";
3404 parLibs += str->GetName();
3405 continue;
3406 }
3bdcb562 3407 }
3408 if (list) delete list;
3409 }
a41b2882 3410 if (!extraLibs.IsNull()) {
3411 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3412 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3413 }
830acc4c 3414 // Check extra includes
3415 if (!fIncludePath.IsNull()) {
3416 TString includePath = fIncludePath;
3417 includePath.ReplaceAll(" ",":");
dc030a24 3418 includePath.ReplaceAll("$ALICE_ROOT/","");
3419 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3420 includePath.ReplaceAll("-I","");
dc030a24 3421 includePath.Remove(TString::kTrailing, ':');
830acc4c 3422 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3423 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3424 }
27734f0e 3425 // Check if connection to grid is requested
3426 if (TestSpecialBit(kProofConnectGrid))
3427 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3428 // Enable AliRoot par
3429 if (testMode) {
3430 // Enable proof lite package
3431 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3432 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3433 TNamed *obj = (TNamed*)optionsList.At(i);
3434 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3435 }
3436 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3437 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3438 Info("StartAnalysis", "AliRootProofLite enabled");
3439 } else {
3440 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3441 return kFALSE;
3442 }
3443 } else {
dc030a24 3444 if ( ! fAliROOTVersion.IsNull() ) {
3445 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3446 fAliROOTVersion.Data(), &optionsList))) {
3447 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3448 return kFALSE;
3449 }
3450 }
27734f0e 3451 }
1ac846e9 3452 // Enable first par files from fAdditionalLibs
3453 if (!parLibs.IsNull()) {
3454 TObjArray *list = parLibs.Tokenize(":");
3455 TIter next(list);
3456 TObjString *package;
3457 while((package=(TObjString*)next())) {
40501bb7 3458 TString spkg = package->GetName();
3459 spkg.ReplaceAll(".par", "");
3460 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3461 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3462 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3463 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3464 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3465 return kFALSE;
3466 }
3467 } else {
3468 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3469 return kFALSE;
3470 }
3471 }
3472 if (list) delete list;
3473 }
3bdcb562 3474 } else {
3475 if (fAdditionalLibs.Contains(".so") && !testMode) {
3476 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3477 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3478 return kFALSE;
3479 }
3480 }
3481 // Enable par files if requested
3482 if (fPackages && fPackages->GetEntries()) {
3483 TIter next(fPackages);
3484 TObject *package;
3485 while ((package=next())) {
1ac846e9 3486 // Skip packages already enabled
3487 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3488 TString spkg = package->GetName();
3489 spkg.ReplaceAll(".par", "");
3490 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3491 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3492 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3493 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3494 return kFALSE;
3495 }
3496 } else {
3497 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3498 return kFALSE;
3499 }
3500 }
3501 }
3502 // Do we need to load analysis source files ?
3503 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3504 if (fAnalysisSource.Length()) {
3505 TObjArray *list = fAnalysisSource.Tokenize(" ");
3506 TIter next(list);
3507 TObjString *str;
3508 while((str=(TObjString*)next())) {
3509 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3510 }
3bdcb562 3511 if (list) delete list;
3512 }
830acc4c 3513 if (testMode) {
3514 // Register dataset to proof lite.
3515 if (fFileForTestMode.IsNull()) {
3516 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3517 return kFALSE;
3518 }
3519 if (gSystem->AccessPathName(fFileForTestMode)) {
3520 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3521 return kFALSE;
3522 }
3523 TFileCollection *coll = new TFileCollection();
3524 coll->AddFromFile(fFileForTestMode);
8ddad121 3525 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3526 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3527 }
3bdcb562 3528 return kTRUE;
3529 }
c57f56b7 3530
43da816a 3531 // Check if output files have to be taken from the analysis manager
3532 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3533 // Add output files and AOD files
3534 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3535 // Add extra files registered to the analysis manager
f790bc1b 3536 TString extra = GetListOfFiles("ext");
3537 if (!extra.IsNull()) {
149d288c 3538 extra.ReplaceAll(".root", "*.root");
f790bc1b 3539 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3540 fOutputFiles += extra;
84fcd93f 3541 }
149d288c 3542 // Compose the output archive.
b83f84f0 3543 fOutputArchive = "log_archive.zip:std*@disk=1 ";
ec5acdce 3544 if (mgr->IsCollectThroughput())
95e16f01 3545 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",fOutputFiles.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 3546 else
3547 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3548 }
f7b1cbc2 3549// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3550 if (TestBit(AliAnalysisGrid::kOffline)) {
3551 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3552 \n there nor any job run. You can revise the JDL and analysis \
3553 \n macro then run the same in \"submit\" mode.");
3554 } else if (TestBit(AliAnalysisGrid::kTest)) {
3555 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3556 \n dataset.");
3557 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3558 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3559 \n space and job submitted.");
3560 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3561 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3562 if (fMergeViaJDL) CheckInputData();
5513444a 3563 return kTRUE;
c57f56b7 3564 } else {
3565 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3566 }
3567
348be253 3568 Print();
c57f56b7 3569 if (!Connect()) {
3570 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3571 return kFALSE;
16a4353c 3572 }
d3b18c4c 3573 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3574 if (!CheckInputData()) {
3575 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3576 return kFALSE;
c57f56b7 3577 }
d3339be3 3578 if (!CreateDataset(fDataPattern)) {
3579 TString serror;
3580 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3581 if (fRunNumbers.Length()) serror = "run numbers";
3582 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3583 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3584 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3585 return kFALSE;
3586 }
21ca8e59 3587 WriteAnalysisFile();
c57f56b7 3588 WriteAnalysisMacro();
3589 WriteExecutable();
3590 WriteValidationScript();
0f389141 3591 if (fMergeViaJDL) {
3592 WriteMergingMacro();
3593 WriteMergeExecutable();
3594 WriteValidationScript(kTRUE);
3595 }
5513444a 3596 if (!CreateJDL()) return kFALSE;
3597 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3598 if (testMode) {
c57f56b7 3599 // Locally testing the analysis
3600 Info("StartAnalysis", "\n_______________________________________________________________________ \
3601 \n Running analysis script in a daughter shell as on a worker node \
3602 \n_______________________________________________________________________");
149d288c 3603 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3604 TIter next(list);
3605 TObjString *str;
a2f5fc01 3606 TString outputFile;
c57f56b7 3607 while((str=(TObjString*)next())) {
a2f5fc01 3608 outputFile = str->GetString();
3609 Int_t index = outputFile.Index("@");
3610 if (index > 0) outputFile.Remove(index);
3611 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3612 }
3613 delete list;
3614 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3615 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3616// gSystem->Exec("cat stdout");
5513444a 3617 return kFALSE;
c57f56b7 3618 }
5513444a 3619 // Check if submitting is managed by LPM manager
a3e84053 3620 if (fProductionMode) {
7ac61826 3621 //TString prodfile = fJDLName;
3622 //prodfile.ReplaceAll(".jdl", ".prod");
3623 //WriteProductionFile(prodfile);
5513444a 3624 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3625 return kFALSE;
3626 }
a8739e8a 3627 // Submit AliEn job(s)
d2a409b2 3628 gGrid->Cd(fGridOutputDir);
a8739e8a 3629 TGridResult *res;
c57f56b7 3630 TString jobID = "";
0ad3ea1f 3631 fGridJobIDs = "";
3632 fGridStages = "";
d2a409b2 3633 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3634 // Submit a given xml or a set of runs
a8739e8a 3635 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3636 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3637 if (res) {
3638 const char *cjobId = res->GetKey(0,"jobId");
3639 if (!cjobId) {
a03be957 3640 gGrid->Stdout();
3641 gGrid->Stderr();
a8739e8a 3642 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3643 return kFALSE;
a8739e8a 3644 } else {
3645 Info("StartAnalysis", "\n_______________________________________________________________________ \
3646 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3647 \n_______________________________________________________________________",
3648 fJDLName.Data(), cjobId);
3649 jobID = cjobId;
0ad3ea1f 3650 if (jobID.Atoi()) {
3651 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3652 fGridJobIDs.Append(jobID);
3653 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3654 fGridStages.Append("full");
3655 }
a8739e8a 3656 }
3657 delete res;
a03be957 3658 } else {
3659 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3660 return kFALSE;
a8739e8a 3661 }
3662 } else {
d2a409b2 3663 // Submit for a range of enumeration of runs.
a03be957 3664 if (!Submit()) return kFALSE;
0ad3ea1f 3665 jobID = fGridJobIDs;
c57f56b7 3666 }
a8739e8a 3667
d88e4159 3668 if (fDropToShell) {
3669 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3670 \n You may exit at any time and terminate the job later using the option <terminate> \
3671 \n ##################################################################################", jobID.Data());
3672 gSystem->Exec("aliensh");
3673 } else {
3674 Info("StartAnalysis", "\n#### SUBMITTED JOB %s TO ALIEN QUEUE #### \
3675 \n Remember to terminate the job later using the option <terminate> \
3676 \n ##################################################################################", jobID.Data());
3677 }
5513444a 3678 return kTRUE;
c57f56b7 3679}
3680
f790bc1b 3681//______________________________________________________________________________
3682const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3683{
3684// Get a comma-separated list of output files of the requested type.
3685// Type can be (case unsensitive):
3686// aod - list of aod files (std, extensions and filters)
3687// out - list of output files connected to containers (but not aod's or extras)
3688// ext - list of extra files registered to the manager
3689// ter - list of files produced in terminate
3690 static TString files;
3691 files = "";
3692 TString stype = type;
3693 stype.ToLower();
3694 TString aodfiles, extra;
3695 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3696 if (!mgr) {
3697 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3698 return files.Data();
3699 }
3700 if (mgr->GetOutputEventHandler()) {
3701 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3702 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3703 if (!extraaod.IsNull()) {
3704 aodfiles += ",";
3705 aodfiles += extraaod;
3706 }
3707 }
3708 if (stype.Contains("aod")) {
3709 files = aodfiles;
3710 if (stype == "aod") return files.Data();
3711 }
3712 // Add output files that are not in the list of AOD files
3713 TString outputfiles = "";
3714 TIter next(mgr->GetOutputs());
3715 AliAnalysisDataContainer *output;
3716 const char *filename = 0;
3717 while ((output=(AliAnalysisDataContainer*)next())) {
3718 filename = output->GetFileName();
3719 if (!(strcmp(filename, "default"))) continue;
3720 if (outputfiles.Contains(filename)) continue;
3721 if (aodfiles.Contains(filename)) continue;
3722 if (!outputfiles.IsNull()) outputfiles += ",";
3723 outputfiles += filename;
3724 }
3725 if (stype.Contains("out")) {
3726 if (!files.IsNull()) files += ",";
3727 files += outputfiles;
3728 if (stype == "out") return files.Data();
3729 }
3730 // Add extra files registered to the analysis manager
3731 TString sextra;
3732 extra = mgr->GetExtraFiles();
3733 if (!extra.IsNull()) {
3734 extra.Strip();
3735 extra.ReplaceAll(" ", ",");
3736 TObjArray *fextra = extra.Tokenize(",");
3737 TIter nextx(fextra);
3738 TObject *obj;
3739 while ((obj=nextx())) {
3740 if (aodfiles.Contains(obj->GetName())) continue;
3741 if (outputfiles.Contains(obj->GetName())) continue;
3742 if (sextra.Contains(obj->GetName())) continue;
3743 if (!sextra.IsNull()) sextra += ",";
3744 sextra += obj->GetName();
3745 }
3746 delete fextra;
3747 if (stype.Contains("ext")) {
3748 if (!files.IsNull()) files += ",";
3749 files += sextra;
3750 }
3751 }
3752 if (stype == "ext") return files.Data();
3753 TString termfiles;
3754 if (!fTerminateFiles.IsNull()) {
3755 fTerminateFiles.Strip();
3756 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3757 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3758 TIter nextx(fextra);
3759 TObject *obj;
3760 while ((obj=nextx())) {
3761 if (aodfiles.Contains(obj->GetName())) continue;
3762 if (outputfiles.Contains(obj->GetName())) continue;
3763 if (termfiles.Contains(obj->GetName())) continue;
3764 if (sextra.Contains(obj->GetName())) continue;
3765 if (!termfiles.IsNull()) termfiles += ",";
3766 termfiles += obj->GetName();
3767 }
3768 delete fextra;
3769 }
3770 if (stype.Contains("ter")) {
3771 if (!files.IsNull() && !termfiles.IsNull()) {
3772 files += ",";
3773 files += termfiles;
3774 }
3775 }
3776 return files.Data();
3777}
3778
d2a409b2 3779//______________________________________________________________________________
a03be957 3780Bool_t AliAnalysisAlien::Submit()
d2a409b2 3781{
3782// Submit all master jobs.
3783 Int_t nmasterjobs = fInputFiles->GetEntries();
3784 Long_t tshoot = gSystem->Now();
a03be957 3785 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3786 while (fNsubmitted < nmasterjobs) {
3787 Long_t now = gSystem->Now();
3788 if ((now-tshoot)>30000) {
3789 tshoot = now;
a03be957 3790 if (!SubmitNext()) return kFALSE;
d2a409b2 3791 }
3792 }
a03be957 3793 return kTRUE;
d2a409b2 3794}
3795
0f389141 3796//______________________________________________________________________________
a03be957 3797Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3798{
3799// Submit all merging jobs.
dc030a24 3800 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3801 gGrid->Cd(fGridOutputDir);
3802 TString mergeJDLName = fExecutable;
3803 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3804 if (!fInputFiles) {
3805 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3806 return kFALSE;
3807 }
0f389141 3808 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3809 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3810 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3811 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3812 if (fOutputToRunNo) {
3813 // The output directory is the run number
3814 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3815 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3816 } else {
17b4e9a4 3817 if (!fRunNumbers.Length() && !fRunRange[0]) {
3818 // The output directory is the grid outdir
3819 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3820 runOutDir = fGridOutputDir;
3821 } else {
3822 // The output directory is the master number in 3 digits format
3823 printf("### Submitting merging job for master <%03d>\n", i);
3824 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3825 }
7c2cd90a 3826 }
3827 // Check now the number of merging stages.
2b222f7c 3828 TObjArray *list = fOutputFiles.Tokenize(",");
3829 TIter next(list);
3830 TObjString *str;
3831 TString outputFile;
3832 while((str=(TObjString*)next())) {
3833 outputFile = str->GetString();
3834 Int_t index = outputFile.Index("@");
3835 if (index > 0) outputFile.Remove(index);
37d21c01 3836 if (!fMergeExcludes.Contains(outputFile) &&
3837 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3838 }
3839 delete list;
b3e07543 3840 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3841 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3842 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3843 }
a03be957 3844 if (!ntosubmit) return kTRUE;
d88e4159 3845 if (fDropToShell) {
3846 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3847 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3848 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3849 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3850 \n ################################################################################################################");
3851 gSystem->Exec("aliensh");
3852 } else {
3853 Info("StartAnalysis", "\n #### STARTED MERGING JOBS FOR YOU #### \
3854 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL') \
3855 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3856 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3857 \n ################################################################################################################");
3858 }
a03be957 3859 return kTRUE;
0f389141 3860}
3861
d2a409b2 3862//______________________________________________________________________________
a03be957 3863Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3864{
08d5b699 3865// Submit next bunch of master jobs if the queue is free. The first master job is
3866// submitted right away, while the next will not be unless the previous was split.
3867// The plugin will not submit new master jobs if there are more that 500 jobs in
3868// waiting phase.
d2a409b2 3869 static Bool_t iscalled = kFALSE;
3870 static Int_t firstmaster = 0;
3871 static Int_t lastmaster = 0;
3872 static Int_t npermaster = 0;
a03be957 3873 if (iscalled) return kTRUE;
d2a409b2 3874 iscalled = kTRUE;
3875 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3876 Int_t ntosubmit = 0;
3877 TGridResult *res;
3878 TString jobID = "";
3bdcb562 3879 Int_t nmasterjobs = fInputFiles->GetEntries();
3880 if (!fNsubmitted) {
3881 ntosubmit = 1;
3882 if (!IsUseSubmitPolicy()) {
90d50a8c 3883 if (nmasterjobs>5)
3bdcb562 3884 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3885 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3886 ntosubmit = nmasterjobs;
3887 }
3888 } else {
d2a409b2 3889 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3890 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3891 // If last master not split, just return
a03be957 3892 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3893 // No more than 100 waiting jobs
08d5b699 3894 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3895 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3896 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3897 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3898 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3899 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3900 }
d2a409b2 3901 for (Int_t i=0; i<ntosubmit; i++) {
3902 // Submit for a range of enumeration of runs.
a03be957 3903 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3904 TString query;
cd11251e 3905 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3906 runOutDir.ReplaceAll(".xml", "");
3907 if (fOutputToRunNo)
3908 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3909 else
3910 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3911 printf("********* %s\n",query.Data());
d2a409b2 3912 res = gGrid->Command(query);
3913 if (res) {
98ca124f 3914 TString cjobId1 = res->GetKey(0,"jobId");
3915 if (!cjobId1.Length()) {
d2a409b2 3916 iscalled = kFALSE;
a03be957 3917 gGrid->Stdout();
3918 gGrid->Stderr();
3919 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3920 return kFALSE;
d2a409b2 3921 } else {
3922 Info("StartAnalysis", "\n_______________________________________________________________________ \
3923 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3924 \n_______________________________________________________________________",
98ca124f 3925 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
0ad3ea1f 3926 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3927 fGridJobIDs.Append(cjobId1);
3928 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3929 fGridStages.Append("full");
d2a409b2 3930 jobID += cjobId1;
3931 jobID += " ";
98ca124f 3932 lastmaster = cjobId1.Atoi();
d2a409b2 3933 if (!firstmaster) firstmaster = lastmaster;
3934 fNsubmitted++;
3935 }
3936 delete res;
a03be957 3937 } else {
3938 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3939 return kFALSE;
d2a409b2 3940 }
3941 }
3942 iscalled = kFALSE;
a03be957 3943 return kTRUE;
d2a409b2 3944}
3945
c57f56b7 3946//______________________________________________________________________________
3947void AliAnalysisAlien::WriteAnalysisFile()
3948{
f10e8481 3949// Write current analysis manager into the file <analysisFile>
3950 TString analysisFile = fExecutable;
3951 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3952 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3953 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3954 if (!mgr || !mgr->IsInitialized()) {
3955 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3956 return;
3957 }
3958 // Check analysis type
3959 TObject *handler;
3960 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3961 handler = (TObject*)mgr->GetInputEventHandler();
3962 if (handler) {
65eb22f8 3963 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3964 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3965 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3966 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3967 } else {
3968 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3969 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3970 }
c57f56b7 3971 }
3972 TDirectory *cdir = gDirectory;
f10e8481 3973 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3974 if (file) {
e16a394c 3975 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3976 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3977 // Unless merging makes no sense
3978 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3979 mgr->Write();
3980 delete file;
fe2d7fc2 3981 // Enable termination for local jobs
3982 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3983 }
3984 if (cdir) cdir->cd();
f10e8481 3985 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3986 }
3987 Bool_t copy = kTRUE;
d3b18c4c 3988 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3989 if (copy) {
3990 CdWork();
3991 TString workdir = gGrid->GetHomeDirectory();
3992 workdir += fGridWorkingDir;
d3b18c4c 3993 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3994 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3995 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3996 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3997 }
3998}
3999
4000//______________________________________________________________________________
4001void AliAnalysisAlien::WriteAnalysisMacro()
4002{
4003// Write the analysis macro that will steer the analysis in grid mode.
4004 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4005 ofstream out;
4006 out.open(fAnalysisMacro.Data(), ios::out);
4007 if (!out.good()) {
4008 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4009 return;
4010 }
5b273635 4011 Bool_t hasSTEERBase = kFALSE;
4012 Bool_t hasESD = kFALSE;
4013 Bool_t hasAOD = kFALSE;
4014 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4015 Bool_t hasOADB = kFALSE;
5b273635 4016 Bool_t hasANALYSISalice = kFALSE;
4017 Bool_t hasCORRFW = kFALSE;
c57f56b7 4018 TString func = fAnalysisMacro;
4019 TString type = "ESD";
4020 TString comment = "// Analysis using ";
715da79c 4021 if (fMCLoop) {
4022 type = "MCGEN";
4023 comment += "MCGEN";
4024 } else {
4025 if (IsUseMCchain()) {
4026 type = "MC";
4027 comment += "MC";
4028 } else {
4029 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
4030 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
4031 type = "AOD";
4032 comment += "AOD";
4033 }
4034 }
4035 }
0df6ccf2 4036 if (type!="AOD" && fFriendChainName!="") {
4037 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
4038 return;
4039 }
c57f56b7 4040 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
4041 else comment += " data";
4042 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
4043 func.ReplaceAll(".C", "");
4044 out << "void " << func.Data() << "()" << endl;
4045 out << "{" << endl;
4046 out << comment.Data() << endl;
4047 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 4048 out << " TStopwatch timer;" << endl;
4049 out << " timer.Start();" << endl << endl;
13f0c00b 4050 // Change temp directory to current one
fdee57f1 4051 if (!IsLocalTest()) {
4052 out << "// connect to AliEn and make the chain" << endl;
4053 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
4054 }
13f0c00b 4055 out << "// Set temporary merging directory to current one" << endl;
4056 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4057 out << "// Set temporary compilation directory to current one" << endl;
4058 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 4059 // Reset existing include path
4060 out << "// Reset existing include path and add current directory first in the search" << endl;
4061 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 4062 if (!fExecutableCommand.Contains("aliroot")) {
4063 out << "// load base root libraries" << endl;
4064 out << " gSystem->Load(\"libTree\");" << endl;
4065 out << " gSystem->Load(\"libGeom\");" << endl;
4066 out << " gSystem->Load(\"libVMC\");" << endl;
4067 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4068 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4069 }
d5c6455a 4070 if (fAdditionalRootLibs.Length()) {
47a4137d 4071 // in principle libtree /lib geom libvmc etc. can go into this list, too
4072 out << "// Add aditional libraries" << endl;
4073 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4074 TIter next(list);
4075 TObjString *str;
4076 while((str=(TObjString*)next())) {
4077 if (str->GetString().Contains(".so"))
4078 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 4079 }
47a4137d 4080 if (list) delete list;
d5c6455a 4081 }
57377eb5 4082 out << "// Load analysis framework libraries" << endl;
652561d5 4083 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 4084 if (!fPackages) {
f790bc1b 4085 if (!fExecutableCommand.Contains("aliroot")) {
4086 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4087 out << " gSystem->Load(\"libESD\");" << endl;
4088 out << " gSystem->Load(\"libAOD\");" << endl;
4089 }
4e5c5506 4090 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4091 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 4092 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4093 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 4094 } else {
4e5c5506 4095 TIter next(fPackages);
4096 TObject *obj;
57377eb5 4097 TString pkgname;
57377eb5 4098 while ((obj=next())) {
4099 pkgname = obj->GetName();
4478e6f1 4100 if (pkgname == "STEERBase" ||
4101 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4102 if (pkgname == "ESD" ||
4103 pkgname == "ESD.par") hasESD = kTRUE;
4104 if (pkgname == "AOD" ||
4105 pkgname == "AOD.par") hasAOD = kTRUE;
4106 if (pkgname == "ANALYSIS" ||
4107 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4108 if (pkgname == "OADB" ||
4109 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 4110 if (pkgname == "ANALYSISalice" ||
4111 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4112 if (pkgname == "CORRFW" ||
4113 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 4114 }
4115 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 4116 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4117 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 4118 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4119 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 4120 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4121 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 4122 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4123 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4124 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4125 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 4126 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4127 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 4128 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4129 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 4130 out << "// Compile other par packages" << endl;
4131 next.Reset();
fcc9bb6f 4132 while ((obj=next())) {
4133 pkgname = obj->GetName();
4478e6f1 4134 if (pkgname == "STEERBase" ||
4135 pkgname == "STEERBase.par" ||
4136 pkgname == "ESD" ||
4137 pkgname == "ESD.par" ||
4138 pkgname == "AOD" ||
4139 pkgname == "AOD.par" ||
4140 pkgname == "ANALYSIS" ||
4141 pkgname == "ANALYSIS.par" ||
7ea40ac8 4142 pkgname == "OADB" ||
4143 pkgname == "OADB.par" ||
4478e6f1 4144 pkgname == "ANALYSISalice" ||
4145 pkgname == "ANALYSISalice.par" ||
4146 pkgname == "CORRFW" ||
4147 pkgname == "CORRFW.par") continue;
5b273635 4148 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 4149 }
4e5c5506 4150 }
b3e07543 4151 out << "// include path" << endl;
4152 // Get the include path from the interpreter and remove entries pointing to AliRoot
4153 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4154 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4155 out << " TIter nextpath(listpaths);" << endl;
4156 out << " TObjString *pname;" << endl;
4157 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4158 out << " TString current = pname->GetName();" << endl;
4159 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4160 out << " gSystem->AddIncludePath(current);" << endl;
4161 out << " }" << endl;
4162 out << " if (listpaths) delete listpaths;" << endl;
4163 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4164 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4165 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
715da79c 4166 if (fMCLoop && !fGeneratorLibs.IsNull()) {
4167 out << "// MC generator libraries" << endl;
4168 TObjArray *list = fGeneratorLibs.Tokenize(" ");
4169 TIter next(list);
4170 TObjString *str;
4171 while((str=(TObjString*)next())) {
4172 out << " gSystem->Load(\"" << str->GetName() << "\");" << endl;
4173 }
4174 delete list;
4175 }
6da75e0b 4176 if (fAdditionalLibs.Length()) {
4177 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4178 TString additionalLibs = fAdditionalLibs;
4179 additionalLibs.Strip();
4180 if (additionalLibs.Length() && fFriendLibs.Length())
4181 additionalLibs += " ";
4182 additionalLibs += fFriendLibs;
4183 TObjArray *list = additionalLibs.Tokenize(" ");
6da75e0b 4184 TIter next(list);
4185 TObjString *str;
4186 while((str=(TObjString*)next())) {
4187 if (str->GetString().Contains(".so"))
4188 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 4189 if (str->GetString().Contains(".par"))
4190 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 4191 }
715da79c 4192 delete list;
6da75e0b 4193 }
4194 out << endl;
c57f56b7 4195 out << "// analysis source to be compiled at runtime (if any)" << endl;
4196 if (fAnalysisSource.Length()) {
4197 TObjArray *list = fAnalysisSource.Tokenize(" ");
4198 TIter next(list);
4199 TObjString *str;
4200 while((str=(TObjString*)next())) {
4201 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4202 }
4203 if (list) delete list;
4204 }
4205 out << endl;
b3e07543 4206// out << " printf(\"Currently load libraries:\\n\");" << endl;
4207// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 4208 if (fFastReadOption) {
d3339be3 4209 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
4210 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 4211 out << "// fast xrootd reading enabled" << endl;
4212 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4213 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4214 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4215 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4216 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4217 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 4218 }
c57f56b7 4219 out << "// read the analysis manager from file" << endl;
f10e8481 4220 TString analysisFile = fExecutable;
4221 analysisFile.ReplaceAll(".sh", ".root");
105631b5 4222 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4223 << analysisFile << "\");" << endl;
4224 out << " if (!mgr) return;" << endl;
21ca8e59 4225 if (IsLocalTest()) {
4226 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
4227 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 4228 if (fFileForTestMode.IsNull())
4229 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
4230 else
4231 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 4232 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4b5d9e21 4233 if (!fFriendChainName.IsNull())
b9f1efa5 4234 out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\",\"" << fFriendLibs << "\");" << endl;
024dfa52 4235 if (IsUseMCchain())
4236 out << " plugin->SetUseMCchain();" << endl;
715da79c 4237 if (fMCLoop)
4238 out << " plugin->SetMCLoop(kTRUE);" << endl;
21ca8e59 4239 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 4240 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 4241 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 4242 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
4243 } else {
7a366e0b 4244 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 4245 out << " mgr->SetNSysInfo(100);" << endl;
4246 }
21ca8e59 4247 }
c57f56b7 4248 out << " mgr->PrintStatus();" << endl;
52b6a92b 4249 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 4250 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 4251 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 4252 } else {
9a8288ff 4253 if (TestBit(AliAnalysisGrid::kTest))
4254 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4255 else
4256 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 4257 }
4258 }
21ca8e59 4259 if (!IsLocalTest()) {
715da79c 4260 if (fMCLoop) {
4261 out << " mgr->SetCacheSize(0);" << endl;
4262 out << " mgr->EventLoop(" << fNMCevents << ");" << endl;
4263 } else {
4264 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
4265 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
4266 }
21ca8e59 4267 } else {
715da79c 4268 if (fMCLoop) {
4269 out << " mgr->SetCacheSize(0);" << endl;
4270 out << " mgr->EventLoop(" << fNMCevents << ");" << endl;
7435f77f
AG		 4271 } else {
4272 out << " mgr->StartAnalysis(\"localfile\");" << endl;
4273 }
242accb2 4274 }
f7498086 4275 out << " timer.Stop();" << endl;
4276 out << " timer.Print();" << endl;
c57f56b7 4277 out << "}" << endl << endl;
715da79c 4278 if (!IsLocalTest() && !fMCLoop) {
fcc9bb6f 4279 out <<"//________________________________________________________________________________" << endl;
c57f56b7 4280 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
4281 out << "{" << endl;
4282 out << "// Create a chain using url's from xml file" << endl;
242accb2 4283 out << " TString filename;" << endl;
4284 out << " Int_t run = 0;" << endl;
c85cfc0f 4285 if (IsUseMCchain()) {
4286 out << " TString treename = \"TE\";" << endl;
4287 } else {
780ec36d 4288 if (!fTreeName.IsNull()) {
4289 out << " TString treename = \"" << fTreeName << "\";" << endl;
4290 } else {
4291 out << " TString treename = type;" << endl;
4292 out << " treename.ToLower();" << endl;
4293 out << " treename += \"Tree\";" << endl;
4294 }
c85cfc0f 4295 }
e02fee64 4296 out << " printf(\"***************************************\\n\");" << endl;
4297 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
4298 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 4299 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
4300 out << " if (!coll) {" << endl;
4301 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
4302 out << " return NULL;" << endl;
4303 out << " }" << endl;
242accb2 4304 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 4305 out << " TChain *chain = new TChain(treename);" << endl;
06f67114 4306 if(!fFriendChainName.IsNull()) {
4307 out << " TList *friends = new TList();" << endl;
4308 out << " TIter nextfriend(friends);" << endl;
4309 out << " TChain *cfriend = 0;" << endl;
4310 TObjArray *list = fFriendChainName.Tokenize(" ");
4311 TIter next(list);
4312 TObjString *str;
4313 while((str=(TObjString*)next())) {
4314 out << " cfriend = new TChain(treename, \"" << str->GetName() << "\");" << endl;
4315 out << " friends->Add(cfriend);" << endl;
4316 out << " chain->AddFriend(cfriend);" << endl;
4317 }
4318 delete list;
4319// out << " TChain *chainFriend = new TChain(treename);" << endl;
0df6ccf2 4320 }
c57f56b7 4321 out << " coll->Reset();" << endl;
0df6ccf2 4322 out << " while (coll->Next()) {" << endl;
242accb2 4323 out << " filename = coll->GetTURL("");" << endl;
4324 out << " if (mgr) {" << endl;
4325 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
4326 out << " if (nrun && nrun != run) {" << endl;
4327 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
4328 out << " mgr->SetRunFromPath(nrun);" << endl;
4329 out << " run = nrun;" << endl;
4330 out << " }" << endl;
4331 out << " }" << endl;
4332 out << " chain->Add(filename);" << endl;
06f67114 4333 if(!fFriendChainName.IsNull()) {
4334 out << " TString bpath=coll->GetTURL(\"\");" << endl;
4335 out << " if (bpath.Index(\"#\") > -1) bpath.Remove(bpath.Index(\"#\"));" << endl;
4336 out << " bpath = gSystem->DirName(bpath);" << endl;
4337 out << " bpath += \"/\";" << endl;
4338 out << " TString fileFriend;" << endl;
4339 out << " nextfriend.Reset();" << endl;
4340 out << " while ((cfriend=(TChain*)nextfriend())) {" << endl;
4341 out << " fileFriend = bpath;" << endl;
4342 out << " fileFriend += cfriend->GetTitle();" << endl;
4343 out << " TFile *file = TFile::Open(fileFriend);" << endl;
4344 out << " if (file) {" << endl;
4345 out << " file->Close();" << endl;
4346 out << " cfriend->Add(fileFriend.Data());" << endl;
4347 out << " } else {" << endl;
4348 out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
4349 out << " return 0;" << endl;
4350 out << " }" << endl;
4b5d9e21 4351 out << " }" << endl;
0df6ccf2 4352 }
4353 out << " }" << endl;
c57f56b7 4354 out << " if (!chain->GetNtrees()) {" << endl;
4355 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
4356 out << " return NULL;" << endl;
4357 out << " }" << endl;
4358 out << " return chain;" << endl;
fcc9bb6f 4359 out << "}" << endl << endl;
c57f56b7 4360 }
5b273635 4361 if (hasANALYSISalice) {
4362 out <<"//________________________________________________________________________________" << endl;
4363 out << "Bool_t SetupPar(const char *package) {" << endl;
4364 out << "// Compile the package and set it up." << endl;
4365 out << " TString pkgdir = package;" << endl;
4366 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4367 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4368 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4369 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4370 out << " // Check for BUILD.sh and execute" << endl;
4371 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4372 out << " printf(\"*******************************\\n\");" << endl;
4373 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4374 out << " printf(\"*******************************\\n\");" << endl;
4375 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4376 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4377 out << " gSystem->ChangeDirectory(cdir);" << endl;
4378 out << " return kFALSE;" << endl;
4379 out << " }" << endl;
4380 out << " } else {" << endl;
4381 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4382 out << " gSystem->ChangeDirectory(cdir);" << endl;
4383 out << " return kFALSE;" << endl;
4384 out << " }" << endl;
4385 out << " // Check for SETUP.C and execute" << endl;
4386 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4387 out << " printf(\"*******************************\\n\");" << endl;
4388 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4389 out << " printf(\"*******************************\\n\");" << endl;
4390 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4391 out << " } else {" << endl;
4392 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4393 out << " gSystem->ChangeDirectory(cdir);" << endl;
4394 out << " return kFALSE;" << endl;
4395 out << " }" << endl;
4396 out << " // Restore original workdir" << endl;
4397 out << " gSystem->ChangeDirectory(cdir);" << endl;
4398 out << " return kTRUE;" << endl;
4399 out << "}" << endl;
4400 }
c57f56b7 4401 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4402 }
4403 Bool_t copy = kTRUE;
d3b18c4c 4404 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4405 if (copy) {
4406 CdWork();
4407 TString workdir = gGrid->GetHomeDirectory();
4408 workdir += fGridWorkingDir;
4409 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4410 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4411// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4412 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4413 Form("alien://%s/%s", workdir.Data(),
4414 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4415 }
4416}
4417
0f389141 4418//______________________________________________________________________________
4419void AliAnalysisAlien::WriteMergingMacro()
4420{
4421// Write a macro to merge the outputs per master job.
4422 if (!fMergeViaJDL) return;
4423 if (!fOutputFiles.Length()) {
4424 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4425 return;
4426 }
ec5acdce 4427 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
0f389141 4428 TString mergingMacro = fExecutable;
4429 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4430 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4431 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4432 ofstream out;
4433 out.open(mergingMacro.Data(), ios::out);
4434 if (!out.good()) {
4435 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4436 return;
4437 }
5b273635 4438 Bool_t hasSTEERBase = kFALSE;
4439 Bool_t hasESD = kFALSE;
4440 Bool_t hasAOD = kFALSE;
4441 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4442 Bool_t hasOADB = kFALSE;
5b273635 4443 Bool_t hasANALYSISalice = kFALSE;
4444 Bool_t hasCORRFW = kFALSE;
0f389141 4445 TString func = mergingMacro;
4446 TString comment;
4447 func.ReplaceAll(".C", "");
f47d5cb4 4448 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4449 out << "{" << endl;
4450 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4451 out << " TStopwatch timer;" << endl;
4452 out << " timer.Start();" << endl << endl;
b3e07543 4453 // Reset existing include path
4454 out << "// Reset existing include path and add current directory first in the search" << endl;
4455 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4456 if (!fExecutableCommand.Contains("aliroot")) {
4457 out << "// load base root libraries" << endl;
4458 out << " gSystem->Load(\"libTree\");" << endl;
4459 out << " gSystem->Load(\"libGeom\");" << endl;
4460 out << " gSystem->Load(\"libVMC\");" << endl;
4461 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4462 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4463 }
0f389141 4464 if (fAdditionalRootLibs.Length()) {
4465 // in principle libtree /lib geom libvmc etc. can go into this list, too
4466 out << "// Add aditional libraries" << endl;
4467 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4468 TIter next(list);
4469 TObjString *str;
4470 while((str=(TObjString*)next())) {
4471 if (str->GetString().Contains(".so"))
4472 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4473 }
4474 if (list) delete list;
4475 }
0f389141 4476 out << "// Load analysis framework libraries" << endl;
4477 if (!fPackages) {
7c2cd90a 4478 if (!fExecutableCommand.Contains("aliroot")) {
4479 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4480 out << " gSystem->Load(\"libESD\");" << endl;
4481 out << " gSystem->Load(\"libAOD\");" << endl;
4482 }
0f389141 4483 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4484 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4485 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4486 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4487 } else {
4488 TIter next(fPackages);
4489 TObject *obj;
4490 TString pkgname;
5b273635 4491 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4492 while ((obj=next())) {
4493 pkgname = obj->GetName();
4494 if (pkgname == "STEERBase" ||
4495 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4496 if (pkgname == "ESD" ||
4497 pkgname == "ESD.par") hasESD = kTRUE;
4498 if (pkgname == "AOD" ||
4499 pkgname == "AOD.par") hasAOD = kTRUE;
4500 if (pkgname == "ANALYSIS" ||
4501 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4502 if (pkgname == "OADB" ||
4503 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4504 if (pkgname == "ANALYSISalice" ||
4505 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4506 if (pkgname == "CORRFW" ||
4507 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4508 }
5b273635 4509 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4510 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4511 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4512 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4513 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4514 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4515 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4516 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4517 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4518 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4519 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4520 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4521 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4522 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4523 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4524 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4525 out << "// Compile other par packages" << endl;
4526 next.Reset();
4527 while ((obj=next())) {
4528 pkgname = obj->GetName();
4529 if (pkgname == "STEERBase" ||
4530 pkgname == "STEERBase.par" ||
4531 pkgname == "ESD" ||
4532 pkgname == "ESD.par" ||
4533 pkgname == "AOD" ||
4534 pkgname == "AOD.par" ||
4535 pkgname == "ANALYSIS" ||
4536 pkgname == "ANALYSIS.par" ||
7ea40ac8 4537 pkgname == "OADB" ||
4538 pkgname == "OADB.par" ||
0f389141 4539 pkgname == "ANALYSISalice" ||
4540 pkgname == "ANALYSISalice.par" ||
4541 pkgname == "CORRFW" ||
4542 pkgname == "CORRFW.par") continue;
5b273635 4543 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4544 }
4545 }
b3e07543 4546 out << "// include path" << endl;
4547 // Get the include path from the interpreter and remove entries pointing to AliRoot
4548 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4549 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4550 out << " TIter nextpath(listpaths);" << endl;
4551 out << " TObjString *pname;" << endl;
4552 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4553 out << " TString current = pname->GetName();" << endl;
4554 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4555 out << " gSystem->AddIncludePath(current);" << endl;
4556 out << " }" << endl;
4557 out << " if (listpaths) delete listpaths;" << endl;
4558 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4559 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4560 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0d239a91
AG		 4561 if (fMCLoop && !fGeneratorLibs.IsNull()) {
4562 out << "// MC generator libraries" << endl;
4563 TObjArray *list = fGeneratorLibs.Tokenize(" ");
4564 TIter next(list);
4565 TObjString *str;
4566 while((str=(TObjString*)next())) {
4567 out << " gSystem->Load(\"" << str->GetName() << "\");" << endl;
4568 }
4569 delete list;
4570 }
0f389141 4571 if (fAdditionalLibs.Length()) {
4572 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4573 TString additionalLibs = fAdditionalLibs;
4574 additionalLibs.Strip();
4575 if (additionalLibs.Length() && fFriendLibs.Length())
4576 additionalLibs += " ";
4577 additionalLibs += fFriendLibs;
4578 TObjArray *list = additionalLibs.Tokenize(" ");
0f389141 4579 TIter next(list);
4580 TObjString *str;
4581 while((str=(TObjString*)next())) {
4582 if (str->GetString().Contains(".so"))
4583 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4584 }
4585 if (list) delete list;
4586 }
4587 out << endl;
4588 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4589 if (fAnalysisSource.Length()) {
4590 TObjArray *list = fAnalysisSource.Tokenize(" ");
4591 TIter next(list);
4592 TObjString *str;
4593 while((str=(TObjString*)next())) {
4594 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4595 }
4596 if (list) delete list;
4597 }
149d288c 4598 out << endl;
4599
0f389141 4600 if (fFastReadOption) {
4601 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4602 out << "// fast xrootd reading enabled" << endl;
4603 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4604 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4605 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4606 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4607 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4608 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4609 }
4610 // Change temp directory to current one
fdee57f1 4611 out << "// Connect to AliEn" << endl;
4612 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
e8b839ab 4613 out << "// Set temporary merging directory to current one" << endl;
4614 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4615 out << "// Set temporary compilation directory to current one" << endl;
4616 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
7c2cd90a 4617 out << " TString outputDir = dir;" << endl;
f790bc1b 4618 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4619 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4620 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4621 out << " TIter *iter = new TIter(list);" << endl;
4622 out << " TObjString *str;" << endl;
a2f5fc01 4623 out << " TString outputFile;" << endl;
0f389141 4624 out << " Bool_t merged = kTRUE;" << endl;
ec5acdce 4625 TString analysisFile = fExecutable;
4626 analysisFile.ReplaceAll(".sh", ".root");
4627 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4628 << analysisFile << "\");" << endl;
4629 out << " if (!mgr) {" << endl;
4630 out << " printf(\"ERROR: Analysis manager could not be extracted from file \");" << endl;
4631 out << " return;" << endl;
4632 out << " }" << endl;
d66aa2a9 4633 if (IsLocalTest()) {
4634 out << " printf(\"===================================\n\");" << endl;
4635 out << " printf(\"Testing merging...\\n\");" << endl;
4636 out << " printf(\"===================================\n\");" << endl;
4637 }
0f389141 4638 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4639 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4640 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4641 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4642 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4643 out << " // Skip already merged outputs" << endl;
a2f5fc01 4644 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
d66aa2a9 4645 out << " printf(\"Output file <%s> found. Not merging again.\\n\",outputFile.Data());" << endl;
0f389141 4646 out << " continue;" << endl;
4647 out << " }" << endl;
a2f5fc01 4648 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4649 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4650 out << " if (!merged) {" << endl;
a2f5fc01 4651 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4652 out << " return;" << endl;
0f389141 4653 out << " }" << endl;
4654 out << " }" << endl;
d66aa2a9 4655 if (mgr && mgr->IsCollectThroughput() && !IsLocalTest()) {
ec5acdce 4656 out << " TString infolog = \"" << mgr->GetFileInfoLog() << "\";" << endl;
4657 out << " AliAnalysisAlien::MergeInfo(infolog, outputDir);" << endl;
4658 }
7c2cd90a 4659 out << " // all outputs merged, validate" << endl;
4660 out << " ofstream out;" << endl;
4661 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4662 out << " out.close();" << endl;
4663 out << " // read the analysis manager from file" << endl;
d66aa2a9 4664 if (IsLocalTest()) {
4665 out << " printf(\"===================================\n\");" << endl;
4666 out << " printf(\"Testing Terminate()...\\n\");" << endl;
4667 out << " printf(\"===================================\n\");" << endl;
4668 } else {
4669 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
4670 }
5c4250fc 4671 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4672 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4673 out << " mgr->PrintStatus();" << endl;
ec5acdce 4674 if (mgr) {
4675 if (mgr->GetDebugLevel()>3) {
f866cba5 4676 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4677 } else {
9a8288ff 4678 if (TestBit(AliAnalysisGrid::kTest))
4679 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4680 else
4681 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4682 }
4683 }
9a8288ff 4684 out << " TTree *tree = NULL;" << endl;
4685 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4686 out << "}" << endl << endl;
5b273635 4687 if (hasANALYSISalice) {
4688 out <<"//________________________________________________________________________________" << endl;
4689 out << "Bool_t SetupPar(const char *package) {" << endl;
4690 out << "// Compile the package and set it up." << endl;
4691 out << " TString pkgdir = package;" << endl;
4692 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4693 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4694 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4695 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4696 out << " // Check for BUILD.sh and execute" << endl;
4697 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4698 out << " printf(\"*******************************\\n\");" << endl;
4699 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4700 out << " printf(\"*******************************\\n\");" << endl;
4701 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4702 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4703 out << " gSystem->ChangeDirectory(cdir);" << endl;
4704 out << " return kFALSE;" << endl;
4705 out << " }" << endl;
4706 out << " } else {" << endl;
4707 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4708 out << " gSystem->ChangeDirectory(cdir);" << endl;
4709 out << " return kFALSE;" << endl;
4710 out << " }" << endl;
4711 out << " // Check for SETUP.C and execute" << endl;
4712 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4713 out << " printf(\"*******************************\\n\");" << endl;
4714 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4715 out << " printf(\"*******************************\\n\");" << endl;
4716 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4717 out << " } else {" << endl;
4718 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4719 out << " gSystem->ChangeDirectory(cdir);" << endl;
4720 out << " return kFALSE;" << endl;
4721 out << " }" << endl;
4722 out << " // Restore original workdir" << endl;
4723 out << " gSystem->ChangeDirectory(cdir);" << endl;
4724 out << " return kTRUE;" << endl;
4725 out << "}" << endl;
4726 }
0f389141 4727 }
4728 Bool_t copy = kTRUE;
d3b18c4c 4729 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4730 if (copy) {
4731 CdWork();
4732 TString workdir = gGrid->GetHomeDirectory();
4733 workdir += fGridWorkingDir;
4734 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4735 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4736// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4737 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4738 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4739 }
4740}
4741
4742//______________________________________________________________________________
4743Bool_t AliAnalysisAlien::SetupPar(const char *package)
4744{
205b201f 4745// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4746// Note that for loading the compiled library. The current directory should have precedence in
4747// LD_LIBRARY_PATH
4748 TString pkgdir = package;
4749 pkgdir.ReplaceAll(".par","");
b50a41c3 4750 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4751 TString cdir = gSystem->WorkingDirectory();
4752 gSystem->ChangeDirectory(pkgdir);
4753 // Check for BUILD.sh and execute
4754 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4755 printf("**************************************************\n");
4756 printf("*** Building PAR archive %s\n", package);
4757 printf("**************************************************\n");
4758 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4759 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4760 gSystem->ChangeDirectory(cdir);
4761 return kFALSE;
4762 }
4763 } else {
4764 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4765 gSystem->ChangeDirectory(cdir);
4766 return kFALSE;
4767 }
4768 // Check for SETUP.C and execute
4769 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4770 printf("**************************************************\n");
4771 printf("*** Setup PAR archive %s\n", package);
4772 printf("**************************************************\n");
4773 gROOT->Macro("PROOF-INF/SETUP.C");
4774 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4775 } else {
4776 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4777 gSystem->ChangeDirectory(cdir);
4778 return kFALSE;
4779 }
4780 // Restore original workdir
4781 gSystem->ChangeDirectory(cdir);
4782 return kTRUE;
4783}
4784
c57f56b7 4785//______________________________________________________________________________
4786void AliAnalysisAlien::WriteExecutable()
4787{
4788// Generate the alien executable script.
5aa94dc3 4789 // Patch executable with -x to catch error code
4790 if (fExecutableCommand.Contains("root") &&
4791 fExecutableCommand.Contains("-q") &&
4792 !fExecutableCommand.Contains("-x")) fExecutableCommand += " -x";
c57f56b7 4793 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4794 ofstream out;
4795 out.open(fExecutable.Data(), ios::out);
4796 if (out.bad()) {
5513444a 4797 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4798 return;
4799 }
4800 out << "#!/bin/bash" << endl;
2a4d5166 4801 // Make sure we can properly compile par files
4802 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4803 out << "echo \"=========================================\"" << endl;
4804 out << "echo \"############## PATH : ##############\"" << endl;
4805 out << "echo $PATH" << endl;
4806 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4807 out << "echo $LD_LIBRARY_PATH" << endl;
4808 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4809 out << "echo $ROOTSYS" << endl;
4810 out << "echo \"############## which root : ##############\"" << endl;
4811 out << "which root" << endl;
4812 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4813 out << "echo $ALICE_ROOT" << endl;
4814 out << "echo \"############## which aliroot : ##############\"" << endl;
4815 out << "which aliroot" << endl;
9c5ddadc 4816 out << "echo \"############## system limits : ##############\"" << endl;
4817 out << "ulimit -a" << endl;
4818 out << "echo \"############## memory : ##############\"" << endl;
4819 out << "free -m" << endl;
c57f56b7 4820 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4821 out << fExecutableCommand << " ";
370c9880 4822 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4823 out << "RET=$?" << endl;
4824 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4825 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4826 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4827 out << " let sig=\"$RET - 128\""<<endl;
4828 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4829 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4830 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4831 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4832 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4833 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4834 out << " fi"<<endl;
4835 out << " exit $RET"<< endl;
4836 out << "fi" << endl << endl ;
4837 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4838 out << "echo \"############## memory after: ##############\"" << endl;
4839 out << "free -m" << endl;
c57f56b7 4840 }
4841 Bool_t copy = kTRUE;
d3b18c4c 4842 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4843 if (copy) {
4844 CdWork();
4845 TString workdir = gGrid->GetHomeDirectory();
4846 workdir += fGridWorkingDir;
6da4839e 4847 TString executable = TString::Format("%s/%s", workdir.Data(), fExecutable.Data());
c57f56b7 4848 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4849 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4850 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4851 executable.Data())) Fatal("","Terminating");
c57f56b7 4852 }
4853}
4854
0f389141 4855//______________________________________________________________________________
4856void AliAnalysisAlien::WriteMergeExecutable()
4857{
4858// Generate the alien executable script for the merging job.
4859 if (!fMergeViaJDL) return;
4860 TString mergeExec = fExecutable;
4861 mergeExec.ReplaceAll(".sh", "_merge.sh");
4862 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4863 ofstream out;
4864 out.open(mergeExec.Data(), ios::out);
4865 if (out.bad()) {
4866 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4867 return;
4868 }
4869 out << "#!/bin/bash" << endl;
2a4d5166 4870 // Make sure we can properly compile par files
4871 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4872 out << "echo \"=========================================\"" << endl;
4873 out << "echo \"############## PATH : ##############\"" << endl;
4874 out << "echo $PATH" << endl;
4875 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4876 out << "echo $LD_LIBRARY_PATH" << endl;
4877 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4878 out << "echo $ROOTSYS" << endl;
4879 out << "echo \"############## which root : ##############\"" << endl;
4880 out << "which root" << endl;
4881 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4882 out << "echo $ALICE_ROOT" << endl;
4883 out << "echo \"############## which aliroot : ##############\"" << endl;
4884 out << "which aliroot" << endl;
4885 out << "echo \"############## system limits : ##############\"" << endl;
4886 out << "ulimit -a" << endl;
4887 out << "echo \"############## memory : ##############\"" << endl;
4888 out << "free -m" << endl;
4889 out << "echo \"=========================================\"" << endl << endl;
0f389141 4890 TString mergeMacro = fExecutable;
4891 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4892 if (IsOneStageMerging())
4893 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4894 else
f47d5cb4 4895 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4896 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4897 out << "RET=$?" << endl;
4898 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4899 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4900 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4901 out << " let sig=\"$RET - 128\""<<endl;
4902 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4903 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4904 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4905 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4906 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4907 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4908 out << " fi"<<endl;
4909 out << " exit $RET"<< endl;
4910 out << "fi" << endl << endl ;
0f389141 4911 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4912 out << "echo \"############## memory after: ##############\"" << endl;
4913 out << "free -m" << endl;
0f389141 4914 }
4915 Bool_t copy = kTRUE;
d3b18c4c 4916 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4917 if (copy) {
4918 CdWork();
4919 TString workdir = gGrid->GetHomeDirectory();
0f389141 4920 workdir += fGridWorkingDir;
6da4839e 4921 TString executable = TString::Format("%s/%s", workdir.Data(), mergeExec.Data());
0f389141 4922 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4923 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6da4839e
AG		 4924 if (!copyLocal2Alien("WriteMergeExecutable",mergeExec.Data(),
4925 executable.Data())) Fatal("","Terminating");
0f389141 4926 }
4927}
4928
c57f56b7 4929//______________________________________________________________________________
5513444a 4930void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4931{
4932// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4933// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4934// Next lines: full_path_to_dataset XXX (XXX is a string)
4935// To submit, one has to: submit jdl XXX for all lines
4936 ofstream out;
4937 out.open(filename, ios::out);
4938 if (out.bad()) {
4939 Error("WriteProductionFile", "Bad file name: %s", filename);
4940 return;
4941 }
d3b18c4c 4942 TString workdir;
4943 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4944 workdir = gGrid->GetHomeDirectory();
5513444a 4945 workdir += fGridWorkingDir;
f5e8c702 4946 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4947 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4948 out << locjdl << " " << njobspermaster << endl;
5513444a 4949 Int_t nmasterjobs = fInputFiles->GetEntries();
4950 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4951 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4952 runOutDir.ReplaceAll(".xml", "");
4953 if (fOutputToRunNo)
4954 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4955 else
4956 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4957 }
d3b18c4c 4958 if (gGrid) {
4959 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4960 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4961// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4962 if (!copyLocal2Alien("WriteProductionFile", filename,
4963 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4964 }
5513444a 4965}
4966
4967//______________________________________________________________________________
0f389141 4968void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4969{
4970// Generate the alien validation script.
4971 // Generate the validation script
4972 TObjString *os;
d3b18c4c 4973 if (fValidationScript.IsNull()) {
4974 fValidationScript = fExecutable;
4975 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4976 }
4977 TString validationScript = fValidationScript;
4978 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4979 if (!Connect()) {
4980 Error("WriteValidationScript", "Alien connection required");
4981 return;
4982 }
d3b18c4c 4983 if (!fTerminateFiles.IsNull()) {
4984 fTerminateFiles.Strip();
4985 fTerminateFiles.ReplaceAll(" ",",");
4986 }
a2f5fc01 4987 TString outStream = "";
4988 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4989 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4990 ofstream out;
0d5d317c 4991 out.open(validationScript, ios::out);
c57f56b7 4992 out << "#!/bin/bash" << endl;
4993 out << "##################################################" << endl;
4994 out << "validateout=`dirname $0`" << endl;
4995 out << "validatetime=`date`" << endl;
4996 out << "validated=\"0\";" << endl;
4997 out << "error=0" << endl;
4998 out << "if [ -z $validateout ]" << endl;
4999 out << "then" << endl;
5000 out << " validateout=\".\"" << endl;
5001 out << "fi" << endl << endl;
5002 out << "cd $validateout;" << endl;
5003 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 5004 out << "echo \"*******************************************************\"" << outStream << endl;
5005 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 5006 out << "" << endl;
a2f5fc01 5007 out << "echo \"* Time: $validatetime \"" << outStream << endl;
5008 out << "echo \"* Dir: $validateout\"" << outStream << endl;
5009 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
5010 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5011 out << "ls -la ./" << outStream << endl;
5012 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 5013 out << "##################################################" << endl;
ebec370a 5014 out << "" << endl;
5015
5016 out << "if [ ! -f stderr ] ; then" << endl;
5017 out << " error=1" << endl;
a2f5fc01 5018 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
5019 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5020 out << "fi" << endl;
5021
b34c9f51 5022 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
5023 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
5024 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
5025 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
5026 out << "" << endl;
5027
ebec370a 5028 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
5029 out << " error=1" << endl;
a2f5fc01 5030 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
5031 out << " echo \"$parArch\" " << outStream << endl;
5032 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5033 out << "fi" << endl;
5034
5035 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
5036 out << " error=1" << endl;
a2f5fc01 5037 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
5038 out << " echo \"$segViol\" " << outStream << endl;
5039 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5040 out << "fi" << endl;
5041
5042 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
5043 out << " error=1" << endl;
a2f5fc01 5044 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
5045 out << " echo \"$segFault\" " << outStream << endl;
5046 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5047 out << "fi" << endl;
5048
b34c9f51 5049 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
5050 out << " error=1" << endl;
a2f5fc01 5051 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
5052 out << " echo \"$glibcErr\" " << outStream << endl;
5053 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 5054 out << "fi" << endl;
5055
ebec370a 5056 // Part dedicated to the specific analyses running into the train
5057
d3b18c4c 5058 TString outputFiles = fOutputFiles;
5059 if (merge && !fTerminateFiles.IsNull()) {
5060 outputFiles += ",";
5061 outputFiles += fTerminateFiles;
5062 }
5063 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 5064 TIter next1(arr);
a2f5fc01 5065 TString outputFile;
bbd88db8 5066 while (!merge && (os=(TObjString*)next1())) {
5067 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 5068 outputFile = os->GetString();
5069 Int_t index = outputFile.Index("@");
5070 if (index > 0) outputFile.Remove(index);
bbd88db8 5071 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 5072 if (outputFile.Contains("*")) continue;
5073 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 5074 out << " error=1" << endl;
7c2cd90a 5075 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
5076 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 5077 out << "fi" << endl;
5078 }
5079 delete arr;
7c2cd90a 5080 out << "if ! [ -f outputs_valid ] ; then" << endl;
5081 out << " error=1" << endl;
5082 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
5083 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
5084 out << "fi" << endl;
923e2ca5 5085
c57f56b7 5086 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 5087 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 5088 if (!IsKeepLogs()) {
5089 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 5090 outStream = "";
149d288c 5091 out << " rm -f std*" << endl;
5092 }
c57f56b7 5093 out << "fi" << endl;
5094
a2f5fc01 5095 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5096 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 5097 out << "cd -" << endl;
5098 out << "exit $error" << endl;
5099 }
5100 Bool_t copy = kTRUE;
d3b18c4c 5101 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 5102 if (copy) {
5103 CdWork();
5104 TString workdir = gGrid->GetHomeDirectory();
5105 workdir += fGridWorkingDir;
d3b18c4c 5106 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 5107 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 5108// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
5109 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
5110 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 5111 }
5112}