]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/ANALYSISalice/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
change order of OADB and ANALYSISalice
[u/mrichter/AliRoot.git] / ANALYSIS / ANALYSISalice / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
715da79c 32#include "TInterpreter.h"
c57f56b7 33#include "TFile.h"
830acc4c 34#include "TFileCollection.h"
3bdcb562 35#include "TChain.h"
c57f56b7 36#include "TObjString.h"
37#include "TObjArray.h"
77f34eae 38#include "TMacro.h"
c57f56b7 39#include "TGrid.h"
40#include "TGridResult.h"
41#include "TGridCollection.h"
42#include "TGridJDL.h"
d2a409b2 43#include "TGridJobStatusList.h"
44#include "TGridJobStatus.h"
c57f56b7 45#include "TFileMerger.h"
46#include "AliAnalysisManager.h"
77f34eae 47#include "AliAnalysisTaskCfg.h"
bb885a9e 48#include "AliVEventHandler.h"
49#include "AliAnalysisDataContainer.h"
65eb22f8 50#include "AliMultiInputEventHandler.h"
c57f56b7 51
c82bb898 52using std::ofstream;
53using std::ifstream;
54using std::ios;
55using std::endl;
c57f56b7 56ClassImp(AliAnalysisAlien)
6c395669 57#if 0
58;
59#endif
c57f56b7 60
6c395669 61namespace {
62 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
63 {
64 TString sl(Form("file:%s", loc));
65 TString sr(Form("alien://%s", rem));
66 Bool_t ret = TFile::Cp(sl, sr);
67 if (!ret) {
68 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
69 }
70 return ret;
71 }
72}
73
c57f56b7 74//______________________________________________________________________________
75AliAnalysisAlien::AliAnalysisAlien()
76 :AliAnalysisGrid(),
77 fGridJDL(NULL),
0f389141 78 fMergingJDL(NULL),
c57f56b7 79 fPrice(0),
80 fTTL(0),
81 fSplitMaxInputFileNumber(0),
82 fMaxInitFailed(0),
83 fMasterResubmitThreshold(0),
bb885a9e 84 fNtestFiles(0),
319593fb 85 fNrunsPerMaster(0),
16a4353c 86 fMaxMergeFiles(0),
b3e07543 87 fMaxMergeStages(0),
d2a409b2 88 fNsubmitted(0),
a3e84053 89 fProductionMode(0),
cd11251e 90 fOutputToRunNo(0),
0f389141 91 fMergeViaJDL(0),
92 fFastReadOption(0),
e1c22e21 93 fOverwriteMode(1),
149d288c 94 fNreplicas(2),
3bdcb562 95 fNproofWorkers(0),
27734f0e 96 fNproofWorkersPerSlave(0),
3bdcb562 97 fProofReset(0),
715da79c 98 fNMCevents(0),
99 fNMCjobs(0),
c57f56b7 100 fRunNumbers(),
101 fExecutable(),
0a1c1f7f 102 fExecutableCommand(),
c57f56b7 103 fArguments(),
631c0b05 104 fExecutableArgs(),
c57f56b7 105 fAnalysisMacro(),
106 fAnalysisSource(),
d3b18c4c 107 fValidationScript(),
d5c6455a 108 fAdditionalRootLibs(),
c57f56b7 109 fAdditionalLibs(),
715da79c 110 fGeneratorLibs(),
c57f56b7 111 fSplitMode(),
112 fAPIVersion(),
113 fROOTVersion(),
114 fAliROOTVersion(),
af78fedd 115 fAliPhysicsVersion(),
648174cf 116 fExternalPackages(),
c57f56b7 117 fUser(),
118 fGridWorkingDir(),
119 fGridDataDir(),
120 fDataPattern(),
121 fGridOutputDir(),
122 fOutputArchive(),
123 fOutputFiles(),
124 fInputFormat(),
e7c71df0 125 fDatasetName(),
c57f56b7 126 fJDLName(),
d3b18c4c 127 fTerminateFiles(),
bb885a9e 128 fMergeExcludes(),
37d21c01 129 fRegisterExcludes(),
f965131e 130 fIncludePath(),
bb885a9e 131 fCloseSE(),
0df6ccf2 132 fFriendChainName(),
c6cb3634 133 fJobTag(),
648174cf 134 fOutputSingle(),
5fce53f4 135 fRunPrefix(),
3bdcb562 136 fProofCluster(),
137 fProofDataSet(),
138 fFileForTestMode(),
3bdcb562 139 fAliRootMode(),
d88e4159 140 fProofProcessOpt(),
f47d5cb4 141 fMergeDirName(),
4e5c5506 142 fInputFiles(0),
28ce6c9c 143 fPackages(0),
77f34eae 144 fModules(0),
d88e4159 145 fProofParam(),
a41b2882 146 fDropToShell(true),
715da79c 147 fMCLoop(false),
a41b2882 148 fGridJobIDs(""),
149 fGridStages(""),
780ec36d 150 fFriendLibs(""),
151 fTreeName()
c57f56b7 152{
153// Dummy ctor.
154 SetDefaults();
155}
156
157//______________________________________________________________________________
158AliAnalysisAlien::AliAnalysisAlien(const char *name)
159 :AliAnalysisGrid(name),
160 fGridJDL(NULL),
0f389141 161 fMergingJDL(NULL),
c57f56b7 162 fPrice(0),
163 fTTL(0),
164 fSplitMaxInputFileNumber(0),
165 fMaxInitFailed(0),
166 fMasterResubmitThreshold(0),
bb885a9e 167 fNtestFiles(0),
319593fb 168 fNrunsPerMaster(0),
16a4353c 169 fMaxMergeFiles(0),
b3e07543 170 fMaxMergeStages(0),
d2a409b2 171 fNsubmitted(0),
a3e84053 172 fProductionMode(0),
cd11251e 173 fOutputToRunNo(0),
0f389141 174 fMergeViaJDL(0),
175 fFastReadOption(0),
e1c22e21 176 fOverwriteMode(1),
149d288c 177 fNreplicas(2),
3bdcb562 178 fNproofWorkers(0),
27734f0e 179 fNproofWorkersPerSlave(0),
3bdcb562 180 fProofReset(0),
715da79c 181 fNMCevents(0),
182 fNMCjobs(0),
c57f56b7 183 fRunNumbers(),
184 fExecutable(),
0a1c1f7f 185 fExecutableCommand(),
c57f56b7 186 fArguments(),
631c0b05 187 fExecutableArgs(),
c57f56b7 188 fAnalysisMacro(),
189 fAnalysisSource(),
d3b18c4c 190 fValidationScript(),
d5c6455a 191 fAdditionalRootLibs(),
c57f56b7 192 fAdditionalLibs(),
715da79c 193 fGeneratorLibs(),
c57f56b7 194 fSplitMode(),
195 fAPIVersion(),
196 fROOTVersion(),
197 fAliROOTVersion(),
af78fedd 198 fAliPhysicsVersion(),
648174cf 199 fExternalPackages(),
c57f56b7 200 fUser(),
201 fGridWorkingDir(),
202 fGridDataDir(),
203 fDataPattern(),
204 fGridOutputDir(),
205 fOutputArchive(),
206 fOutputFiles(),
207 fInputFormat(),
e7c71df0 208 fDatasetName(),
c57f56b7 209 fJDLName(),
d3b18c4c 210 fTerminateFiles(),
bb885a9e 211 fMergeExcludes(),
37d21c01 212 fRegisterExcludes(),
f965131e 213 fIncludePath(),
bb885a9e 214 fCloseSE(),
0df6ccf2 215 fFriendChainName(),
c6cb3634 216 fJobTag(),
648174cf 217 fOutputSingle(),
5fce53f4 218 fRunPrefix(),
3bdcb562 219 fProofCluster(),
220 fProofDataSet(),
221 fFileForTestMode(),
3bdcb562 222 fAliRootMode(),
d88e4159 223 fProofProcessOpt(),
f47d5cb4 224 fMergeDirName(),
4e5c5506 225 fInputFiles(0),
28ce6c9c 226 fPackages(0),
77f34eae 227 fModules(0),
d88e4159 228 fProofParam(),
0ad3ea1f 229 fDropToShell(true),
715da79c 230 fMCLoop(false),
0ad3ea1f 231 fGridJobIDs(""),
a41b2882 232 fGridStages(""),
780ec36d 233 fFriendLibs(""),
234 fTreeName()
c57f56b7 235{
236// Default ctor.
237 SetDefaults();
238}
239
240//______________________________________________________________________________
241AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
242 :AliAnalysisGrid(other),
243 fGridJDL(NULL),
0f389141 244 fMergingJDL(NULL),
c57f56b7 245 fPrice(other.fPrice),
246 fTTL(other.fTTL),
247 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
248 fMaxInitFailed(other.fMaxInitFailed),
249 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 250 fNtestFiles(other.fNtestFiles),
319593fb 251 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 252 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 253 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 254 fNsubmitted(other.fNsubmitted),
a3e84053 255 fProductionMode(other.fProductionMode),
cd11251e 256 fOutputToRunNo(other.fOutputToRunNo),
0f389141 257 fMergeViaJDL(other.fMergeViaJDL),
258 fFastReadOption(other.fFastReadOption),
259 fOverwriteMode(other.fOverwriteMode),
149d288c 260 fNreplicas(other.fNreplicas),
3bdcb562 261 fNproofWorkers(other.fNproofWorkers),
27734f0e 262 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 263 fProofReset(other.fProofReset),
715da79c 264 fNMCevents(other.fNMCevents),
265 fNMCjobs(other.fNMCjobs),
c57f56b7 266 fRunNumbers(other.fRunNumbers),
267 fExecutable(other.fExecutable),
0a1c1f7f 268 fExecutableCommand(other.fExecutableCommand),
c57f56b7 269 fArguments(other.fArguments),
631c0b05 270 fExecutableArgs(other.fExecutableArgs),
c57f56b7 271 fAnalysisMacro(other.fAnalysisMacro),
272 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 273 fValidationScript(other.fValidationScript),
d5c6455a 274 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 275 fAdditionalLibs(other.fAdditionalLibs),
715da79c 276 fGeneratorLibs(other.fGeneratorLibs),
c57f56b7 277 fSplitMode(other.fSplitMode),
278 fAPIVersion(other.fAPIVersion),
279 fROOTVersion(other.fROOTVersion),
280 fAliROOTVersion(other.fAliROOTVersion),
af78fedd 281 fAliPhysicsVersion(other.fAliPhysicsVersion),
648174cf 282 fExternalPackages(other.fExternalPackages),
c57f56b7 283 fUser(other.fUser),
284 fGridWorkingDir(other.fGridWorkingDir),
285 fGridDataDir(other.fGridDataDir),
286 fDataPattern(other.fDataPattern),
287 fGridOutputDir(other.fGridOutputDir),
288 fOutputArchive(other.fOutputArchive),
289 fOutputFiles(other.fOutputFiles),
290 fInputFormat(other.fInputFormat),
e7c71df0 291 fDatasetName(other.fDatasetName),
c57f56b7 292 fJDLName(other.fJDLName),
d3b18c4c 293 fTerminateFiles(other.fTerminateFiles),
bb885a9e 294 fMergeExcludes(other.fMergeExcludes),
37d21c01 295 fRegisterExcludes(other.fRegisterExcludes),
f965131e 296 fIncludePath(other.fIncludePath),
bb885a9e 297 fCloseSE(other.fCloseSE),
0df6ccf2 298 fFriendChainName(other.fFriendChainName),
c6cb3634 299 fJobTag(other.fJobTag),
648174cf 300 fOutputSingle(other.fOutputSingle),
5fce53f4 301 fRunPrefix(other.fRunPrefix),
3bdcb562 302 fProofCluster(other.fProofCluster),
303 fProofDataSet(other.fProofDataSet),
304 fFileForTestMode(other.fFileForTestMode),
3bdcb562 305 fAliRootMode(other.fAliRootMode),
d88e4159 306 fProofProcessOpt(other.fProofProcessOpt),
f47d5cb4 307 fMergeDirName(other.fMergeDirName),
4e5c5506 308 fInputFiles(0),
28ce6c9c 309 fPackages(0),
77f34eae 310 fModules(0),
d88e4159 311 fProofParam(),
0ad3ea1f 312 fDropToShell(other.fDropToShell),
715da79c 313 fMCLoop(other.fMCLoop),
0ad3ea1f 314 fGridJobIDs(other.fGridJobIDs),
a41b2882 315 fGridStages(other.fGridStages),
780ec36d 316 fFriendLibs(other.fFriendLibs),
317 fTreeName(other.fTreeName)
c57f56b7 318{
319// Copy ctor.
320 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 321 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 322 fRunRange[0] = other.fRunRange[0];
323 fRunRange[1] = other.fRunRange[1];
c57f56b7 324 if (other.fInputFiles) {
325 fInputFiles = new TObjArray();
326 TIter next(other.fInputFiles);
327 TObject *obj;
328 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
329 fInputFiles->SetOwner();
330 }
4e5c5506 331 if (other.fPackages) {
332 fPackages = new TObjArray();
333 TIter next(other.fPackages);
334 TObject *obj;
335 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
336 fPackages->SetOwner();
337 }
77f34eae 338 if (other.fModules) {
339 fModules = new TObjArray();
340 fModules->SetOwner();
341 TIter next(other.fModules);
342 AliAnalysisTaskCfg *mod, *crt;
343 while ((crt=(AliAnalysisTaskCfg*)next())) {
344 mod = new AliAnalysisTaskCfg(*crt);
345 fModules->Add(mod);
346 }
347 }
c57f56b7 348}
349
350//______________________________________________________________________________
351AliAnalysisAlien::~AliAnalysisAlien()
352{
353// Destructor.
77f34eae 354 delete fGridJDL;
355 delete fMergingJDL;
356 delete fInputFiles;
357 delete fPackages;
358 delete fModules;
28ce6c9c 359 fProofParam.DeleteAll();
c57f56b7 360}
361
362//______________________________________________________________________________
363AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
364{
365// Assignment.
366 if (this != &other) {
367 AliAnalysisGrid::operator=(other);
368 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 369 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 370 fPrice = other.fPrice;
371 fTTL = other.fTTL;
372 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
373 fMaxInitFailed = other.fMaxInitFailed;
374 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 375 fNtestFiles = other.fNtestFiles;
a3e84053 376 fNrunsPerMaster = other.fNrunsPerMaster;
377 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 378 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 379 fNsubmitted = other.fNsubmitted;
380 fProductionMode = other.fProductionMode;
cd11251e 381 fOutputToRunNo = other.fOutputToRunNo;
0f389141 382 fMergeViaJDL = other.fMergeViaJDL;
383 fFastReadOption = other.fFastReadOption;
384 fOverwriteMode = other.fOverwriteMode;
149d288c 385 fNreplicas = other.fNreplicas;
3bdcb562 386 fNproofWorkers = other.fNproofWorkers;
27734f0e 387 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 388 fProofReset = other.fProofReset;
715da79c 389 fNMCevents = other.fNMCevents;
390 fNMCjobs = other.fNMCjobs;
c57f56b7 391 fRunNumbers = other.fRunNumbers;
392 fExecutable = other.fExecutable;
0a1c1f7f 393 fExecutableCommand = other.fExecutableCommand;
c57f56b7 394 fArguments = other.fArguments;
631c0b05 395 fExecutableArgs = other.fExecutableArgs;
c57f56b7 396 fAnalysisMacro = other.fAnalysisMacro;
397 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 398 fValidationScript = other.fValidationScript;
d5c6455a 399 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 400 fAdditionalLibs = other.fAdditionalLibs;
715da79c 401 fGeneratorLibs = other.fGeneratorLibs;
c57f56b7 402 fSplitMode = other.fSplitMode;
403 fAPIVersion = other.fAPIVersion;
404 fROOTVersion = other.fROOTVersion;
405 fAliROOTVersion = other.fAliROOTVersion;
af78fedd 406 fAliPhysicsVersion = other.fAliPhysicsVersion;
648174cf 407 fExternalPackages = other.fExternalPackages;
c57f56b7 408 fUser = other.fUser;
409 fGridWorkingDir = other.fGridWorkingDir;
410 fGridDataDir = other.fGridDataDir;
411 fDataPattern = other.fDataPattern;
412 fGridOutputDir = other.fGridOutputDir;
413 fOutputArchive = other.fOutputArchive;
414 fOutputFiles = other.fOutputFiles;
415 fInputFormat = other.fInputFormat;
e7c71df0 416 fDatasetName = other.fDatasetName;
c57f56b7 417 fJDLName = other.fJDLName;
d3b18c4c 418 fTerminateFiles = other.fTerminateFiles;
bb885a9e 419 fMergeExcludes = other.fMergeExcludes;
37d21c01 420 fRegisterExcludes = other.fRegisterExcludes;
f965131e 421 fIncludePath = other.fIncludePath;
bb885a9e 422 fCloseSE = other.fCloseSE;
0df6ccf2 423 fFriendChainName = other.fFriendChainName;
c6cb3634 424 fJobTag = other.fJobTag;
648174cf 425 fOutputSingle = other.fOutputSingle;
5fce53f4 426 fRunPrefix = other.fRunPrefix;
3bdcb562 427 fProofCluster = other.fProofCluster;
428 fProofDataSet = other.fProofDataSet;
429 fFileForTestMode = other.fFileForTestMode;
3bdcb562 430 fAliRootMode = other.fAliRootMode;
d88e4159 431 fProofProcessOpt = other.fProofProcessOpt;
f47d5cb4 432 fMergeDirName = other.fMergeDirName;
d88e4159 433 fDropToShell = other.fDropToShell;
715da79c 434 fMCLoop = other.fMCLoop;
0ad3ea1f 435 fGridJobIDs = other.fGridJobIDs;
436 fGridStages = other.fGridStages;
a41b2882 437 fFriendLibs = other.fFriendLibs;
780ec36d 438 fTreeName = other.fTreeName;
c57f56b7 439 if (other.fInputFiles) {
440 fInputFiles = new TObjArray();
441 TIter next(other.fInputFiles);
442 TObject *obj;
443 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
444 fInputFiles->SetOwner();
445 }
4e5c5506 446 if (other.fPackages) {
447 fPackages = new TObjArray();
448 TIter next(other.fPackages);
449 TObject *obj;
450 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
451 fPackages->SetOwner();
452 }
77f34eae 453 if (other.fModules) {
454 fModules = new TObjArray();
455 fModules->SetOwner();
456 TIter next(other.fModules);
457 AliAnalysisTaskCfg *mod, *crt;
458 while ((crt=(AliAnalysisTaskCfg*)next())) {
459 mod = new AliAnalysisTaskCfg(*crt);
460 fModules->Add(mod);
461 }
462 }
c57f56b7 463 }
464 return *this;
465}
466
37d21c01 467//______________________________________________________________________________
468void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
469{
470// Add a single additional library to be loaded. Extension must be present.
471 TString lib(name);
472 if (!lib.Contains(".")) {
473 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
474 return;
475 }
476 if (fAdditionalLibs.Contains(name)) {
477 Warning("AddAdditionalLibrary", "Library %s already added.", name);
478 return;
479 }
480 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
481 fAdditionalLibs += lib;
482}
483
77f34eae 484//______________________________________________________________________________
485void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
486{
487// Adding a module. Checks if already existing. Becomes owned by this.
488 if (!module) return;
489 if (GetModule(module->GetName())) {
490 Error("AddModule", "A module having the same name %s already added", module->GetName());
491 return;
492 }
493 if (!fModules) {
494 fModules = new TObjArray();
495 fModules->SetOwner();
496 }
497 fModules->Add(module);
498}
499
500//______________________________________________________________________________
501void AliAnalysisAlien::AddModules(TObjArray *list)
502{
503// Adding a list of modules. Checks if already existing. Becomes owned by this.
504 TIter next(list);
505 AliAnalysisTaskCfg *module;
506 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
507}
508
509//______________________________________________________________________________
510Bool_t AliAnalysisAlien::CheckDependencies()
511{
512// Check if all dependencies are satisfied. Reorder modules if needed.
513 Int_t nmodules = GetNmodules();
514 if (!nmodules) {
515 Warning("CheckDependencies", "No modules added yet to check their dependencies");
516 return kTRUE;
517 }
518 AliAnalysisTaskCfg *mod = 0;
519 AliAnalysisTaskCfg *dep = 0;
520 TString depname;
521 Int_t i, j, k;
522 for (i=0; i<nmodules; i++) {
523 mod = (AliAnalysisTaskCfg*) fModules->At(i);
524 Int_t ndeps = mod->GetNdeps();
525 Int_t istart = i;
526 for (j=0; j<ndeps; j++) {
527 depname = mod->GetDependency(j);
528 dep = GetModule(depname);
529 if (!dep) {
530 Error("CheckDependencies","Dependency %s not added for module %s",
531 depname.Data(), mod->GetName());
532 return kFALSE;
533 }
534 if (dep->NeedsDependency(mod->GetName())) {
535 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
536 mod->GetName(), dep->GetName());
537 return kFALSE;
538 }
539 Int_t idep = fModules->IndexOf(dep);
540 // The dependency task must come first
541 if (idep>i) {
542 // Remove at idep and move all objects below up one slot
543 // down to index i included.
544 fModules->RemoveAt(idep);
3e40fd4c 545 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 546 fModules->AddAt(dep, i++);
547 }
548 //Redo from istart if dependencies were inserted
549 if (i>istart) i=istart-1;
550 }
551 }
552 return kTRUE;
553}
554
3e40fd4c 555//______________________________________________________________________________
556AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
557{
558// Create the analysis manager and optionally execute the macro in filename.
559 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
715da79c 560 if (mgr) {
561 mgr->SetMCLoop(fMCLoop);
562 return mgr;
563 }
3e40fd4c 564 mgr = new AliAnalysisManager(name);
565 mgr->SetGridHandler((AliAnalysisGrid*)this);
715da79c 566 mgr->SetMCLoop(fMCLoop);
3e40fd4c 567 if (strlen(filename)) {
568 TString line = gSystem->ExpandPathName(filename);
569 line.Prepend(".x ");
570 gROOT->ProcessLine(line.Data());
571 }
572 return mgr;
573}
574
77f34eae 575//______________________________________________________________________________
576Int_t AliAnalysisAlien::GetNmodules() const
577{
578// Get number of modules.
579 if (!fModules) return 0;
580 return fModules->GetEntries();
581}
582
583//______________________________________________________________________________
584AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
585{
586// Get a module by name.
587 if (!fModules) return 0;
588 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
589}
590
591//______________________________________________________________________________
592Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
593{
594// Load a given module.
595 if (mod->IsLoaded()) return kTRUE;
192ddca2 596 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
597 if (!mgr) {
598 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
599 return kFALSE;
600 }
77f34eae 601 Int_t ndeps = mod->GetNdeps();
602 TString depname;
603 for (Int_t j=0; j<ndeps; j++) {
604 depname = mod->GetDependency(j);
605 AliAnalysisTaskCfg *dep = GetModule(depname);
606 if (!dep) {
607 Error("LoadModule","Dependency %s not existing for module %s",
608 depname.Data(), mod->GetName());
609 return kFALSE;
610 }
611 if (!LoadModule(dep)) {
612 Error("LoadModule","Dependency %s for module %s could not be loaded",
613 depname.Data(), mod->GetName());
614 return kFALSE;
615 }
616 }
617 // Load libraries for the module
618 if (!mod->CheckLoadLibraries()) {
619 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
620 return kFALSE;
621 }
192ddca2 622 // Check if a custom file name was requested
623 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
624
625 // Check if a custom terminate file name was requested
626 if (strlen(mod->GetTerminateFileName())) {
627 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
628 fTerminateFiles += mod->GetTerminateFileName();
629 }
630
77f34eae 631 // Execute the macro
632 if (mod->ExecuteMacro()<0) {
633 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
634 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
635 return kFALSE;
636 }
637 // Configure dependencies
638 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
639 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
640 mod->GetConfigMacro()->GetTitle(), mod->GetName());
641 return kFALSE;
642 }
5e8c7c22 643 // Adjust extra libraries
644 Int_t nlibs = mod->GetNlibs();
645 TString lib;
646 for (Int_t i=0; i<nlibs; i++) {
647 lib = mod->GetLibrary(i);
5e8c7c22 648 lib = Form("lib%s.so", lib.Data());
c9d6bd12 649 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 650 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
651 fAdditionalLibs += lib;
652 }
77f34eae 653 return kTRUE;
654}
655
139fe6ba 656//______________________________________________________________________________
657Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
658{
659// Generate the full train.
660 fAdditionalLibs = "";
661 if (!LoadModules()) return kFALSE;
662 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
663 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 664 mgr->RunLocalInit();
139fe6ba 665 mgr->PrintStatus();
666 Int_t productionMode = fProductionMode;
667 SetProductionMode();
668 TString macro = fAnalysisMacro;
669 TString executable = fExecutable;
670 TString validation = fValidationScript;
671 TString execCommand = fExecutableCommand;
672 SetAnalysisMacro(Form("%s.C", name));
673 SetExecutable(Form("%s.sh", name));
a8e31fb0 674// SetExecutableCommand("aliroot -b -q ");
139fe6ba 675 SetValidationScript(Form("%s_validation.sh", name));
676 StartAnalysis();
677 SetProductionMode(productionMode);
678 fAnalysisMacro = macro;
679 fExecutable = executable;
680 fExecutableCommand = execCommand;
681 fValidationScript = validation;
682 return kTRUE;
683}
684
77f34eae 685//______________________________________________________________________________
21ca8e59 686Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 687{
688// Generate test macros for a single module or for the full train.
5e8c7c22 689 fAdditionalLibs = "";
77f34eae 690 if (strlen(modname)) {
21ca8e59 691 if (!CheckDependencies()) return kFALSE;
692 AliAnalysisTaskCfg *mod = GetModule(modname);
693 if (!mod) {
694 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
695 return kFALSE;
696 }
697 if (!LoadModule(mod)) return kFALSE;
a41b2882 698 if (!LoadFriendLibs()) return kFALSE;
21ca8e59 699 } else if (!LoadModules()) return kFALSE;
700 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
701 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 702 mgr->RunLocalInit();
21ca8e59 703 mgr->PrintStatus();
704 SetLocalTest(kTRUE);
705 Int_t productionMode = fProductionMode;
706 SetProductionMode();
707 TString macro = fAnalysisMacro;
708 TString executable = fExecutable;
709 TString validation = fValidationScript;
710 TString execCommand = fExecutableCommand;
711 SetAnalysisMacro(Form("%s.C", name));
712 SetExecutable(Form("%s.sh", name));
d66aa2a9 713 fOutputFiles = GetListOfFiles("outaod");
714 // Add extra files registered to the analysis manager
715 TString extra = GetListOfFiles("ext");
716 if (!extra.IsNull()) {
717 extra.ReplaceAll(".root", "*.root");
718 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
719 fOutputFiles += extra;
720 }
a8e31fb0 721// SetExecutableCommand("aliroot -b -q ");
21ca8e59 722 SetValidationScript(Form("%s_validation.sh", name));
723 WriteAnalysisFile();
724 WriteAnalysisMacro();
725 WriteExecutable();
726 WriteValidationScript();
ffbe06b9 727 WriteMergingMacro();
728 WriteMergeExecutable();
729 WriteValidationScript(kTRUE);
21ca8e59 730 SetLocalTest(kFALSE);
731 SetProductionMode(productionMode);
732 fAnalysisMacro = macro;
733 fExecutable = executable;
734 fExecutableCommand = execCommand;
735 fValidationScript = validation;
77f34eae 736 return kTRUE;
737}
738
739//______________________________________________________________________________
740Bool_t AliAnalysisAlien::LoadModules()
741{
742// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 743 fAdditionalLibs = "";
21ca8e59 744 Int_t nmodules = GetNmodules();
745 if (!nmodules) {
746 Warning("LoadModules", "No module to be loaded");
747 return kTRUE;
748 }
3e40fd4c 749 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
750 if (!mgr) {
751 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
752 return kFALSE;
753 }
77f34eae 754 if (!CheckDependencies()) return kFALSE;
21ca8e59 755 nmodules = GetNmodules();
77f34eae 756 AliAnalysisTaskCfg *mod;
21ca8e59 757 for (Int_t imod=0; imod<nmodules; imod++) {
758 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 759 if (!LoadModule(mod)) return kFALSE;
760 }
a41b2882 761 // Load additional friend libraries
762 return LoadFriendLibs();
77f34eae 763}
764
a41b2882 765//______________________________________________________________________________
766Bool_t AliAnalysisAlien::LoadFriendLibs() const
767{
768// Load libraries required for reading friends.
769 if (fFriendLibs.Length()) {
770 TObjArray *list = 0;
771 TString lib;
772 if (fFriendLibs.Contains(",")) list = fFriendLibs.Tokenize(",");
773 else list = fFriendLibs.Tokenize(" ");
774 for (Int_t ilib=0; ilib<list->GetEntriesFast(); ilib++) {
775 lib = list->At(ilib)->GetName();
776 lib.ReplaceAll(".so","");
2dffffd6 777 lib.ReplaceAll(".dylib","");
a41b2882 778 lib.ReplaceAll(" ","");
779 if (lib.BeginsWith("lib")) lib.Remove(0, 3);
780 lib.Prepend("lib");
781 Int_t loaded = strlen(gSystem->GetLibraries(lib,"",kFALSE));
782 if (!loaded) loaded = gSystem->Load(lib);
783 if (loaded < 0) {
784 Error("LoadModules", "Cannot load library for friends %s", lib.Data());
785 return kFALSE;
786 }
787 }
788 delete list;
789 }
790 return kTRUE;
791}
792
74013535 793//______________________________________________________________________________
794void AliAnalysisAlien::SetRunPrefix(const char *prefix)
795{
796// Set the run number format. Can be a prefix or a format like "%09d"
797 fRunPrefix = prefix;
798 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
799}
800
f965131e 801//______________________________________________________________________________
802void AliAnalysisAlien::AddIncludePath(const char *path)
803{
804// Add include path in the remote analysis macro.
805 TString p(path);
806 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
807 else fIncludePath += Form("-I%s ", path);
808}
809
c57f56b7 810//______________________________________________________________________________
811void AliAnalysisAlien::AddRunNumber(Int_t run)
812{
813// Add a run number to the list of runs to be processed.
814 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 815 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 816}
817
98690895 818//______________________________________________________________________________
819void AliAnalysisAlien::AddRunList(const char* runList)
820{
821// Add several runs into the list of runs; they are expected to be separated by a blank character.
822 TString sList = runList;
823 TObjArray *list = sList.Tokenize(" ");
824 Int_t n = list->GetEntries();
825 for (Int_t i = 0; i < n; i++) {
826 TObjString *os = (TObjString*)list->At(i);
827 AddRunNumber(os->GetString().Atoi());
828 }
829 delete list;
830}
831
ee75cfc3 832//______________________________________________________________________________
833void AliAnalysisAlien::AddRunNumber(const char* run)
834{
835// Add a run number to the list of runs to be processed.
7aa5d539 836 TString runs = run;
837 TObjString *os;
838 TObjArray *arr = runs.Tokenize(" ");
839 TIter next(arr);
840 TString prefix;
841 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
842 while ((os=(TObjString*)next())){
843 if (fRunNumbers.Length()) fRunNumbers += " ";
844 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
845 }
846 delete arr;
ee75cfc3 847}
848
c57f56b7 849//______________________________________________________________________________
850void AliAnalysisAlien::AddDataFile(const char *lfn)
851{
852// Adds a data file to the input to be analysed. The file should be a valid LFN
853// or point to an existing file in the alien workdir.
854 if (!fInputFiles) fInputFiles = new TObjArray();
855 fInputFiles->Add(new TObjString(lfn));
856}
648174cf 857
858//______________________________________________________________________________
859void AliAnalysisAlien::AddExternalPackage(const char *package)
860{
861// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
862 if (fExternalPackages) fExternalPackages += " ";
863 fExternalPackages += package;
864}
865
c57f56b7 866//______________________________________________________________________________
867Bool_t AliAnalysisAlien::Connect()
868{
869// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
870 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 871 if (fProductionMode) return kTRUE;
c57f56b7 872 if (!gGrid) {
873 Info("Connect", "Trying to connect to AliEn ...");
874 TGrid::Connect("alien://");
875 }
876 if (!gGrid || !gGrid->IsConnected()) {
877 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
878 return kFALSE;
879 }
880 fUser = gGrid->GetUser();
881 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
882 return kTRUE;
883}
884
885//______________________________________________________________________________
886void AliAnalysisAlien::CdWork()
887{
888// Check validity of alien workspace. Create directory if possible.
889 if (!Connect()) {
890 Error("CdWork", "Alien connection required");
891 return;
892 }
893 TString homedir = gGrid->GetHomeDirectory();
894 TString workdir = homedir + fGridWorkingDir;
923e2ca5 895 if (DirectoryExists(workdir)) {
896 gGrid->Cd(workdir);
897 return;
898 }
899 // Work directory not existing - create it
900 gGrid->Cd(homedir);
b93f8109 901 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 902 gGrid->Cd(fGridWorkingDir);
d3b18c4c 903 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 904 } else {
d3b18c4c 905 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 906 workdir.Data(), homedir.Data());
907 fGridWorkingDir = "";
908 }
c57f56b7 909}
910
348be253 911//______________________________________________________________________________
912Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
913{
914// Check if file copying is possible.
d3b18c4c 915 if (fProductionMode) return kTRUE;
b5ae4fe0 916 TString salienpath(alienpath);
917 if (salienpath.Contains(" ")) {
918 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
919 return kFALSE;
920 }
348be253 921 if (!Connect()) {
922 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
923 return kFALSE;
924 }
d3339be3 925 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
926 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 927 // Check if alien_CLOSE_SE is defined
928 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
929 if (!closeSE.IsNull()) {
930 Info("CheckFileCopy", "Your current close storage is pointing to: \
931 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
932 } else {
933 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
934 }
935 // Check if grid directory exists.
936 if (!DirectoryExists(alienpath)) {
937 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
938 return kFALSE;
939 }
b5ae4fe0 940 TString stest = "plugin_test_copy";
941 TFile f(stest, "RECREATE");
348be253 942 // User may not have write permissions to current directory
943 if (f.IsZombie()) {
944 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
945 gSystem->WorkingDirectory());
946 return kFALSE;
947 }
948 f.Close();
b5ae4fe0 949 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
950 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 951 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 952 \n# 1. Make sure you have write permissions there. If this is the case: \
953 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
954 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
955 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
956 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 957 gSystem->Unlink(stest.Data());
348be253 958 return kFALSE;
959 }
b5ae4fe0 960 gSystem->Unlink(stest.Data());
961 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 962 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 963 return kTRUE;
964}
965
c57f56b7 966//______________________________________________________________________________
967Bool_t AliAnalysisAlien::CheckInputData()
968{
969// Check validity of input data. If necessary, create xml files.
d3b18c4c 970 if (fProductionMode) return kTRUE;
a8739e8a 971 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 972 if (!fGridDataDir.Length()) {
973 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
974 return kFALSE;
975 }
9f020952 976 if (fMergeViaJDL) {
977 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
978 return kFALSE;
979 }
d2a409b2 980 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 981 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
982 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 983 return kTRUE;
c57f56b7 984 }
985 // Process declared files
a2f5fc01 986 Bool_t isCollection = kFALSE;
987 Bool_t isXml = kFALSE;
988 Bool_t useTags = kFALSE;
c57f56b7 989 Bool_t checked = kFALSE;
d3b18c4c 990 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 991 TString file;
992 TString workdir = gGrid->GetHomeDirectory();
993 workdir += fGridWorkingDir;
994 if (fInputFiles) {
995 TObjString *objstr;
996 TIter next(fInputFiles);
997 while ((objstr=(TObjString*)next())) {
998 file = workdir;
999 file += "/";
1000 file += objstr->GetString();
1001 // Store full lfn path
1002 if (FileExists(file)) objstr->SetString(file);
1003 else {
1004 file = objstr->GetName();
1005 if (!FileExists(objstr->GetName())) {
1006 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
1007 objstr->GetName(), workdir.Data());
1008 return kFALSE;
1009 }
1010 }
1011 Bool_t iscoll, isxml, usetags;
1012 CheckDataType(file, iscoll, isxml, usetags);
1013 if (!checked) {
1014 checked = kTRUE;
a2f5fc01 1015 isCollection = iscoll;
1016 isXml = isxml;
1017 useTags = usetags;
1018 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1019 } else {
a2f5fc01 1020 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 1021 Error("CheckInputData", "Some conflict was found in the types of inputs");
1022 return kFALSE;
1023 }
1024 }
1025 }
1026 }
1027 // Process requested run numbers
a8739e8a 1028 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 1029 // Check validity of alien data directory
1030 if (!fGridDataDir.Length()) {
1031 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
1032 return kFALSE;
1033 }
923e2ca5 1034 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 1035 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
1036 return kFALSE;
1037 }
a2f5fc01 1038 if (isCollection) {
c57f56b7 1039 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
1040 return kFALSE;
1041 }
1042
a2f5fc01 1043 if (checked && !isXml) {
c57f56b7 1044 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
1045 return kFALSE;
1046 }
1047 // Check validity of run number(s)
1048 TObjArray *arr;
1049 TObjString *os;
74013535 1050 TString format;
319593fb 1051 Int_t nruns = 0;
904f9f5f 1052 TString schunk, schunk2;
c57f56b7 1053 TString path;
1054 if (!checked) {
1055 checked = kTRUE;
a2f5fc01 1056 useTags = fDataPattern.Contains("tag");
1057 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1058 }
a2f5fc01 1059 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 1060 Error("CheckInputData", "Cannot mix input files using/not using tags");
1061 return kFALSE;
1062 }
1063 if (fRunNumbers.Length()) {
a8739e8a 1064 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 1065 arr = fRunNumbers.Tokenize(" ");
1066 TIter next(arr);
1067 while ((os=(TObjString*)next())) {
1068 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1069 if (!DirectoryExists(path)) {
1070 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 1071 continue;
c57f56b7 1072 }
1073 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
1074 TString msg = "\n##### file: ";
1075 msg += path;
1076 msg += " type: xml_collection;";
a2f5fc01 1077 if (useTags) msg += " using_tags: Yes";
c57f56b7 1078 else msg += " using_tags: No";
23329835 1079 Info("CheckDataType", "%s", msg.Data());
319593fb 1080 if (fNrunsPerMaster<2) {
d2a409b2 1081 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 1082 } else {
1083 nruns++;
1084 if (((nruns-1)%fNrunsPerMaster) == 0) {
1085 schunk = os->GetString();
1086 }
1087 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1088 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1089 AddDataFile(schunk);
319593fb 1090 }
c57f56b7 1091 }
1092 delete arr;
a8739e8a 1093 } else {
1094 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1095 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1096 format = Form("%%s/%s ", fRunPrefix.Data());
1097 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1098 if (!DirectoryExists(path)) {
a8739e8a 1099 continue;
1100 }
74013535 1101 format = Form("%%s/%s.xml", fRunPrefix.Data());
1102 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1103 TString msg = "\n##### file: ";
1104 msg += path;
1105 msg += " type: xml_collection;";
a2f5fc01 1106 if (useTags) msg += " using_tags: Yes";
a8739e8a 1107 else msg += " using_tags: No";
23329835 1108 Info("CheckDataType", "%s", msg.Data());
319593fb 1109 if (fNrunsPerMaster<2) {
74013535 1110 format = Form("%s.xml", fRunPrefix.Data());
1111 AddDataFile(Form(format.Data(),irun));
319593fb 1112 } else {
1113 nruns++;
1114 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1115 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1116 }
74013535 1117 format = Form("_%s.xml", fRunPrefix.Data());
1118 schunk2 = Form(format.Data(), irun);
319593fb 1119 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1120 schunk += schunk2;
d2a409b2 1121 AddDataFile(schunk);
319593fb 1122 }
a8739e8a 1123 }
904f9f5f 1124 if (!fInputFiles) {
1125 schunk += schunk2;
1126 AddDataFile(schunk);
1127 }
c57f56b7 1128 }
1129 return kTRUE;
1130}
1131
21ca8e59 1132//______________________________________________________________________________
4a8b667c 1133Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1134{
1135// Copy data from the given grid directory according a pattern and make a local
1136// dataset.
2ba4c287 1137// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1138 if (!Connect()) {
1139 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
4a8b667c 1140 return 0;
21ca8e59 1141 }
1142 if (!DirectoryExists(griddir)) {
1143 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
4a8b667c 1144 return 0;
21ca8e59 1145 }
1146 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1147 printf("Running command: %s\n", command.Data());
1148 TGridResult *res = gGrid->Command(command);
1149 Int_t nfound = res->GetEntries();
1150 if (!nfound) {
1151 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
4a8b667c 1152 return 0;
21ca8e59 1153 }
1154 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1155
1156 // archives
1157 TObjArray* additionalArchives = 0;
1158 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1159 additionalArchives = TString(archivefile).Tokenize(";");
1160 archivefile = additionalArchives->At(0)->GetName();
1161 additionalArchives->RemoveAt(0);
1162 additionalArchives->Compress();
1163 }
1164
21ca8e59 1165 // Copy files locally
1166 ofstream out;
1167 out.open(output, ios::out);
1168 TMap *map;
1169 TString turl, dirname, filename, temp;
1170 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1171 gSystem->MakeDirectory(outputdir);
1172 gSystem->ChangeDirectory(outputdir);
4a8b667c 1173 Int_t ncopied = 0;
21ca8e59 1174 for (Int_t i=0; i<nfound; i++) {
1175 map = (TMap*)res->At(i);
1176 turl = map->GetValue("turl")->GetName();
1177 filename = gSystem->BaseName(turl.Data());
1178 dirname = gSystem->DirName(turl.Data());
1179 dirname = gSystem->BaseName(dirname.Data());
1180 gSystem->MakeDirectory(dirname);
43b807f6 1181
1182 TString source(turl);
1183 TString targetFileName(filename);
1184
1185 if (strlen(archivefile) > 0) {
4a8b667c 1186// TODO here the archive in which the file resides should be determined
1187// however whereis returns only a guid, and guid2lfn does not work
1188// Therefore we use the one provided as argument for now
1189 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1190 targetFileName = archivefile;
43b807f6 1191 }
1192 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
4a8b667c 1193 Bool_t success = kTRUE;
1194 if (additionalArchives) {
1195 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1196 TString target;
1197 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1198 gSystem->MakeDirectory(gSystem->DirName(target));
1199 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1200 }
1201 }
2ba4c287 1202
4a8b667c 1203 if (success) {
1204 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1205 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1206 ncopied++;
1207 }
21ca8e59 1208 }
1209 }
1210 gSystem->ChangeDirectory(cdir);
1211 delete res;
4a8b667c 1212 delete additionalArchives;
1213 return ncopied;
21ca8e59 1214}
1215
c57f56b7 1216//______________________________________________________________________________
1217Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1218{
1219// Create dataset for the grid data directory + run number.
3af861ff 1220 const Int_t gMaxEntries = 15000;
d3b18c4c 1221 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1222 if (!Connect()) {
1223 Error("CreateDataset", "Cannot create dataset with no grid connection");
1224 return kFALSE;
1225 }
1226
1227 // Cd workspace
d3b18c4c 1228 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1229 TString workdir = gGrid->GetHomeDirectory();
1230 workdir += fGridWorkingDir;
1231
1232 // Compose the 'find' command arguments
74013535 1233 TString format;
c57f56b7 1234 TString command;
780ec36d 1235 TString delimiter = pattern;
1236 delimiter.Strip();
1237 if (delimiter.Contains(" ")) delimiter = "";
1238 else delimiter = " ";
c57f56b7 1239 TString options = "-x collection ";
bb885a9e 1240 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1241 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1242 TString conditions = "";
3af861ff 1243 Int_t nstart = 0;
1244 Int_t ncount = 0;
1245 Int_t stage = 0;
c57f56b7 1246 TString file;
1247 TString path;
319593fb 1248 Int_t nruns = 0;
904f9f5f 1249 TString schunk, schunk2;
ab254fd1 1250 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1251 if (!fRunNumbers.Length() && !fRunRange[0]) {
1252 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1253 // Make a single data collection from data directory.
1254 path = fGridDataDir;
923e2ca5 1255 if (!DirectoryExists(path)) {
d2a409b2 1256 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1257 return kFALSE;
780ec36d 1258 }
923e2ca5 1259// CdWork();
d2a409b2 1260 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1261 else file = Form("%s.xml", gSystem->BaseName(path));
829ae708 1262 // cholm - Identical loop - should be put in common function for code simplification
3af861ff 1263 while (1) {
1264 ncount = 0;
1265 stage++;
1266 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1267 command = "find ";
1268 command += Form("%s -o %d ",options.Data(), nstart);
1269 command += path;
780ec36d 1270 command += delimiter;
3af861ff 1271 command += pattern;
1272 command += conditions;
1273 printf("command: %s\n", command.Data());
1274 TGridResult *res = gGrid->Command(command);
1275 if (res) delete res;
1276 // Write standard output to file
1277 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1278 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1279 Bool_t nullFile = kFALSE;
1280 if (!hasGrep) {
1281 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1282 } else {
1283 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1284 if (nullFile) {
1285 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1286 gSystem->Exec("rm -f __tmp*");
1287 return kFALSE;
1288 }
1289 TString line;
1290 ifstream in;
1291 in.open("__tmp__");
1292 in >> line;
1293 in.close();
1294 gSystem->Exec("rm -f __tmp__");
1295 ncount = line.Atoi();
1296 }
1297 }
1298 if (ncount == gMaxEntries) {
1299 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1300 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1301 if (!cbase) cbase = cadd;
1302 else {
829ae708 1303 // cholm - Avoid using very slow TAlienCollection
1304 // cbase->Add(cadd);
1305 // cholm - Use AddFast (via interpreter)
1306 gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
3af861ff 1307 delete cadd;
defd7a3a 1308 }
3af861ff 1309 nstart += ncount;
1310 } else {
1311 if (cbase) {
1312 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1313 printf("... please wait - TAlienCollection::Add() scales badly...\n");
829ae708 1314 // cholm - Avoid using very slow TAlienCollection
1315 // cbase->Add(cadd);
1316 // cholm - Use AddFast (via interpreter)
1317 gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
3af861ff 1318 delete cadd;
1319 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1320 delete cbase; cbase = 0;
1321 } else {
1322 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1323 }
1324 gSystem->Exec("rm -f __tmp*");
1325 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1326 break;
1327 }
0f389141 1328 }
1329 Bool_t fileExists = FileExists(file);
1330 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1331 // Copy xml file to alien space
0f389141 1332 if (fileExists) gGrid->Rm(file);
d2a409b2 1333 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1334 if (!FileExists(file)) {
1335 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1336 return kFALSE;
1337 }
1338 // Update list of files to be processed.
1339 }
1340 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1341 return kTRUE;
1342 }
c57f56b7 1343 // Several runs
a2f5fc01 1344 Bool_t nullResult = kTRUE;
a8739e8a 1345 if (fRunNumbers.Length()) {
1346 TObjArray *arr = fRunNumbers.Tokenize(" ");
1347 TObjString *os;
1348 TIter next(arr);
1349 while ((os=(TObjString*)next())) {
3af861ff 1350 nstart = 0;
1351 stage = 0;
0dab9d6d 1352 path = Form("%s/%s", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1353 if (!DirectoryExists(path)) continue;
1354// CdWork();
a8739e8a 1355 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1356 else file = Form("%s.xml", os->GetString().Data());
829ae708 1357 // cholm - Identical loop - should be put in common function for code simplification
319593fb 1358 // If local collection file does not exist, create it via 'find' command.
3af861ff 1359 while (1) {
1360 ncount = 0;
1361 stage++;
1362 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1363 command = "find ";
1364 command += Form("%s -o %d ",options.Data(), nstart);
1365 command += path;
780ec36d 1366 command += delimiter;
3af861ff 1367 command += pattern;
1368 command += conditions;
1369 TGridResult *res = gGrid->Command(command);
1370 if (res) delete res;
1371 // Write standard output to file
1372 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1373 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1374 Bool_t nullFile = kFALSE;
1375 if (!hasGrep) {
1376 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1377 } else {
1378 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1379 if (nullFile) {
1380 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1381 gSystem->Exec("rm -f __tmp*");
1382 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1383 break;
3af861ff 1384 }
1385 TString line;
1386 ifstream in;
1387 in.open("__tmp__");
1388 in >> line;
1389 in.close();
1390 gSystem->Exec("rm -f __tmp__");
1391 ncount = line.Atoi();
1392 }
1393 nullResult = kFALSE;
1394 }
1395 if (ncount == gMaxEntries) {
1396 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1397 if (fNrunsPerMaster > 1) {
1398 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1399 file.Data(),gMaxEntries);
1400 return kFALSE;
1401 }
1402 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1403 if (!cbase) cbase = cadd;
1404 else {
829ae708 1405 // cholm - Avoid using very slow TAlienCollection
1406 // cbase->Add(cadd);
1407 // cholm - Use AddFast (via interpreter)
1408 gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
1409
3af861ff 1410 delete cadd;
defd7a3a 1411 }
3af861ff 1412 nstart += ncount;
1413 } else {
c89a84bc 1414 if (cbase && fNrunsPerMaster<2) {
3af861ff 1415 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1416 printf("... please wait - TAlienCollection::Add() scales badly...\n");
829ae708 1417 // cholm - Avoid using very slow TAlienCollection
1418 // cbase->Add(cadd);
1419 // cholm - Use AddFast (via interpreter)
1420 gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
3af861ff 1421 delete cadd;
1422 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1423 delete cbase; cbase = 0;
1424 } else {
1425 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1426 }
1427 gSystem->Exec("rm -f __tmp*");
1428 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1429 break;
d3339be3 1430 }
3af861ff 1431 }
a8739e8a 1432 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1433 // Check if there is one run per master job.
1434 if (fNrunsPerMaster<2) {
1435 if (FileExists(file)) {
0f389141 1436 if (fOverwriteMode) gGrid->Rm(file);
1437 else {
1438 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1439 continue;
1440 }
319593fb 1441 }
1442 // Copy xml file to alien space
1443 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1444 if (!FileExists(file)) {
1445 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1446 delete arr;
1447 return kFALSE;
1448 }
1449 } else {
1450 nruns++;
1451 if (((nruns-1)%fNrunsPerMaster) == 0) {
1452 schunk = os->GetString();
1453 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1454 } else {
1455 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1456 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
829ae708 1457 // cholm - Avoid using very slow TAlienCollection
1458 // cbase->Add(cadd);
1459 // cholm - Use AddFast (via interpreter)
1460 gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
319593fb 1461 delete cadd;
1462 }
1463 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1464 continue;
1465 }
1466 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1467 if (FileExists(schunk)) {
1468 if (fOverwriteMode) gGrid->Rm(file);
1469 else {
1470 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1471 continue;
1472 }
319593fb 1473 }
84fcd93f 1474 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1475 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1476 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1477 if (!FileExists(schunk)) {
1478 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1479 delete arr;
1480 return kFALSE;
1481 }
d3339be3 1482 }
a8739e8a 1483 }
1484 delete arr;
a2f5fc01 1485 if (nullResult) {
d3339be3 1486 Error("CreateDataset", "No valid dataset corresponding to the query!");
1487 return kFALSE;
1488 }
a8739e8a 1489 } else {
1490 // Process a full run range.
1491 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
0dab9d6d 1492 format = Form("%%s/%s", fRunPrefix.Data());
3af861ff 1493 nstart = 0;
1494 stage = 0;
74013535 1495 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1496 if (!DirectoryExists(path)) continue;
1497// CdWork();
74013535 1498 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1499 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1500 else file = Form(format.Data(), irun);
0f389141 1501 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1502 if (fOverwriteMode) gGrid->Rm(file);
1503 else {
1504 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1505 continue;
1506 }
a8739e8a 1507 }
829ae708 1508 // cholm - Identical loop - should be put in common function for code simplification
319593fb 1509 // If local collection file does not exist, create it via 'find' command.
3af861ff 1510 while (1) {
1511 ncount = 0;
1512 stage++;
1513 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1514 command = "find ";
1515 command += Form("%s -o %d ",options.Data(), nstart);
1516 command += path;
780ec36d 1517 command += delimiter;
3af861ff 1518 command += pattern;
1519 command += conditions;
1520 TGridResult *res = gGrid->Command(command);
1521 if (res) delete res;
1522 // Write standard output to file
1523 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1524 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1525 Bool_t nullFile = kFALSE;
1526 if (!hasGrep) {
1527 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1528 } else {
1529 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1530 if (nullFile) {
1531 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1532 gSystem->Exec("rm -f __tmp*");
1604c9ed 1533 break;
3af861ff 1534 }
1535 TString line;
1536 ifstream in;
1537 in.open("__tmp__");
1538 in >> line;
1539 in.close();
1540 gSystem->Exec("rm -f __tmp__");
1541 ncount = line.Atoi();
1542 }
1543 nullResult = kFALSE;
1544 }
1545 if (ncount == gMaxEntries) {
1546 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1547 if (fNrunsPerMaster > 1) {
1548 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1549 file.Data(),gMaxEntries);
1550 return kFALSE;
1551 }
1552 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1553 if (!cbase) cbase = cadd;
1554 else {
829ae708 1555 // cholm - Avoid using very slow TAlienCollection
1556 // cbase->Add(cadd);
1557 // cholm - Use AddFast (via interpreter)
1558 gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
3af861ff 1559 delete cadd;
defd7a3a 1560 }
3af861ff 1561 nstart += ncount;
1562 } else {
c89a84bc 1563 if (cbase && fNrunsPerMaster<2) {
3af861ff 1564 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1565 printf("... please wait - TAlienCollection::Add() scales badly...\n");
829ae708 1566 // cholm - Avoid using very slow TAlienCollection
1567 // cbase->Add(cadd);
1568 // cholm - Use AddFast (via interpreter)
1569 gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
3af861ff 1570 delete cadd;
1571 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1572 delete cbase; cbase = 0;
1573 } else {
1574 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1575 }
1576 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1577 break;
d3339be3 1578 }
319593fb 1579 }
a8739e8a 1580 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1581 // Check if there is one run per master job.
1582 if (fNrunsPerMaster<2) {
1583 if (FileExists(file)) {
0f389141 1584 if (fOverwriteMode) gGrid->Rm(file);
1585 else {
1586 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1587 continue;
1588 }
319593fb 1589 }
1590 // Copy xml file to alien space
1591 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1592 if (!FileExists(file)) {
1593 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1594 return kFALSE;
1595 }
1596 } else {
1597 nruns++;
95e5b448 1598 // Check if the collection for the chunk exist locally.
1599 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1600 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1601 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1602 else continue;
1603 }
84fcd93f 1604 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1605 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1606 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1607 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1608 } else {
1609 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
829ae708 1610 // cholm - Avoid using very slow TAlienCollection
1611 // cbase->Add(cadd);
1612 // cholm - Use AddFast (via interpreter)
1613 gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
319593fb 1614 delete cadd;
1615 }
74013535 1616 format = Form("%%s_%s.xml", fRunPrefix.Data());
1617 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1618 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1619 continue;
1620 }
904f9f5f 1621 schunk = schunk2;
319593fb 1622 if (FileExists(schunk)) {
0f389141 1623 if (fOverwriteMode) gGrid->Rm(schunk);
1624 else {
1625 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1626 continue;
1627 }
319593fb 1628 }
84fcd93f 1629 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1630 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1631 if (FileExists(schunk)) {
0f389141 1632 if (fOverwriteMode) gGrid->Rm(schunk);
1633 else {
1634 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1635 continue;
1636 }
95e5b448 1637 }
319593fb 1638 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1639 if (!FileExists(schunk)) {
1640 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1641 return kFALSE;
1642 }
1643 }
c57f56b7 1644 }
a2f5fc01 1645 if (nullResult) {
d3339be3 1646 Error("CreateDataset", "No valid dataset corresponding to the query!");
1647 return kFALSE;
1648 }
a8739e8a 1649 }
c57f56b7 1650 return kTRUE;
1651}
1652
1653//______________________________________________________________________________
1654Bool_t AliAnalysisAlien::CreateJDL()
1655{
1656// Generate a JDL file according to current settings. The name of the file is
1657// specified by fJDLName.
ec5acdce 1658 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
c57f56b7 1659 Bool_t error = kFALSE;
1660 TObjArray *arr = 0;
1661 Bool_t copy = kTRUE;
d3b18c4c 1662 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1663 Bool_t generate = kTRUE;
1664 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1665 if (!Connect()) {
1666 Error("CreateJDL", "Alien connection required");
1667 return kFALSE;
1668 }
1669 // Check validity of alien workspace
d3b18c4c 1670 TString workdir;
1671 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1672 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1673 workdir += fGridWorkingDir;
1674 if (generate) {
1675 TObjString *os;
1e466628 1676 if (!fInputFiles && !fMCLoop) {
c57f56b7 1677 Error("CreateJDL()", "Define some input files for your analysis.");
1678 error = kTRUE;
1679 }
1680 // Compose list of input files
1681 // Check if output files were defined
1682 if (!fOutputFiles.Length()) {
1683 Error("CreateJDL", "You must define at least one output file");
1684 error = kTRUE;
1685 }
1686 // Check if an output directory was defined and valid
1687 if (!fGridOutputDir.Length()) {
1688 Error("CreateJDL", "You must define AliEn output directory");
1689 error = kTRUE;
1690 } else {
d3b18c4c 1691 if (!fProductionMode) {
1692 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1693 if (!DirectoryExists(fGridOutputDir)) {
1694 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1695 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1696 } else {
1697 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1698 // error = kTRUE;
1699 }
67a3b254 1700 } else {
1701 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1702 }
1703 gGrid->Cd(workdir);
1704 }
c57f56b7 1705 }
1706 // Exit if any error up to now
1707 if (error) return kFALSE;
1708 // Set JDL fields
0f389141 1709 if (!fUser.IsNull()) {
1710 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1711 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
6da4839e
AG		 1712 }
1713 TString executable = fExecutable;
1714 if (!executable.BeginsWith("/"))
1715 executable.Prepend(Form("%s/", workdir.Data()));
1716 fGridJDL->SetExecutable(executable, "This is the startup script");
1717 TString mergeExec = executable;
0f389141 1718 mergeExec.ReplaceAll(".sh", "_merge.sh");
1719 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1720 mergeExec.ReplaceAll(".sh", ".C");
9c5cde18 1721 fMergingJDL->AddToInputSandbox(Form("LF:%s", mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1722 if (!fArguments.IsNull())
1723 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1724 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1725 else {
1726 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1727 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1728 }
b3e07543 1729
1f0d1ca2 1730 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1731 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1732 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1733 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1734
0f389141 1735 if (fMaxInitFailed > 0) {
c57f56b7 1736 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1737 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1738 }
6da4839e 1739 if (fSplitMaxInputFileNumber > 0 && !fMCLoop) {
c57f56b7 1740 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1741 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1742 }
d7eb1f39 1743 if (!IsOneStageMerging()) {
f47d5cb4 1744 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1745 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1746 }
1747 if (fSplitMode.Length()) {
c57f56b7 1748 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1749 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1750 }
1751 fMergingJDL->SetValue("Split", "\"se\"");
1752 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1753 if (!fAliROOTVersion.IsNull()) {
1754 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1755 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1756 }
af78fedd 1757 if (!fAliPhysicsVersion.IsNull()) {
1758 fGridJDL->AddToPackages("AliPhysics", fAliPhysicsVersion,"VO_ALICE", "List of requested packages");
1759 fMergingJDL->AddToPackages("AliPhysics", fAliPhysicsVersion, "VO_ALICE", "List of requested packages");
1760 }
0f389141 1761 if (!fROOTVersion.IsNull()) {
c57f56b7 1762 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1763 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1764 }
1765 if (!fAPIVersion.IsNull()) {
c57f56b7 1766 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1767 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1768 }
648174cf 1769 if (!fExternalPackages.IsNull()) {
1770 arr = fExternalPackages.Tokenize(" ");
1771 TIter next(arr);
1772 while ((os=(TObjString*)next())) {
1773 TString pkgname = os->GetString();
1774 Int_t index = pkgname.Index("::");
1775 TString pkgversion = pkgname(index+2, pkgname.Length());
1776 pkgname.Remove(index);
1777 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1778 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1779 }
1780 delete arr;
1781 }
6da4839e
AG		 1782 if (!fMCLoop) {
1783 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1784 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
6da4839e 1785 }
d7eb1f39
AG		 1786 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1787 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1788 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1789 TString analysisFile = fExecutable;
1790 analysisFile.ReplaceAll(".sh", ".root");
1791 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1792 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1793 if (fAdditionalLibs.Length()) {
1794 arr = fAdditionalLibs.Tokenize(" ");
1795 TIter next(arr);
1796 while ((os=(TObjString*)next())) {
2dffffd6 1797 if (os->GetString().Contains(".so") ||
1798 os->GetString().Contains(".dylib")) continue;
c57f56b7 1799 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1800 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1801 }
1802 delete arr;
1803 }
4e5c5506 1804 if (fPackages) {
1805 TIter next(fPackages);
1806 TObject *obj;
0f389141 1807 while ((obj=next())) {
4e5c5506 1808 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1809 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1810 }
4e5c5506 1811 }
95e16f01 1812 const char *comment = "List of output files and archives";
c57f56b7 1813 if (fOutputArchive.Length()) {
e4d9a069 1814 TString outputArchive = fOutputArchive;
1815 if (!fRegisterExcludes.IsNull()) {
1816 arr = fRegisterExcludes.Tokenize(" ");
1817 TIter next1(arr);
1818 while ((os=(TObjString*)next1())) {
1819 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1820 outputArchive.ReplaceAll(os->GetString(),"");
1821 }
1822 delete arr;
1823 }
1824 arr = outputArchive.Tokenize(" ");
c57f56b7 1825 TIter next(arr);
0f389141 1826 Bool_t first = kTRUE;
0f389141 1827 while ((os=(TObjString*)next())) {
0f389141 1828 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1829 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
0f389141 1830 else
95e16f01 1831 fGridJDL->AddToSet("Output", os->GetString());
1832 if (first) fGridJDL->AddToSetDescription("Output", comment);
0f389141 1833 first = kFALSE;
1834 }
c57f56b7 1835 delete arr;
f790bc1b 1836 // Output archive for the merging jdl
f790bc1b 1837 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1838 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1839 // Add normal output files, extra files + terminate files
0c0825e4 1840 TString files;
1841 if (IsMergeAOD()) files = GetListOfFiles("outaodextter");
1842 else files = GetListOfFiles("outextter");
37d21c01 1843 // Do not register files in fRegisterExcludes
1844 if (!fRegisterExcludes.IsNull()) {
1845 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1846 TIter next1(arr);
1847 while ((os=(TObjString*)next1())) {
1848 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1849 files.ReplaceAll(os->GetString(),"");
1850 }
66d71516 1851 delete arr;
0f389141 1852 }
f790bc1b 1853 files.ReplaceAll(".root", "*.root");
ec5acdce 1854
1855 if (mgr->IsCollectThroughput())
95e16f01 1856 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",files.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 1857 else
1858 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1859 } else {
95f84779 1860 TString files = fOutputArchive;
1861 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1862 outputArchive = files;
f790bc1b 1863 }
0f389141 1864 arr = outputArchive.Tokenize(" ");
705adb3e 1865 TIter next2(arr);
0f389141 1866 first = kTRUE;
705adb3e 1867 while ((os=(TObjString*)next2())) {
0cdf65a8 1868 TString currentfile = os->GetString();
0cdf65a8 1869 if (!currentfile.Contains("@") && fCloseSE.Length())
95e16f01 1870 fMergingJDL->AddToSet("Output", Form("%s@%s",currentfile.Data(), fCloseSE.Data()));
0f389141 1871 else
95e16f01 1872 fMergingJDL->AddToSet("Output", currentfile);
1873 if (first) fMergingJDL->AddToSetDescription("Output", comment);
0f389141 1874 first = kFALSE;
1875 }
1876 delete arr;
c57f56b7 1877 }
149d288c 1878 arr = fOutputFiles.Tokenize(",");
c57f56b7 1879 TIter next(arr);
0f389141 1880 Bool_t first = kTRUE;
43da816a 1881 while ((os=(TObjString*)next())) {
e1eaf596 1882 // Ignore ouputs in jdl that are also in outputarchive
1883 TString sout = os->GetString();
f790bc1b 1884 sout.ReplaceAll("*", "");
1885 sout.ReplaceAll(".root", "");
e1eaf596 1886 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1887 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1888 // Ignore fRegisterExcludes
1889 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1890 if (!first) comment = NULL;
43da816a 1891 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1892 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
43da816a 1893 else
95e16f01 1894 fGridJDL->AddToSet("Output", os->GetString());
1895 if (first) fGridJDL->AddToSetDescription("Output", comment);
f790bc1b 1896 if (fMergeExcludes.Contains(sout)) continue;
1897 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1898 fMergingJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
f790bc1b 1899 else
95e16f01 1900 fMergingJDL->AddToSet("Output", os->GetString());
1901 if (first) fMergingJDL->AddToSetDescription("Output", comment);
1902 first = kFALSE;
43da816a 1903 }
c57f56b7 1904 delete arr;
0f389141 1905 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1906 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1907 TString validationScript = fValidationScript;
0f389141 1908 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1909 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1910 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1911 if (fMasterResubmitThreshold) {
1912 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1913 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1914 }
d2a409b2 1915 // Write a jdl with 2 input parameters: collection name and output dir name.
1916 WriteJDL(copy);
c57f56b7 1917 }
1918 // Copy jdl to grid workspace
a8739e8a 1919 if (copy) {
b5e4aaa7 1920 // Check if an output directory was defined and valid
1921 if (!fGridOutputDir.Length()) {
1922 Error("CreateJDL", "You must define AliEn output directory");
1923 return kFALSE;
1924 } else {
1925 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1926 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1927 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1928 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1929 } else {
1930 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1931 return kFALSE;
1932 }
1933 }
1934 gGrid->Cd(workdir);
1935 }
648174cf 1936 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1937 TString mergeJDLName = fExecutable;
1938 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1939 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1940 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1941 if (fProductionMode) {
648174cf 1942 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1943 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1944 }
648174cf 1945 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1946 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1947 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1948 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1949 Fatal("","Terminating");
1950// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1951 if (fMergeViaJDL) {
1952 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1953// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1954 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1955 Fatal("","Terminating");
0f389141 1956 }
648174cf 1957 }
c57f56b7 1958 if (fAdditionalLibs.Length()) {
e7c71df0 1959 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1960 TObjString *os;
1961 TIter next(arr);
1962 while ((os=(TObjString*)next())) {
2dffffd6 1963 if (os->GetString().Contains(".so") ||
1964 os->GetString().Contains(".dylib")) continue;
4e5c5506 1965 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1966 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1967// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1968 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1969 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1970 Fatal("","Terminating");
c57f56b7 1971 }
1972 delete arr;
1973 }
4e5c5506 1974 if (fPackages) {
1975 TIter next(fPackages);
1976 TObject *obj;
1977 while ((obj=next())) {
fdbbc7be 1978 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1979 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1980// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1981 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1982 Form("%s/%s", workdir.Data(), obj->GetName())))
1983 Fatal("","Terminating");
4e5c5506 1984 }
1985 }
c57f56b7 1986 }
1987 return kTRUE;
1988}
1989
a8739e8a 1990//______________________________________________________________________________
d2a409b2 1991Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1992{
1993// Writes one or more JDL's corresponding to findex. If findex is negative,
1994// all run numbers are considered in one go (jdl). For non-negative indices
1995// they correspond to the indices in the array fInputFiles.
dec33edc 1996 if (!fInputFiles && !fMCLoop) return kFALSE;
d3b18c4c 1997 TObject *os;
1998 TString workdir;
1999 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 2000 workdir += fGridWorkingDir;
f47d5cb4 2001 TString stageName = "$2";
2002 if (fProductionMode) stageName = "$4";
2003 if (!fMergeDirName.IsNull()) {
2004 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
2005 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
2006 } else {
2007 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
2008 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
2009 }
d3b18c4c 2010 if (fProductionMode) {
a8739e8a 2011 TIter next(fInputFiles);
b3e07543 2012 while ((os=next())) {
d3b18c4c 2013 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 2014 }
56754bf6 2015 if (!fOutputToRunNo)
2016 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
2017 else
2018 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 2019 } else {
2020 if (!fRunNumbers.Length() && !fRunRange[0]) {
2021 // One jdl with no parameters in case input data is specified by name.
2022 TIter next(fInputFiles);
2023 while ((os=next()))
2024 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
2025 if (!fOutputSingle.IsNull())
2026 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
2027 else {
2028 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
b1dade98 2029// fMergingJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 2030 }
2031 } else {
2032 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
2033 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
2034 if (!fOutputSingle.IsNull()) {
2035 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
2036 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
2037 } else {
2038 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 2039 }
2040 }
a8739e8a 2041 }
2042
a8739e8a 2043 // Generate the JDL as a string
2044 TString sjdl = fGridJDL->Generate();
0f389141 2045 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 2046 // Final merge jdl
2047 if (!fMergeDirName.IsNull()) {
2048 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
2049 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
2050 } else {
2051 fMergingJDL->SetOutputDirectory("$1", "Output directory");
2052 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
2053 }
b3e07543 2054 TString sjdl2 = fMergingJDL->Generate();
2055 Int_t index, index1;
95e16f01 2056 sjdl.ReplaceAll("\",\"", "\",\n \"");
a8739e8a 2057 sjdl.ReplaceAll("(member", "\n (member");
2058 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2059 sjdl.ReplaceAll("{", "{\n ");
2060 sjdl.ReplaceAll("};", "\n};");
2061 sjdl.ReplaceAll("{\n \n", "{\n");
2062 sjdl.ReplaceAll("\n\n", "\n");
2063 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2064 sjdl1.ReplaceAll("\",\"", "\",\n \"");
0f389141 2065 sjdl1.ReplaceAll("(member", "\n (member");
2066 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2067 sjdl1.ReplaceAll("{", "{\n ");
2068 sjdl1.ReplaceAll("};", "\n};");
2069 sjdl1.ReplaceAll("{\n \n", "{\n");
2070 sjdl1.ReplaceAll("\n\n", "\n");
2071 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2072 sjdl2.ReplaceAll("\",\"", "\",\n \"");
b3e07543 2073 sjdl2.ReplaceAll("(member", "\n (member");
2074 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2075 sjdl2.ReplaceAll("{", "{\n ");
2076 sjdl2.ReplaceAll("};", "\n};");
2077 sjdl2.ReplaceAll("{\n \n", "{\n");
2078 sjdl2.ReplaceAll("\n\n", "\n");
2079 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 2080 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 2081 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 2082 index = sjdl.Index("JDLVariables");
2083 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 2084 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2085 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 2086 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 2087 index = fJobTag.Index(":");
2088 if (index < 0) index = fJobTag.Length();
2089 TString jobTag = fJobTag;
f47d5cb4 2090 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 2091 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 2092 if (fProductionMode) {
2093 sjdl1.Prepend("# Generated merging jdl (production mode) \
2094 \n# $1 = full alien path to output directory to be merged \
2095 \n# $2 = train number \
2096 \n# $3 = production (like LHC10b) \
2097 \n# $4 = merging stage \
2098 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2099 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2100 sjdl2.Prepend("# Generated merging jdl \
2101 \n# $1 = full alien path to output directory to be merged \
2102 \n# $2 = train number \
2103 \n# $3 = production (like LHC10b) \
2104 \n# $4 = merging stage \
2105 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2106 } else {
2107 sjdl1.Prepend("# Generated merging jdl \
2108 \n# $1 = full alien path to output directory to be merged \
2109 \n# $2 = merging stage \
03335324 2110 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2111 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2112 sjdl2.Prepend("# Generated merging jdl \
2113 \n# $1 = full alien path to output directory to be merged \
2114 \n# $2 = merging stage \
03335324 2115 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2116 }
0f389141 2117 index = sjdl1.Index("JDLVariables");
2118 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 2119 index = sjdl2.Index("JDLVariables");
2120 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 2121 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2122 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
2123 index = sjdl2.Index("Split =");
2124 if (index>=0) {
2125 index1 = sjdl2.Index("\n", index);
2126 sjdl2.Remove(index, index1-index+1);
2127 }
2128 index = sjdl2.Index("SplitMaxInputFileNumber");
2129 if (index>=0) {
2130 index1 = sjdl2.Index("\n", index);
2131 sjdl2.Remove(index, index1-index+1);
2132 }
2133 index = sjdl2.Index("InputDataCollection");
2134 if (index>=0) {
2135 index1 = sjdl2.Index(";", index);
2136 sjdl2.Remove(index, index1-index+1);
2137 }
2138 index = sjdl2.Index("InputDataListFormat");
2139 if (index>=0) {
2140 index1 = sjdl2.Index("\n", index);
2141 sjdl2.Remove(index, index1-index+1);
2142 }
2143 index = sjdl2.Index("InputDataList");
2144 if (index>=0) {
2145 index1 = sjdl2.Index("\n", index);
2146 sjdl2.Remove(index, index1-index+1);
2147 }
f47d5cb4 2148 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2149 // Write jdl to file
a8739e8a 2150 ofstream out;
d2a409b2 2151 out.open(fJDLName.Data(), ios::out);
a8739e8a 2152 if (out.bad()) {
d3b18c4c 2153 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2154 return kFALSE;
2155 }
2156 out << sjdl << endl;
b3e07543 2157 out.close();
0f389141 2158 TString mergeJDLName = fExecutable;
2159 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2160 if (fMergeViaJDL) {
2161 ofstream out1;
2162 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2163 if (out1.bad()) {
d3b18c4c 2164 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2165 return kFALSE;
2166 }
2167 out1 << sjdl1 << endl;
b3e07543 2168 out1.close();
2169 ofstream out2;
2170 TString finalJDL = mergeJDLName;
2171 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2172 out2.open(finalJDL.Data(), ios::out);
2173 if (out2.bad()) {
2174 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2175 return kFALSE;
2176 }
2177 out2 << sjdl2 << endl;
2178 out2.close();
0f389141 2179 }
a8739e8a 2180
2181 // Copy jdl to grid workspace
2182 if (!copy) {
d3b18c4c 2183 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2184 } else {
d2a409b2 2185 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2186 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2187 TString finalJDL = mergeJDLName;
2188 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2189 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2190 if (fProductionMode) {
b5fe9cba 2191 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2192 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2193 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2194 }
d2a409b2 2195 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2196 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2197 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2198 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2199// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2200 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2201 Fatal("","Terminating");
0f389141 2202 if (fMergeViaJDL) {
b3e07543 2203 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2204// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2205// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2206 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2207 Fatal("","Terminating");
2208 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2209 Fatal("","Terminating");
0f389141 2210 }
a8739e8a 2211 }
2212 return kTRUE;
2213}
2214
c57f56b7 2215//______________________________________________________________________________
5513444a 2216Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2217{
2218// Returns true if file exists.
5513444a 2219 if (!gGrid) return kFALSE;
2ea55496 2220 TString slfn = lfn;
2221 slfn.ReplaceAll("alien://","");
2222 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2223 if (!res) return kFALSE;
2224 TMap *map = dynamic_cast<TMap*>(res->At(0));
2225 if (!map) {
2226 delete res;
2227 return kFALSE;
2228 }
2229 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2230 if (!objs || !objs->GetString().Length()) {
2231 delete res;
2232 return kFALSE;
2233 }
2234 delete res;
2235 return kTRUE;
2236}
2237
923e2ca5 2238//______________________________________________________________________________
2239Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2240{
2241// Returns true if directory exists. Can be also a path.
d7aac489 2242// Since there is not API in TAlien, we use the Cd trick:
923e2ca5 2243 if (!gGrid) return kFALSE;
d7aac489 2244 // Backup current path
2245 TString cpath = gGrid->Pwd();
2246 TString command = "cd ";
2247 TString sdir(dirname);
2248 sdir.ReplaceAll("alien://", "");
2249 command += sdir;
2250 TGridResult *res = gGrid->Command(command);
2251 if (!res) {
2252 gGrid->Cd(cpath);
2253 return kFALSE;
2254 }
2255 TMap *map = (TMap*)res->At(0);
2256 if (!map) {
2257 gGrid->Cd(cpath);
2258 delete res;
2259 return kFALSE;
923e2ca5 2260 }
d7aac489 2261 TString sval = map->GetValue("__result__")->GetName();
2262 Bool_t retval = (Bool_t)sval.Atoi();
2263 gGrid->Cd(cpath);
923e2ca5 2264 delete res;
d7aac489 2265 return retval;
2266}
923e2ca5 2267
c57f56b7 2268//______________________________________________________________________________
a2f5fc01 2269void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2270{
2271// Check input data type.
a2f5fc01 2272 isCollection = kFALSE;
2273 isXml = kFALSE;
2274 useTags = kFALSE;
c57f56b7 2275 if (!gGrid) {
2276 Error("CheckDataType", "No connection to grid");
2277 return;
2278 }
a2f5fc01 2279 isCollection = IsCollection(lfn);
c57f56b7 2280 TString msg = "\n##### file: ";
2281 msg += lfn;
a2f5fc01 2282 if (isCollection) {
c57f56b7 2283 msg += " type: raw_collection;";
2284 // special treatment for collections
a2f5fc01 2285 isXml = kFALSE;
c57f56b7 2286 // check for tag files in the collection
2287 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2288 if (!res) {
2289 msg += " using_tags: No (unknown)";
23329835 2290 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2291 return;
2292 }
2293 const char* typeStr = res->GetKey(0, "origLFN");
2294 if (!typeStr || !strlen(typeStr)) {
2295 msg += " using_tags: No (unknown)";
23329835 2296 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2297 return;
2298 }
2299 TString file = typeStr;
a2f5fc01 2300 useTags = file.Contains(".tag");
2301 if (useTags) msg += " using_tags: Yes";
c57f56b7 2302 else msg += " using_tags: No";
23329835 2303 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2304 return;
2305 }
2306 TString slfn(lfn);
2307 slfn.ToLower();
a2f5fc01 2308 isXml = slfn.Contains(".xml");
2309 if (isXml) {
c57f56b7 2310 // Open xml collection and check if there are tag files inside
2311 msg += " type: xml_collection;";
2312 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2313 if (!coll) {
2314 msg += " using_tags: No (unknown)";
23329835 2315 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2316 return;
2317 }
2318 TMap *map = coll->Next();
2319 if (!map) {
2320 msg += " using_tags: No (unknown)";
23329835 2321 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2322 return;
2323 }
2324 map = (TMap*)map->GetValue("");
2325 TString file;
2326 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2327 useTags = file.Contains(".tag");
c57f56b7 2328 delete coll;
a2f5fc01 2329 if (useTags) msg += " using_tags: Yes";
c57f56b7 2330 else msg += " using_tags: No";
23329835 2331 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2332 return;
2333 }
a2f5fc01 2334 useTags = slfn.Contains(".tag");
c57f56b7 2335 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2336 else msg += " type: unknown file;";
a2f5fc01 2337 if (useTags) msg += " using_tags: Yes";
c57f56b7 2338 else msg += " using_tags: No";
23329835 2339 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2340}
2341
4e5c5506 2342//______________________________________________________________________________
2343void AliAnalysisAlien::EnablePackage(const char *package)
2344{
2345// Enables a par file supposed to exist in the current directory.
2346 TString pkg(package);
2347 pkg.ReplaceAll(".par", "");
2348 pkg += ".par";
2349 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2350 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2351 return;
2352 }
2353 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2354 Info("EnablePackage", "AliEn plugin will use .par packages");
2355 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2356 if (!fPackages) {
2357 fPackages = new TObjArray();
2358 fPackages->SetOwner();
2359 }
2360 fPackages->Add(new TObjString(pkg));
2361}
2362
3bdcb562 2363//______________________________________________________________________________
2364TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2365{
2366// Make a tree from files having the location specified in fFileForTestMode.
2367// Inspired from JF's CreateESDChain.
2368 if (fFileForTestMode.IsNull()) {
2369 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2370 return NULL;
2371 }
2372 if (gSystem->AccessPathName(fFileForTestMode)) {
2373 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2374 return NULL;
2375 }
2376 // Open the file
2377 ifstream in;
2378 in.open(fFileForTestMode);
2379 Int_t count = 0;
2380 // Read the input list of files and add them to the chain
b505f1f8 2381 TString line;
d6c60197 2382 TString streeName(treeName);
2383 if (IsUseMCchain()) streeName = "TE";
2384 TChain *chain = new TChain(streeName);
06f67114 2385 TList *friends = new TList();
2386 TChain *cfriend = 0;
2387 if (!fFriendChainName.IsNull()) {
2388 TObjArray *list = fFriendChainName.Tokenize(" ");
2389 TIter next(list);
2390 TObjString *str;
2391 while((str=(TObjString*)next())) {
2392 cfriend = new TChain(streeName, str->GetName());
2393 friends->Add(cfriend);
2394 chain->AddFriend(cfriend);
2395 }
2396 delete list;
2397 }
2398 TString bpath;
2399 TIter nextfriend(friends);
b505f1f8 2400 while (in.good())
2401 {
3bdcb562 2402 in >> line;
ffbe06b9 2403 if (line.IsNull() || line.BeginsWith("#")) continue;
3bdcb562 2404 if (count++ == fNtestFiles) break;
2405 TString esdFile(line);
2406 TFile *file = TFile::Open(esdFile);
b505f1f8 2407 if (file && !file->IsZombie()) {
2408 chain->Add(esdFile);
3bdcb562 2409 file->Close();
b505f1f8 2410 if (!fFriendChainName.IsNull()) {
4b5d9e21 2411 if (esdFile.Index("#") > -1)
2412 esdFile.Remove(esdFile.Index("#"));
06f67114 2413 bpath = gSystem->DirName(esdFile);
2414 bpath += "/";
2415 TString fileFriend;
2416 nextfriend.Reset();
2417 while ((cfriend=(TChain*)nextfriend())) {
2418 fileFriend = bpath;
2419 fileFriend += cfriend->GetTitle();
2420 file = TFile::Open(fileFriend);
2421 if (file && !file->IsZombie()) {
2422 file->Close();
2423 cfriend->Add(fileFriend);
2424 } else {
2425 Fatal("GetChainForTestMode", "Cannot open friend file: %s", fileFriend.Data());
2426 return 0;
2427 }
2428 }
4b5d9e21 2429 }
3bdcb562 2430 } else {
2431 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2432 }
b505f1f8 2433 }
2434 in.close();
2435 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2436 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2437 delete chain;
06f67114 2438 friends->Delete();
2439 delete friends;
3bdcb562 2440 return NULL;
b505f1f8 2441 }
b505f1f8 2442 return chain;
3bdcb562 2443}
2444
d2a409b2 2445//______________________________________________________________________________
2446const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2447{
2448// Get job status for all jobs with jobid>jobidstart.
2449 static char mstatus[20];
2450 mstatus[0] = '\0';
2451 nrunning = 0;
2452 nwaiting = 0;
2453 nerror = 0;
2454 ndone = 0;
2455 TGridJobStatusList *list = gGrid->Ps("");
2456 if (!list) return mstatus;
2457 Int_t nentries = list->GetSize();
2458 TGridJobStatus *status;
2459 Int_t pid;
2460 for (Int_t ijob=0; ijob<nentries; ijob++) {
2461 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2462 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2463 if (pid<jobidstart) continue;
2464 if (pid == lastid) {
8ddad121 2465 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2466 }
2467 switch (status->GetStatus()) {
2468 case TGridJobStatus::kWAITING:
2469 nwaiting++; break;
2470 case TGridJobStatus::kRUNNING:
2471 nrunning++; break;
2472 case TGridJobStatus::kABORTED:
2473 case TGridJobStatus::kFAIL:
2474 case TGridJobStatus::kUNKNOWN:
2475 nerror++; break;
2476 case TGridJobStatus::kDONE:
2477 ndone++;
2478 }
2479 }
2480 list->Delete();
2481 delete list;
2482 return mstatus;
2483}
2484
c57f56b7 2485//______________________________________________________________________________
2486Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2487{
2488// Returns true if file is a collection. Functionality duplicated from
2489// TAlien::Type() because we don't want to directly depend on TAlien.
2490 if (!gGrid) {
2491 Error("IsCollection", "No connection to grid");
2492 return kFALSE;
2493 }
2494 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2495 if (!res) return kFALSE;
2496 const char* typeStr = res->GetKey(0, "type");
2497 if (!typeStr || !strlen(typeStr)) return kFALSE;
2498 if (!strcmp(typeStr, "collection")) return kTRUE;
2499 delete res;
2500 return kFALSE;
2501}
2502
fe2d7fc2 2503//______________________________________________________________________________
2504Bool_t AliAnalysisAlien::IsSingleOutput() const
2505{
2506// Check if single-ouput option is on.
2507 return (!fOutputSingle.IsNull());
2508}
715da79c 2509
2510//______________________________________________________________________________
2511Long64_t AliAnalysisAlien::RunMacroAndExtractLibs(const char* macro, const char *args, TString &libs)
2512{
2513// Tries to run the specified macro and return the libraries that it loads.
2514 TString expname;
2515 if (strlen(macro)) expname = gSystem->ExpandPathName(macro);
2516 if (expname.IsNull() || gSystem->AccessPathName(expname)) {
2517 ::Error("RunMacroAndExtractLibs","Cannot find macro %s in current directory", macro);
2518 return -1;
2519 }
2520 TString oldlibs = gSystem->GetLibraries();
2521 TMacro m(expname);
2522 Int_t error = 0;
2523 Long64_t retval = m.Exec(args, &error);
2524 if (error != TInterpreter::kNoError)
2525 {
2526 ::Error("RunMacroAndExtractLibs", "Macro interpretation %s failed", macro);
2527 return -1;
2528 }
2529 libs = gSystem->GetLibraries();
2530 libs.ReplaceAll(oldlibs, "");
2531 libs.Strip(TString::kLeading);
2532 TObjArray *libTokens = libs.Tokenize(" ");
2533 libs = "";
2534 for (Int_t i=0; i<libTokens->GetEntries(); i++) {
2535 if (!libs.IsNull()) libs += " ";
2536 libs += gSystem->BaseName(libTokens->At(i)->GetName());
2537 }
2538 delete libTokens;
2539 return retval;
2540}
2541
16a4353c 2542//______________________________________________________________________________
2543void AliAnalysisAlien::Print(Option_t *) const
2544{
2545// Print current plugin settings.
84fcd93f 2546 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2547 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2548 if (mgr && mgr->IsProofMode()) {
2549 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2550 if (TestBit(AliAnalysisGrid::kTest))
2551 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2552 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2553 if (!fProofDataSet.IsNull())
2554 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2555 if (fProofReset==1)
2556 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2557 if (fProofReset>1)
2558 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
874de7ea 2559 if (!fROOTVersion.IsNull())
2560 printf("= ROOT version requested________________________ %s\n", fROOTVersion.Data());
d3b18c4c 2561 else
2562 printf("= ROOT version requested________________________ default\n");
2563 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
af78fedd 2564 printf("= AliPhysics version requested__________________ %s\n", fAliPhysicsVersion.Data());
d3b18c4c 2565 if (!fAliRootMode.IsNull())
2566 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2567 if (fNproofWorkers)
2568 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2569 if (fNproofWorkersPerSlave)
2570 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2571 if (TestSpecialBit(kClearPackages))
2572 printf("= ClearPackages requested...\n");
2573 if (fIncludePath.Data())
2574 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2575 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2576 if (fPackages && fPackages->GetEntries()) {
2577 TIter next(fPackages);
2578 TObject *obj;
2579 TString list;
2580 while ((obj=next())) list += obj->GetName();
2581 printf("= Par files to be used: ________________________ %s\n", list.Data());
2582 }
2583 if (TestSpecialBit(kProofConnectGrid))
2584 printf("= Requested PROOF connection to grid\n");
2585 return;
2586 }
e1c22e21 2587 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2588 if (fOverwriteMode) {
2589 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2590 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2591 }
348be253 2592 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
780ec36d 2593 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO:Print");
84fcd93f 2594 printf("= Production mode:______________________________ %d\n", fProductionMode);
2595 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2596 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2597 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
af78fedd 2598 printf("= Version of AliPhysics requested: _____________ %s\n", fAliPhysicsVersion.Data());
16a4353c 2599 if (fUser.Length())
84fcd93f 2600 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2601 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2602 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
780ec36d 2603 TString basedatadir = fGridDataDir;
2604 TString pattern = fDataPattern;
2605 pattern.Strip();
2606 Int_t ind = pattern.Index(" ");
2607 if (ind>=0) {
2608 basedatadir += "/%run%/";
2609 basedatadir += pattern(0, ind);
2610 pattern = pattern(ind+1, pattern.Length());
2611 }
2612 printf("= Data base directory path requested: __________ %s\n", basedatadir.Data());
2613 printf("= Data search pattern: _________________________ %s\n", pattern.Data());
84fcd93f 2614 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2615 if (fRunNumbers.Length())
84fcd93f 2616 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2617 if (fRunRange[0])
74013535 2618 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2619 if (!fRunRange[0] && !fRunNumbers.Length()) {
2620 TIter next(fInputFiles);
2621 TObject *obj;
2622 TString list;
2623 while ((obj=next())) list += obj->GetName();
84fcd93f 2624 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2625 }
2626 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2627 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2628 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2629 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2630 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2631 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2632 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2633 printf("=====================================================================\n");
2634 printf("= Job price: ___________________________________ %d\n", fPrice);
2635 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2636 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2637 if (fMaxInitFailed>0)
84fcd93f 2638 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2639 if (fMasterResubmitThreshold>0)
84fcd93f 2640 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2641 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2642 if (fNrunsPerMaster>0)
84fcd93f 2643 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2644 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2645 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2646 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2647 if (fArguments.Length())
84fcd93f 2648 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2649 if (fExecutableArgs.Length())
2650 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2651 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2652 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2653 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2654 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2655 if (fDatasetName)
84fcd93f 2656 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2657 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2658 if (fIncludePath.Data())
84fcd93f 2659 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2660 if (fCloseSE.Length())
84fcd93f 2661 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2662 if (fFriendChainName.Length())
84fcd93f 2663 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2664 if (fPackages && fPackages->GetEntries()) {
16a4353c 2665 TIter next(fPackages);
2666 TObject *obj;
2667 TString list;
2668 while ((obj=next())) list += obj->GetName();
84fcd93f 2669 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2670 }
2671}
2672
c57f56b7 2673//______________________________________________________________________________
2674void AliAnalysisAlien::SetDefaults()
2675{
2676// Set default values for everything. What cannot be filled will be left empty.
2677 if (fGridJDL) delete fGridJDL;
2678 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2679 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2680 fPrice = 1;
2681 fTTL = 30000;
2682 fSplitMaxInputFileNumber = 100;
2683 fMaxInitFailed = 0;
2684 fMasterResubmitThreshold = 0;
bb885a9e 2685 fNtestFiles = 10;
149d288c 2686 fNreplicas = 2;
a8739e8a 2687 fRunRange[0] = 0;
2688 fRunRange[1] = 0;
8e3bd06a 2689 fRunPrefix = "%d";
319593fb 2690 fNrunsPerMaster = 1;
16a4353c 2691 fMaxMergeFiles = 100;
c57f56b7 2692 fRunNumbers = "";
2693 fExecutable = "analysis.sh";
5aa94dc3 2694 fExecutableCommand = "root -b -q -x";
c57f56b7 2695 fArguments = "";
631c0b05 2696 fExecutableArgs = "";
c57f56b7 2697 fAnalysisMacro = "myAnalysis.C";
2698 fAnalysisSource = "";
2699 fAdditionalLibs = "";
2700 fSplitMode = "se";
2701 fAPIVersion = "";
2702 fROOTVersion = "";
2703 fAliROOTVersion = "";
af78fedd 2704 fAliPhysicsVersion = "";
c57f56b7 2705 fUser = ""; // Your alien user name
2706 fGridWorkingDir = "";
2707 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2708 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2709 fFriendChainName = "";
c57f56b7 2710 fGridOutputDir = "output";
b83f84f0 2711 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2712 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2713 fInputFormat = "xml-single";
2714 fJDLName = "analysis.jdl";
c6cb3634 2715 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2716 fMergeExcludes = "";
0f389141 2717 fMergeViaJDL = 0;
348be253 2718 SetUseCopy(kTRUE);
2719 SetCheckCopy(kTRUE);
149d288c 2720 SetDefaultOutputs(kTRUE);
e1c22e21 2721 fOverwriteMode = 1;
c57f56b7 2722}
2723
a41b2882 2724//______________________________________________________________________________
2725void AliAnalysisAlien::SetFriendChainName(const char *name, const char *libnames)
2726{
2727 // Set file name for the chain of friends and optionally additional libs to be loaded.
2728 // Libs should be separated by blancs.
2729 fFriendChainName = name;
06f67114 2730 fFriendChainName.ReplaceAll(",", " ");
2731 fFriendChainName.Strip();
2732 fFriendChainName.ReplaceAll(" ", " ");
2733
a41b2882 2734 fFriendLibs = libnames;
1a3f255c 2735 if (fFriendLibs.Length()) {
2dffffd6 2736 if(!fFriendLibs.Contains(".so") &&
2737 !fFriendLibs.Contains(".dylib"))
1a3f255c 2738 Fatal("SetFriendChainName()", "You should provide explicit library names (with extension)");
2739 fFriendLibs.ReplaceAll(",", " ");
2740 fFriendLibs.Strip();
2741 fFriendLibs.ReplaceAll(" ", " ");
a41b2882 2742 }
2743}
2744
874de7ea 2745//______________________________________________________________________________
2746void AliAnalysisAlien::SetRootVersionForProof(const char *version)
2747{
2748// Obsolete method. Use SetROOTVersion instead
2749 Warning("SetRootVersionForProof", "Obsolete. Use SetROOTVersion instead");
95e16f01 2750 if (fROOTVersion.IsNull()) SetROOTVersion(version);
2751 else Error("SetRootVersionForProof", "ROOT version already set to %s", fROOTVersion.Data());
874de7ea 2752}
2753
0f389141 2754//______________________________________________________________________________
b3e07543 2755Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2756{
b3e07543 2757// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2758 // First check if the result is already in the output directory.
2759 if (FileExists(Form("%s/%s",aliendir,filename))) {
2760 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2761 return kFALSE;
2762 }
b3e07543 2763 // Now check the last stage done.
2764 Int_t stage = 0;
2765 while (1) {
2766 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2767 stage++;
2768 }
2769 // Next stage of merging
2770 stage++;
2771 TString pattern = "*root_archive.zip";
2772 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2773 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2774 if (res) delete res;
2775 // Write standard output to file
2776 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2777 // Count the number of files inside
2778 ifstream ifile;
2779 ifile.open(Form("Stage_%d.xml",stage));
2780 if (!ifile.good()) {
2781 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2782 return kFALSE;
b3e07543 2783 }
2784 TString line;
2785 Int_t nfiles = 0;
2786 while (!ifile.eof()) {
2787 ifile >> line;
2788 if (line.Contains("/event")) nfiles++;
7c2cd90a 2789 }
b3e07543 2790 ifile.close();
2791 if (!nfiles) {
2792 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2793 return kFALSE;
2794 } else {
2795 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2796 }
b3e07543 2797 // Copy the file in the output directory
2798 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2799// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2800 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2801 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2802 // Check if this is the last stage to be done.
2803 Bool_t laststage = (nfiles<nperchunk);
2804 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
0ad3ea1f 2805 Int_t jobId = 0;
b3e07543 2806 if (laststage) {
2807 printf("### Submiting final merging stage %d\n", stage);
2808 TString finalJDL = jdl;
2809 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2810 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
0ad3ea1f 2811 jobId = SubmitSingleJob(query);
b3e07543 2812 } else {
2813 printf("### Submiting merging stage %d\n", stage);
03335324 2814 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
0ad3ea1f 2815 jobId = SubmitSingleJob(query);
7c2cd90a 2816 }
0ad3ea1f 2817 if (!jobId) return kFALSE;
2818
2819 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
2820 fGridJobIDs.Append(Form("%d", jobId));
2821 if (!fGridStages.IsNull()) fGridStages.Append(" ");
2822 fGridStages.Append(Form("%s_merge_stage%d",
2823 laststage ? "final" : "partial", stage));
2824
b3e07543 2825 return kTRUE;
2826}
7c2cd90a 2827
105631b5 2828//______________________________________________________________________________
2829AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2830{
2831// Loat the analysis manager from a file.
2832 TFile *file = TFile::Open(fname);
2833 if (!file) {
2834 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2835 return 0;
2836 }
2837 TIter nextkey(file->GetListOfKeys());
2838 AliAnalysisManager *mgr = 0;
2839 TKey *key;
2840 while ((key=(TKey*)nextkey())) {
2841 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2842 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2843 }
2844 if (!mgr)
2845 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2846 return mgr;
2847}
2848
7c2cd90a 2849//______________________________________________________________________________
2850Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2851{
2852// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2853 if (!gGrid) return 0;
2854 printf("=> %s ------> ",query);
2855 TGridResult *res = gGrid->Command(query);
2856 if (!res) return 0;
2857 TString jobId = res->GetKey(0,"jobId");
2858 delete res;
2859 if (jobId.IsNull()) {
2860 printf("submission failed. Reason:\n");
2861 gGrid->Stdout();
2862 gGrid->Stderr();
2863 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2864 return 0;
2865 }
0ad3ea1f 2866 Int_t ijobId = jobId.Atoi();
2867 printf(" Job id: '%s' (%d)\n", jobId.Data(), ijobId);
2868 return ijobId;
7c2cd90a 2869}
2870
ec5acdce 2871//______________________________________________________________________________
2872Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2873{
2874// Merges a collection of output files using concatenation.
d66aa2a9 2875 TString scoll(collection);
2876 if (!scoll.Contains(".xml")) return kFALSE;
ec5acdce 2877 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2878 if (!coll) {
2879 ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2880 return kFALSE;
2881 }
2882 // Iterate grid collection
2883 TString outtmp;
2884 Bool_t merged = kFALSE;
2885 Int_t ifile = 0;
2886 while (coll->Next()) {
2887 TString fname = gSystem->DirName(coll->GetTURL());
2888 fname += "/";
2889 fname += output;
2890 outtmp = Form("%d_%s", ifile, output);
2891 if (!TFile::Cp(fname, outtmp)) {
2892 ::Error("MergeInfo", "Could not copy %s", fname.Data());
2893 continue;
2894 }
2895 ifile++;
2896 if (ifile<2) {
2897 gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2898 continue;
2899 }
2900 gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2901 gSystem->Exec("cp tempmerged lastmerged");
2902 }
2903 if (ifile) {
2904 gSystem->Exec(Form("cp lastmerged %s", output));
2905 gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2906 merged = kTRUE;
2907 }
2908 return merged;
2909}
2910
7c2cd90a 2911//______________________________________________________________________________
b3e07543 2912Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2913{
b3e07543 2914// Merge given output files from basedir. Basedir can be an alien output directory
2915// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2916// files in a group (ignored for xml input). Merging can be done in stages:
2917// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2918// stage=1 : works with an xml of all root_archive.zip in the output directory
2919// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2920 TString outputFile = output;
0f389141 2921 TString command;
a2f5fc01 2922 TString outputChunk;
2923 TString previousChunk = "";
b3e07543 2924 TObjArray *listoffiles = new TObjArray();
2925// listoffiles->SetOwner();
a2f5fc01 2926 Int_t countChunk = 0;
2927 Int_t countZero = nmaxmerge;
0f389141 2928 Bool_t merged = kTRUE;
d66aa2a9 2929 Bool_t isGrid = kTRUE;
a2f5fc01 2930 Int_t index = outputFile.Index("@");
2931 if (index > 0) outputFile.Remove(index);
7c2cd90a 2932 TString inputFile = outputFile;
b3e07543 2933 TString sbasedir = basedir;
2934 if (sbasedir.Contains(".xml")) {
2935 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2936 nmaxmerge = 9999999;
2937 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2938 if (!coll) {
2939 ::Error("MergeOutput", "Input XML collection empty.");
2940 return kFALSE;
2941 }
2942 // Iterate grid collection
2943 while (coll->Next()) {
2944 TString fname = gSystem->DirName(coll->GetTURL());
2945 fname += "/";
2946 fname += inputFile;
2947 listoffiles->Add(new TNamed(fname.Data(),""));
2948 }
ffbe06b9 2949 } else if (sbasedir.Contains(".txt")) {
d66aa2a9 2950 // The file having the .txt extension is expected to contain a list of
2951 // folders where the output files will be looked. For alien folders,
2952 // the full folder LFN is expected (starting with alien://)
ffbe06b9 2953 // Assume lfn's on each line
2954 TString line;
2955 ifstream in;
2956 in.open(sbasedir);
d66aa2a9 2957 if (in.fail()) {
2958 ::Error("MergeOutput", "File %s cannot be opened. Merging stopped." ,sbasedir.Data());
2959 return kTRUE;
2960 }
ffbe06b9 2961 Int_t nfiles = 0;
2962 while (in.good()) {
2963 in >> line;
2964 if (line.IsNull() || line.BeginsWith("#")) continue;
d66aa2a9 2965 line.Strip();
2966 if (!line.Contains("alien:")) isGrid = kFALSE;
2967 line += "/";
2968 line += outputFile;
ffbe06b9 2969 nfiles++;
2970 listoffiles->Add(new TNamed(line.Data(),""));
2971 }
d66aa2a9 2972 in.close();
ffbe06b9 2973 if (!nfiles) {
2974 ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data());
2975 delete listoffiles;
2976 return kFALSE;
2977 }
b3e07543 2978 } else {
2979 command = Form("find %s/ *%s", basedir, inputFile.Data());
2980 printf("command: %s\n", command.Data());
2981 TGridResult *res = gGrid->Command(command);
2982 if (!res) {
2983 ::Error("MergeOutput","No result for the find command\n");
2984 delete listoffiles;
2985 return kFALSE;
2986 }
2987 TIter nextmap(res);
2988 TMap *map = 0;
2989 while ((map=(TMap*)nextmap())) {
2990 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2991 if (!objs || !objs->GetString().Length()) {
2992 // Nothing found - skip this output
2993 delete res;
2994 delete listoffiles;
2995 return kFALSE;
2996 }
2997 listoffiles->Add(new TNamed(objs->GetName(),""));
2998 }
2999 delete res;
3000 }
3001 if (!listoffiles->GetEntries()) {
7c2cd90a 3002 ::Error("MergeOutput","No result for the find command\n");
b3e07543 3003 delete listoffiles;
0f389141 3004 return kFALSE;
3005 }
3006
3007 TFileMerger *fm = 0;
b3e07543 3008 TIter next0(listoffiles);
3009 TObjArray *listoffilestmp = new TObjArray();
3010 listoffilestmp->SetOwner();
3011 TObject *nextfile;
3012 TString snextfile;
3013 // Keep only the files at upper level
3014 Int_t countChar = 0;
3015 while ((nextfile=next0())) {
3016 snextfile = nextfile->GetName();
3017 Int_t crtCount = snextfile.CountChar('/');
3018 if (nextfile == listoffiles->First()) countChar = crtCount;
3019 if (crtCount < countChar) countChar = crtCount;
3020 }
3021 next0.Reset();
3022 while ((nextfile=next0())) {
3023 snextfile = nextfile->GetName();
3024 Int_t crtCount = snextfile.CountChar('/');
3025 if (crtCount > countChar) {
3026 delete nextfile;
3027 continue;
3028 }
3029 listoffilestmp->Add(nextfile);
3030 }
3031 delete listoffiles;
3032 listoffiles = listoffilestmp; // Now contains 'good' files
3033 listoffiles->Print();
3034 TIter next(listoffiles);
7c2cd90a 3035 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 3036 outputChunk = outputFile;
3037 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 3038 // Check for existent temporary merge files
e1c22e21 3039 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 3040 // Preserve old merging functionality for stage 0.
3041 if (stage==0) {
3042 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
3043 while (1) {
3044 // Skip as many input files as in a chunk
2ea55496 3045 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 3046 nextfile = next();
3047 if (!nextfile) {
2ea55496 3048 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 3049 delete listoffiles;
2ea55496 3050 return kFALSE;
3051 }
b3e07543 3052 snextfile = nextfile->GetName();
7c2cd90a 3053 }
3054 outputChunk = outputFile;
3055 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
3056 countChunk++;
3057 if (gSystem->AccessPathName(outputChunk)) continue;
3058 // Merged file with chunks up to <countChunk> found
3059 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
3060 previousChunk = outputChunk;
3061 break;
3062 }
3063 }
3064 countZero = nmaxmerge;
3065
b3e07543 3066 while ((nextfile=next())) {
3067 snextfile = nextfile->GetName();
2ea55496 3068 // Loop 'find' results and get next LFN
7c2cd90a 3069 if (countZero == nmaxmerge) {
3070 // First file in chunk - create file merger and add previous chunk if any.
d66aa2a9 3071 fm = new TFileMerger(isGrid);
7c2cd90a 3072 fm->SetFastMethod(kTRUE);
3073 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
3074 outputChunk = outputFile;
3075 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
3076 }
3077 // If last file found, put merged results in the output file
b3e07543 3078 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 3079 // Add file to be merged and decrement chunk counter.
b3e07543 3080 fm->AddFile(snextfile);
7c2cd90a 3081 countZero--;
b3e07543 3082 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 3083 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3084 // Nothing found - skip this output
3085 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 3086 merged = kFALSE;
3087 break;
7c2cd90a 3088 }
3089 fm->OutputFile(outputChunk);
3090 // Merge the outputs, then go to next chunk
3091 if (!fm->Merge()) {
3092 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 3093 merged = kFALSE;
3094 break;
7c2cd90a 3095 } else {
3096 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
3097 gSystem->Unlink(previousChunk);
3098 }
b3e07543 3099 if (nextfile == listoffiles->Last()) break;
7c2cd90a 3100 countChunk++;
3101 countZero = nmaxmerge;
3102 previousChunk = outputChunk;
0f389141 3103 }
0f389141 3104 }
b3e07543 3105 delete listoffiles;
3106 delete fm;
7c2cd90a 3107 return merged;
3108 }
3109 // Merging stage different than 0.
3110 // Move to the begining of the requested chunk.
d66aa2a9 3111 fm = new TFileMerger(isGrid);
7c2cd90a 3112 fm->SetFastMethod(kTRUE);
b3e07543 3113 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
3114 delete listoffiles;
7c2cd90a 3115 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3116 // Nothing found - skip this output
3117 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
3118 delete fm;
3119 return kFALSE;
3120 }
b3e07543 3121 fm->OutputFile(outputFile);
7c2cd90a 3122 // Merge the outputs
3123 if (!fm->Merge()) {
3124 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
3125 delete fm;
3126 return kFALSE;
3127 } else {
b3e07543 3128 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 3129 }
7c2cd90a 3130 delete fm;
3131 return kTRUE;
0f389141 3132}
3133
c57f56b7 3134//______________________________________________________________________________
3135Bool_t AliAnalysisAlien::MergeOutputs()
3136{
3137// Merge analysis outputs existing in the AliEn space.
3138 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
3139 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3140 if (!Connect()) {
3141 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
3142 return kFALSE;
0f389141 3143 }
c9e8f7fd 3144 if (fMergeViaJDL) {
3145 if (!TestBit(AliAnalysisGrid::kMerge)) {
3146 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
3147 return kFALSE;
3148 }
3149 if (fProductionMode) {
3150 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
3151 return kFALSE;
3152 }
0f389141 3153 Info("MergeOutputs", "Submitting merging JDL");
a03be957 3154 if (!SubmitMerging()) return kFALSE;
0f389141 3155 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 3156 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 3157 return kFALSE;
c57f56b7 3158 }
3159 // Get the output path
dc030a24 3160 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 3161 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 3162 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
3163 return kFALSE;
3164 }
3165 if (!fOutputFiles.Length()) {
3166 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3167 return kFALSE;
0f389141 3168 }
3169 // Check if fast read option was requested
e1c22e21 3170 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
3171 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 3172 if (fFastReadOption) {
d3339be3 3173 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
3174 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 3175 gEnv->SetValue("XNet.ConnectTimeout",50);
3176 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 3177 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 3178 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 3179 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 3180 }
e8b839ab 3181 // Make sure we change the temporary directory
3182 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 3183 // Set temporary compilation directory to current one
3184 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 3185 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3186 TIter next(list);
3187 TObjString *str;
a2f5fc01 3188 TString outputFile;
c57f56b7 3189 Bool_t merged = kTRUE;
3190 while((str=(TObjString*)next())) {
a2f5fc01 3191 outputFile = str->GetString();
3192 Int_t index = outputFile.Index("@");
3193 if (index > 0) outputFile.Remove(index);
3194 TString outputChunk = outputFile;
3195 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 3196 // Skip already merged outputs
a2f5fc01 3197 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 3198 if (fOverwriteMode) {
a2f5fc01 3199 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
3200 gSystem->Unlink(outputFile);
3201 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3202 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3203 outputChunk.Data());
3204 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3205 }
3206 } else {
a2f5fc01 3207 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 3208 continue;
3209 }
3210 } else {
a2f5fc01 3211 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3212 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3213 outputChunk.Data());
3214 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3215 }
3216 }
37d21c01 3217 if (fMergeExcludes.Contains(outputFile.Data()) ||
3218 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 3219 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 3220 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 3221 if (!merged) {
3222 Error("MergeOutputs", "Terminate() will NOT be executed");
09d5920f 3223 delete list;
3224 return kFALSE;
ff07ec61 3225 }
3226 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
3227 if (fileOpened) fileOpened->Close();
c57f56b7 3228 }
09d5920f 3229 delete list;
0f389141 3230 return kTRUE;
c57f56b7 3231}
3232
bb885a9e 3233//______________________________________________________________________________
3234void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
3235{
3236// Use the output files connected to output containers from the analysis manager
3237// rather than the files defined by SetOutputFiles
3238 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 3239 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 3240 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
3241}
3242
149d288c 3243//______________________________________________________________________________
3244void AliAnalysisAlien::SetOutputFiles(const char *list)
3245{
3246// Manually set the output files list.
3247// Removes duplicates. Not allowed if default outputs are not disabled.
3248 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3249 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
3250 return;
3251 }
3252 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
3253 fOutputFiles = "";
3254 TString slist = list;
3255 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
3256 TObjArray *arr = slist.Tokenize(" ");
3257 TObjString *os;
3258 TIter next(arr);
3259 TString sout;
3260 while ((os=(TObjString*)next())) {
3261 sout = os->GetString();
3262 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
3263 if (fOutputFiles.Contains(sout)) continue;
3264 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
3265 fOutputFiles += sout;
3266 }
3267 delete arr;
f790bc1b 3268}
149d288c 3269
3270//______________________________________________________________________________
3271void AliAnalysisAlien::SetOutputArchive(const char *list)
3272{
3273// Manually set the output archive list. Free text - you are on your own...
3274// Not allowed if default outputs are not disabled.
3275 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3276 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
3277 return;
3278 }
3279 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
3280 fOutputArchive = list;
3281}
3282
28ce6c9c 3283//______________________________________________________________________________
3284void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
3285{
3286// Set some PROOF special parameter.
3287 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3288 if (pair) {
3289 TObject *old = pair->Key();
3290 TObject *val = pair->Value();
3291 fProofParam.Remove(old);
3292 delete old;
3293 delete val;
3294 }
3295 fProofParam.Add(new TObjString(pname), new TObjString(value));
3296}
3297
3298//______________________________________________________________________________
3299const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
3300{
3301// Returns a special PROOF parameter.
3302 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3303 if (!pair) return 0;
3304 return pair->Value()->GetName();
3305}
3306
5513444a 3307//______________________________________________________________________________
3308Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 3309{
3310// Start remote grid analysis.
3bdcb562 3311 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3312 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
3313 if (!mgr || !mgr->IsInitialized()) {
3314 Error("StartAnalysis", "You need an initialized analysis manager for this");
3315 return kFALSE;
3316 }
3317 // Are we in PROOF mode ?
3318 if (mgr->IsProofMode()) {
baa34407 3319 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3320 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3321 if (fProofCluster.IsNull()) {
3322 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3323 return kFALSE;
3324 }
3325 if (fProofDataSet.IsNull() && !testMode) {
3326 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3327 return kFALSE;
3328 }
3329 // Set the needed environment
3330 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3331 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3332 if (fProofReset && !testMode) {
3333 if (fProofReset==1) {
3334 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3335 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3336 } else {
3337 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3338 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3339 }
3340 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3341 return kFALSE;
3342 }
baa34407 3343
3344 if (!testMode) {
3345 // Check if there is an old active session
3346 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3347 if (nsessions) {
3348 Error("StartAnalysis","You have to reset your old session first\n");
3349 return kFALSE;
3350 }
53911ca6 3351 }
3bdcb562 3352 // Do we need to change the ROOT version ? The success of this cannot be checked.
874de7ea 3353 if (!fROOTVersion.IsNull() && !testMode) {
cbccef47 3354 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"VO_ALICE@ROOT::%s\");",
874de7ea 3355 fProofCluster.Data(), fROOTVersion.Data()));
3bdcb562 3356 }
3357 // Connect to PROOF and check the status
3358 Long_t proof = 0;
27734f0e 3359 TString sworkers;
3360 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3361 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3362 if (!testMode) {
27734f0e 3363 if (!sworkers.IsNull())
3364 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3365 else
3366 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3367 } else {
3368 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3369 if (!proof) {
3370 Error("StartAnalysis", "Could not start PROOF in test mode");
3371 return kFALSE;
3372 }
3373 }
3374 if (!proof) {
3375 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3376 return kFALSE;
3377 }
27734f0e 3378 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3379 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3380 // Set proof special parameters if any
3381 TIter nextpp(&fProofParam);
3382 TObject *proofparam;
3383 while ((proofparam=nextpp())) {
3384 TString svalue = GetProofParameter(proofparam->GetName());
3385 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3386 }
3bdcb562 3387 // Is dataset existing ?
3388 if (!testMode) {
3389 TString dataset = fProofDataSet;
3390 Int_t index = dataset.Index("#");
3391 if (index>=0) dataset.Remove(index);
830acc4c 3392// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3393// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3394// return kFALSE;
3395// }
3396// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3397 }
3398 // Is ClearPackages() needed ?
3399 if (TestSpecialBit(kClearPackages)) {
3400 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3401 gROOT->ProcessLine("gProof->ClearPackages();");
3402 }
3403 // Is a given aliroot mode requested ?
3404 TList optionsList;
1ac846e9 3405 TString parLibs;
830acc4c 3406 if (!fAliRootMode.IsNull()) {
3bdcb562 3407 TString alirootMode = fAliRootMode;
3408 if (alirootMode == "default") alirootMode = "";
3409 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3410 optionsList.SetOwner();
3411 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3412 // Check the additional libs to be loaded
3413 TString extraLibs;
1ac846e9 3414 Bool_t parMode = kFALSE;
8a95d6a2 3415 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:ANALYSISalice:OADB";
2dffffd6 3416 // Parse the extra libs for .so or .dylib
3bdcb562 3417 if (fAdditionalLibs.Length()) {
a41b2882 3418 TString additionalLibs = fAdditionalLibs;
3419 additionalLibs.Strip();
3420 if (additionalLibs.Length() && fFriendLibs.Length())
3421 additionalLibs += " ";
3422 additionalLibs += fFriendLibs;
3423 TObjArray *list = additionalLibs.Tokenize(" ");
3bdcb562 3424 TIter next(list);
3425 TObjString *str;
1ac846e9 3426 while((str=(TObjString*)next())) {
2dffffd6 3427 if (str->GetString().Contains(".so") ||
3428 str->GetString().Contains(".dylib") ) {
1ac846e9 3429 if (parMode) {
3430 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3431 break;
3432 }
3433 TString stmp = str->GetName();
3434 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3435 stmp.ReplaceAll(".so","");
2dffffd6 3436 stmp.ReplaceAll(".dylib","");
1ac846e9 3437 if (!extraLibs.IsNull()) extraLibs += ":";
3438 extraLibs += stmp;
3439 continue;
3440 }
3441 if (str->GetString().Contains(".par")) {
3442 // The first par file found in the list will not allow any further .so
3443 parMode = kTRUE;
3444 if (!parLibs.IsNull()) parLibs += ":";
3445 parLibs += str->GetName();
3446 continue;
3447 }
3bdcb562 3448 }
3449 if (list) delete list;
3450 }
a41b2882 3451 if (!extraLibs.IsNull()) {
3452 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3453 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3454 }
830acc4c 3455 // Check extra includes
3456 if (!fIncludePath.IsNull()) {
3457 TString includePath = fIncludePath;
3458 includePath.ReplaceAll(" ",":");
dc030a24 3459 includePath.ReplaceAll("$ALICE_ROOT/","");
3460 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3461 includePath.ReplaceAll("-I","");
dc030a24 3462 includePath.Remove(TString::kTrailing, ':');
830acc4c 3463 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3464 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3465 }
27734f0e 3466 // Check if connection to grid is requested
3467 if (TestSpecialBit(kProofConnectGrid))
3468 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3469 // Enable AliRoot par
3470 if (testMode) {
3471 // Enable proof lite package
3472 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3473 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3474 TNamed *obj = (TNamed*)optionsList.At(i);
3475 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3476 }
3477 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3478 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3479 Info("StartAnalysis", "AliRootProofLite enabled");
3480 } else {
3481 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3482 return kFALSE;
3483 }
3484 } else {
dc030a24 3485 if ( ! fAliROOTVersion.IsNull() ) {
3486 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3487 fAliROOTVersion.Data(), &optionsList))) {
3488 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3489 return kFALSE;
3490 }
3491 }
af78fedd 3492 if ( ! fAliPhysicsVersion.IsNull() ) {
3493 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliPhysics::%s\", (TList*)%p, kTRUE);",
3494 fAliPhysicsVersion.Data(), &optionsList))) {
3495 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliPhysics::%s", fAliPhysicsVersion.Data());
3496 return kFALSE;
3497 }
3498 }
27734f0e 3499 }
1ac846e9 3500 // Enable first par files from fAdditionalLibs
3501 if (!parLibs.IsNull()) {
3502 TObjArray *list = parLibs.Tokenize(":");
3503 TIter next(list);
3504 TObjString *package;
3505 while((package=(TObjString*)next())) {
40501bb7 3506 TString spkg = package->GetName();
3507 spkg.ReplaceAll(".par", "");
3508 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3509 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3510 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3511 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3512 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3513 return kFALSE;
3514 }
3515 } else {
3516 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3517 return kFALSE;
3518 }
3519 }
3520 if (list) delete list;
3521 }
3bdcb562 3522 } else {
2dffffd6 3523 if ((fAdditionalLibs.Contains(".so") || fAdditionalLibs.Contains(".dylib")) &&
3524 !testMode) {
3bdcb562 3525 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3526 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3527 return kFALSE;
3528 }
3529 }
3530 // Enable par files if requested
3531 if (fPackages && fPackages->GetEntries()) {
3532 TIter next(fPackages);
3533 TObject *package;
3534 while ((package=next())) {
1ac846e9 3535 // Skip packages already enabled
3536 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3537 TString spkg = package->GetName();
3538 spkg.ReplaceAll(".par", "");
3539 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3540 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3541 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3542 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3543 return kFALSE;
3544 }
3545 } else {
3546 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3547 return kFALSE;
3548 }
3549 }
3550 }
3551 // Do we need to load analysis source files ?
3552 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3553 if (fAnalysisSource.Length()) {
3554 TObjArray *list = fAnalysisSource.Tokenize(" ");
3555 TIter next(list);
3556 TObjString *str;
3557 while((str=(TObjString*)next())) {
3558 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3559 }
3bdcb562 3560 if (list) delete list;
3561 }
830acc4c 3562 if (testMode) {
3563 // Register dataset to proof lite.
3564 if (fFileForTestMode.IsNull()) {
3565 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3566 return kFALSE;
3567 }
3568 if (gSystem->AccessPathName(fFileForTestMode)) {
3569 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3570 return kFALSE;
3571 }
3572 TFileCollection *coll = new TFileCollection();
3573 coll->AddFromFile(fFileForTestMode);
8ddad121 3574 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3575 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3576 }
3bdcb562 3577 return kTRUE;
3578 }
c57f56b7 3579
43da816a 3580 // Check if output files have to be taken from the analysis manager
3581 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3582 // Add output files and AOD files
3583 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3584 // Add extra files registered to the analysis manager
f790bc1b 3585 TString extra = GetListOfFiles("ext");
3586 if (!extra.IsNull()) {
149d288c 3587 extra.ReplaceAll(".root", "*.root");
f790bc1b 3588 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3589 fOutputFiles += extra;
84fcd93f 3590 }
149d288c 3591 // Compose the output archive.
b83f84f0 3592 fOutputArchive = "log_archive.zip:std*@disk=1 ";
ec5acdce 3593 if (mgr->IsCollectThroughput())
95e16f01 3594 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",fOutputFiles.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 3595 else
3596 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3597 }
f7b1cbc2 3598// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3599 if (TestBit(AliAnalysisGrid::kOffline)) {
3600 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3601 \n there nor any job run. You can revise the JDL and analysis \
3602 \n macro then run the same in \"submit\" mode.");
3603 } else if (TestBit(AliAnalysisGrid::kTest)) {
3604 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3605 \n dataset.");
3606 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3607 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3608 \n space and job submitted.");
3609 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3610 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3611 if (fMergeViaJDL) CheckInputData();
5513444a 3612 return kTRUE;
c57f56b7 3613 } else {
3614 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3615 }
3616
348be253 3617 Print();
c57f56b7 3618 if (!Connect()) {
3619 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3620 return kFALSE;
16a4353c 3621 }
d3b18c4c 3622 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3623 if (!CheckInputData()) {
3624 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3625 return kFALSE;
c57f56b7 3626 }
d3339be3 3627 if (!CreateDataset(fDataPattern)) {
3628 TString serror;
3629 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3630 if (fRunNumbers.Length()) serror = "run numbers";
3631 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3632 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3633 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3634 return kFALSE;
3635 }
21ca8e59 3636 WriteAnalysisFile();
c57f56b7 3637 WriteAnalysisMacro();
3638 WriteExecutable();
3639 WriteValidationScript();
0f389141 3640 if (fMergeViaJDL) {
3641 WriteMergingMacro();
3642 WriteMergeExecutable();
3643 WriteValidationScript(kTRUE);
3644 }
5513444a 3645 if (!CreateJDL()) return kFALSE;
3646 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3647 if (testMode) {
c57f56b7 3648 // Locally testing the analysis
3649 Info("StartAnalysis", "\n_______________________________________________________________________ \
3650 \n Running analysis script in a daughter shell as on a worker node \
3651 \n_______________________________________________________________________");
149d288c 3652 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3653 TIter next(list);
3654 TObjString *str;
a2f5fc01 3655 TString outputFile;
c57f56b7 3656 while((str=(TObjString*)next())) {
a2f5fc01 3657 outputFile = str->GetString();
3658 Int_t index = outputFile.Index("@");
3659 if (index > 0) outputFile.Remove(index);
3660 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3661 }
3662 delete list;
3663 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3664 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3665// gSystem->Exec("cat stdout");
5513444a 3666 return kFALSE;
c57f56b7 3667 }
5513444a 3668 // Check if submitting is managed by LPM manager
a3e84053 3669 if (fProductionMode) {
7ac61826 3670 //TString prodfile = fJDLName;
3671 //prodfile.ReplaceAll(".jdl", ".prod");
3672 //WriteProductionFile(prodfile);
5513444a 3673 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3674 return kFALSE;
3675 }
a8739e8a 3676 // Submit AliEn job(s)
d2a409b2 3677 gGrid->Cd(fGridOutputDir);
a8739e8a 3678 TGridResult *res;
c57f56b7 3679 TString jobID = "";
0ad3ea1f 3680 fGridJobIDs = "";
3681 fGridStages = "";
d2a409b2 3682 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3683 // Submit a given xml or a set of runs
a8739e8a 3684 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3685 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3686 if (res) {
3687 const char *cjobId = res->GetKey(0,"jobId");
3688 if (!cjobId) {
a03be957 3689 gGrid->Stdout();
3690 gGrid->Stderr();
a8739e8a 3691 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3692 return kFALSE;
a8739e8a 3693 } else {
3694 Info("StartAnalysis", "\n_______________________________________________________________________ \
3695 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3696 \n_______________________________________________________________________",
3697 fJDLName.Data(), cjobId);
3698 jobID = cjobId;
0ad3ea1f 3699 if (jobID.Atoi()) {
3700 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3701 fGridJobIDs.Append(jobID);
3702 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3703 fGridStages.Append("full");
3704 }
a8739e8a 3705 }
3706 delete res;
a03be957 3707 } else {
3708 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3709 return kFALSE;
a8739e8a 3710 }
3711 } else {
d2a409b2 3712 // Submit for a range of enumeration of runs.
a03be957 3713 if (!Submit()) return kFALSE;
0ad3ea1f 3714 jobID = fGridJobIDs;
c57f56b7 3715 }
a8739e8a 3716
d88e4159 3717 if (fDropToShell) {
3718 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3719 \n You may exit at any time and terminate the job later using the option <terminate> \
3720 \n ##################################################################################", jobID.Data());
3721 gSystem->Exec("aliensh");
3722 } else {
3723 Info("StartAnalysis", "\n#### SUBMITTED JOB %s TO ALIEN QUEUE #### \
3724 \n Remember to terminate the job later using the option <terminate> \
3725 \n ##################################################################################", jobID.Data());
3726 }
5513444a 3727 return kTRUE;
c57f56b7 3728}
3729
f790bc1b 3730//______________________________________________________________________________
3731const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3732{
3733// Get a comma-separated list of output files of the requested type.
3734// Type can be (case unsensitive):
3735// aod - list of aod files (std, extensions and filters)
3736// out - list of output files connected to containers (but not aod's or extras)
3737// ext - list of extra files registered to the manager
3738// ter - list of files produced in terminate
3739 static TString files;
3740 files = "";
3741 TString stype = type;
3742 stype.ToLower();
3743 TString aodfiles, extra;
3744 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3745 if (!mgr) {
3746 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3747 return files.Data();
3748 }
3749 if (mgr->GetOutputEventHandler()) {
6be131bd 3750 aodfiles = "";
3751 if (mgr->GetOutputEventHandler()->GetFillAOD())
3752 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
f6eaa73b 3753 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs(kTRUE);
6be131bd 3754 if (!extraaod.IsNull() && mgr->GetOutputEventHandler()->GetFillExtension()) {
0c0825e4 3755 if (!aodfiles.IsNull()) aodfiles += ",";
f790bc1b 3756 aodfiles += extraaod;
3757 }
3758 }
3759 if (stype.Contains("aod")) {
3760 files = aodfiles;
3761 if (stype == "aod") return files.Data();
3762 }
3763 // Add output files that are not in the list of AOD files
3764 TString outputfiles = "";
3765 TIter next(mgr->GetOutputs());
3766 AliAnalysisDataContainer *output;
3767 const char *filename = 0;
3768 while ((output=(AliAnalysisDataContainer*)next())) {
3769 filename = output->GetFileName();
3770 if (!(strcmp(filename, "default"))) continue;
3771 if (outputfiles.Contains(filename)) continue;
3772 if (aodfiles.Contains(filename)) continue;
0c0825e4 3773 if (!outputfiles.IsNull() && strlen(filename)) outputfiles += ",";
f790bc1b 3774 outputfiles += filename;
3775 }
3776 if (stype.Contains("out")) {
0c0825e4 3777 if (!files.IsNull() && !outputfiles.IsNull()) files += ",";
f790bc1b 3778 files += outputfiles;
3779 if (stype == "out") return files.Data();
3780 }
3781 // Add extra files registered to the analysis manager
3782 TString sextra;
3783 extra = mgr->GetExtraFiles();
3784 if (!extra.IsNull()) {
3785 extra.Strip();
3786 extra.ReplaceAll(" ", ",");
3787 TObjArray *fextra = extra.Tokenize(",");
3788 TIter nextx(fextra);
3789 TObject *obj;
3790 while ((obj=nextx())) {
3791 if (aodfiles.Contains(obj->GetName())) continue;
3792 if (outputfiles.Contains(obj->GetName())) continue;
3793 if (sextra.Contains(obj->GetName())) continue;
3794 if (!sextra.IsNull()) sextra += ",";
3795 sextra += obj->GetName();
3796 }
3797 delete fextra;
3798 if (stype.Contains("ext")) {
0c0825e4 3799 if (!files.IsNull() && !sextra.IsNull()) files += ",";
f790bc1b 3800 files += sextra;
3801 }
3802 }
3803 if (stype == "ext") return files.Data();
3804 TString termfiles;
3805 if (!fTerminateFiles.IsNull()) {
3806 fTerminateFiles.Strip();
3807 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3808 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3809 TIter nextx(fextra);
3810 TObject *obj;
3811 while ((obj=nextx())) {
3812 if (aodfiles.Contains(obj->GetName())) continue;
3813 if (outputfiles.Contains(obj->GetName())) continue;
3814 if (termfiles.Contains(obj->GetName())) continue;
3815 if (sextra.Contains(obj->GetName())) continue;
3816 if (!termfiles.IsNull()) termfiles += ",";
3817 termfiles += obj->GetName();
3818 }
3819 delete fextra;
3820 }
3821 if (stype.Contains("ter")) {
3822 if (!files.IsNull() && !termfiles.IsNull()) {
3823 files += ",";
3824 files += termfiles;
3825 }
3826 }
3827 return files.Data();
3828}
3829
d2a409b2 3830//______________________________________________________________________________
a03be957 3831Bool_t AliAnalysisAlien::Submit()
d2a409b2 3832{
3833// Submit all master jobs.
3834 Int_t nmasterjobs = fInputFiles->GetEntries();
3835 Long_t tshoot = gSystem->Now();
a03be957 3836 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3837 while (fNsubmitted < nmasterjobs) {
3838 Long_t now = gSystem->Now();
3839 if ((now-tshoot)>30000) {
3840 tshoot = now;
a03be957 3841 if (!SubmitNext()) return kFALSE;
d2a409b2 3842 }
3843 }
a03be957 3844 return kTRUE;
d2a409b2 3845}
3846
0f389141 3847//______________________________________________________________________________
a03be957 3848Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3849{
3850// Submit all merging jobs.
dc030a24 3851 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3852 gGrid->Cd(fGridOutputDir);
3853 TString mergeJDLName = fExecutable;
3854 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3855 if (!fInputFiles) {
3856 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3857 return kFALSE;
3858 }
0f389141 3859 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3860 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3861 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3862 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3863 if (fOutputToRunNo) {
3864 // The output directory is the run number
3865 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3866 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3867 } else {
17b4e9a4 3868 if (!fRunNumbers.Length() && !fRunRange[0]) {
3869 // The output directory is the grid outdir
3870 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3871 runOutDir = fGridOutputDir;
3872 } else {
3873 // The output directory is the master number in 3 digits format
3874 printf("### Submitting merging job for master <%03d>\n", i);
3875 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3876 }
7c2cd90a 3877 }
3878 // Check now the number of merging stages.
2b222f7c 3879 TObjArray *list = fOutputFiles.Tokenize(",");
3880 TIter next(list);
3881 TObjString *str;
3882 TString outputFile;
3883 while((str=(TObjString*)next())) {
3884 outputFile = str->GetString();
3885 Int_t index = outputFile.Index("@");
3886 if (index > 0) outputFile.Remove(index);
37d21c01 3887 if (!fMergeExcludes.Contains(outputFile) &&
3888 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3889 }
3890 delete list;
b3e07543 3891 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3892 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3893 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3894 }
a03be957 3895 if (!ntosubmit) return kTRUE;
d88e4159 3896 if (fDropToShell) {
3897 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3898 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3899 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3900 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3901 \n ################################################################################################################");
3902 gSystem->Exec("aliensh");
3903 } else {
3904 Info("StartAnalysis", "\n #### STARTED MERGING JOBS FOR YOU #### \
3905 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL') \
3906 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3907 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3908 \n ################################################################################################################");
3909 }
a03be957 3910 return kTRUE;
0f389141 3911}
3912
d2a409b2 3913//______________________________________________________________________________
a03be957 3914Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3915{
08d5b699 3916// Submit next bunch of master jobs if the queue is free. The first master job is
3917// submitted right away, while the next will not be unless the previous was split.
3918// The plugin will not submit new master jobs if there are more that 500 jobs in
3919// waiting phase.
d2a409b2 3920 static Bool_t iscalled = kFALSE;
3921 static Int_t firstmaster = 0;
3922 static Int_t lastmaster = 0;
3923 static Int_t npermaster = 0;
a03be957 3924 if (iscalled) return kTRUE;
d2a409b2 3925 iscalled = kTRUE;
3926 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3927 Int_t ntosubmit = 0;
3928 TGridResult *res;
3929 TString jobID = "";
3bdcb562 3930 Int_t nmasterjobs = fInputFiles->GetEntries();
3931 if (!fNsubmitted) {
3932 ntosubmit = 1;
3933 if (!IsUseSubmitPolicy()) {
90d50a8c 3934 if (nmasterjobs>5)
3bdcb562 3935 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3936 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3937 ntosubmit = nmasterjobs;
3938 }
3939 } else {
d2a409b2 3940 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3941 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3942 // If last master not split, just return
a03be957 3943 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3944 // No more than 100 waiting jobs
08d5b699 3945 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3946 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3947 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3948 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3949 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3950 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3951 }
d2a409b2 3952 for (Int_t i=0; i<ntosubmit; i++) {
3953 // Submit for a range of enumeration of runs.
a03be957 3954 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3955 TString query;
cd11251e 3956 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3957 runOutDir.ReplaceAll(".xml", "");
3958 if (fOutputToRunNo)
3959 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3960 else
3961 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3962 printf("********* %s\n",query.Data());
d2a409b2 3963 res = gGrid->Command(query);
3964 if (res) {
98ca124f 3965 TString cjobId1 = res->GetKey(0,"jobId");
3966 if (!cjobId1.Length()) {
d2a409b2 3967 iscalled = kFALSE;
a03be957 3968 gGrid->Stdout();
3969 gGrid->Stderr();
3970 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3971 return kFALSE;
d2a409b2 3972 } else {
3973 Info("StartAnalysis", "\n_______________________________________________________________________ \
3974 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3975 \n_______________________________________________________________________",
98ca124f 3976 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
0ad3ea1f 3977 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3978 fGridJobIDs.Append(cjobId1);
3979 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3980 fGridStages.Append("full");
d2a409b2 3981 jobID += cjobId1;
3982 jobID += " ";
98ca124f 3983 lastmaster = cjobId1.Atoi();
d2a409b2 3984 if (!firstmaster) firstmaster = lastmaster;
3985 fNsubmitted++;
3986 }
3987 delete res;
a03be957 3988 } else {
3989 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3990 return kFALSE;
d2a409b2 3991 }
3992 }
3993 iscalled = kFALSE;
a03be957 3994 return kTRUE;
d2a409b2 3995}
3996
c57f56b7 3997//______________________________________________________________________________
3998void AliAnalysisAlien::WriteAnalysisFile()
3999{
f10e8481 4000// Write current analysis manager into the file <analysisFile>
4001 TString analysisFile = fExecutable;
4002 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 4003 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4004 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
4005 if (!mgr || !mgr->IsInitialized()) {
4006 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
4007 return;
4008 }
4009 // Check analysis type
4010 TObject *handler;
4011 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
4012 handler = (TObject*)mgr->GetInputEventHandler();
4013 if (handler) {
65eb22f8 4014 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
4015 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
4016 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
4017 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
4018 } else {
4019 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
4020 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
4021 }
c57f56b7 4022 }
4023 TDirectory *cdir = gDirectory;
f10e8481 4024 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 4025 if (file) {
e16a394c 4026 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
4027 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 4028 // Unless merging makes no sense
4029 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 4030 mgr->Write();
4031 delete file;
fe2d7fc2 4032 // Enable termination for local jobs
4033 mgr->SetSkipTerminate(kFALSE);
c57f56b7 4034 }
4035 if (cdir) cdir->cd();
f10e8481 4036 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 4037 }
4038 Bool_t copy = kTRUE;
d3b18c4c 4039 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4040 if (copy) {
4041 CdWork();
4042 TString workdir = gGrid->GetHomeDirectory();
4043 workdir += fGridWorkingDir;
d3b18c4c 4044 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 4045 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 4046 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
4047 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 4048 }
4049}
4050
4051//______________________________________________________________________________
4052void AliAnalysisAlien::WriteAnalysisMacro()
4053{
4054// Write the analysis macro that will steer the analysis in grid mode.
4055 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4056 ofstream out;
4057 out.open(fAnalysisMacro.Data(), ios::out);
4058 if (!out.good()) {
4059 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4060 return;
4061 }
5b273635 4062 Bool_t hasSTEERBase = kFALSE;
4063 Bool_t hasESD = kFALSE;
4064 Bool_t hasAOD = kFALSE;
4065 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4066 Bool_t hasOADB = kFALSE;
5b273635 4067 Bool_t hasANALYSISalice = kFALSE;
4068 Bool_t hasCORRFW = kFALSE;
c57f56b7 4069 TString func = fAnalysisMacro;
4070 TString type = "ESD";
4071 TString comment = "// Analysis using ";
715da79c 4072 if (fMCLoop) {
4073 type = "MCGEN";
4074 comment += "MCGEN";
4075 } else {
4076 if (IsUseMCchain()) {
4077 type = "MC";
4078 comment += "MC";
4079 } else {
4080 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
4081 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
4082 type = "AOD";
4083 comment += "AOD";
4084 }
4085 }
4086 }
0df6ccf2 4087 if (type!="AOD" && fFriendChainName!="") {
4088 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
4089 return;
4090 }
c57f56b7 4091 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
4092 else comment += " data";
4093 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
4094 func.ReplaceAll(".C", "");
4095 out << "void " << func.Data() << "()" << endl;
4096 out << "{" << endl;
4097 out << comment.Data() << endl;
4098 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 4099 out << " TStopwatch timer;" << endl;
4100 out << " timer.Start();" << endl << endl;
13f0c00b 4101 // Change temp directory to current one
fdee57f1 4102 if (!IsLocalTest()) {
4103 out << "// connect to AliEn and make the chain" << endl;
4104 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
4105 }
13f0c00b 4106 out << "// Set temporary merging directory to current one" << endl;
4107 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4108 out << "// Set temporary compilation directory to current one" << endl;
4109 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 4110 // Reset existing include path
4111 out << "// Reset existing include path and add current directory first in the search" << endl;
4112 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 4113 if (!fExecutableCommand.Contains("aliroot")) {
4114 out << "// load base root libraries" << endl;
4115 out << " gSystem->Load(\"libTree\");" << endl;
4116 out << " gSystem->Load(\"libGeom\");" << endl;
4117 out << " gSystem->Load(\"libVMC\");" << endl;
4118 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4119 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4120 }
d5c6455a 4121 if (fAdditionalRootLibs.Length()) {
47a4137d 4122 // in principle libtree /lib geom libvmc etc. can go into this list, too
4123 out << "// Add aditional libraries" << endl;
4124 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4125 TIter next(list);
4126 TObjString *str;
4529acca 4127 TString buf;
47a4137d 4128 while((str=(TObjString*)next())) {
4529acca 4129 buf = str->GetString();
4130 if (buf.Contains(".so") || buf.Contains(".dylib")) {
4131 buf.ReplaceAll(".so", "");
4132 buf.ReplaceAll(".dylib", "");
4133 out << " gSystem->Load(\"" << buf.Data() << "\");" << endl;
4134 }
d5c6455a 4135 }
47a4137d 4136 if (list) delete list;
d5c6455a 4137 }
57377eb5 4138 out << "// Load analysis framework libraries" << endl;
652561d5 4139 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 4140 if (!fPackages) {
f790bc1b 4141 if (!fExecutableCommand.Contains("aliroot")) {
4142 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4143 out << " gSystem->Load(\"libESD\");" << endl;
4144 out << " gSystem->Load(\"libAOD\");" << endl;
4145 }
4e5c5506 4146 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 4147 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
8a95d6a2 4148 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 4149 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 4150 } else {
4e5c5506 4151 TIter next(fPackages);
4152 TObject *obj;
57377eb5 4153 TString pkgname;
57377eb5 4154 while ((obj=next())) {
4155 pkgname = obj->GetName();
4478e6f1 4156 if (pkgname == "STEERBase" ||
4157 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4158 if (pkgname == "ESD" ||
4159 pkgname == "ESD.par") hasESD = kTRUE;
4160 if (pkgname == "AOD" ||
4161 pkgname == "AOD.par") hasAOD = kTRUE;
4162 if (pkgname == "ANALYSIS" ||
4163 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4164 if (pkgname == "OADB" ||
4165 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 4166 if (pkgname == "ANALYSISalice" ||
4167 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4168 if (pkgname == "CORRFW" ||
4169 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 4170 }
4171 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 4172 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4173 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 4174 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4175 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 4176 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4177 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 4178 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4179 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
57377eb5 4180 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4181 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
8a95d6a2 4182 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4183 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 4184 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4185 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 4186 out << "// Compile other par packages" << endl;
4187 next.Reset();
fcc9bb6f 4188 while ((obj=next())) {
4189 pkgname = obj->GetName();
4478e6f1 4190 if (pkgname == "STEERBase" ||
4191 pkgname == "STEERBase.par" ||
4192 pkgname == "ESD" ||
4193 pkgname == "ESD.par" ||
4194 pkgname == "AOD" ||
4195 pkgname == "AOD.par" ||
4196 pkgname == "ANALYSIS" ||
4197 pkgname == "ANALYSIS.par" ||
7ea40ac8 4198 pkgname == "OADB" ||
4199 pkgname == "OADB.par" ||
4478e6f1 4200 pkgname == "ANALYSISalice" ||
4201 pkgname == "ANALYSISalice.par" ||
4202 pkgname == "CORRFW" ||
4203 pkgname == "CORRFW.par") continue;
5b273635 4204 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 4205 }
4e5c5506 4206 }
b3e07543 4207 out << "// include path" << endl;
4208 // Get the include path from the interpreter and remove entries pointing to AliRoot
4209 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4210 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4211 out << " TIter nextpath(listpaths);" << endl;
4212 out << " TObjString *pname;" << endl;
4213 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4214 out << " TString current = pname->GetName();" << endl;
4215 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4216 out << " gSystem->AddIncludePath(current);" << endl;
4217 out << " }" << endl;
4218 out << " if (listpaths) delete listpaths;" << endl;
4219 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4220 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4221 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
715da79c 4222 if (fMCLoop && !fGeneratorLibs.IsNull()) {
4223 out << "// MC generator libraries" << endl;
4224 TObjArray *list = fGeneratorLibs.Tokenize(" ");
4225 TIter next(list);
4226 TObjString *str;
4227 while((str=(TObjString*)next())) {
4228 out << " gSystem->Load(\"" << str->GetName() << "\");" << endl;
4229 }
4230 delete list;
4231 }
6da75e0b 4232 if (fAdditionalLibs.Length()) {
4233 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4234 TString additionalLibs = fAdditionalLibs;
4235 additionalLibs.Strip();
4236 if (additionalLibs.Length() && fFriendLibs.Length())
4237 additionalLibs += " ";
4238 additionalLibs += fFriendLibs;
4239 TObjArray *list = additionalLibs.Tokenize(" ");
6da75e0b 4240 TIter next(list);
4241 TObjString *str;
4529acca 4242 TString buf;
6da75e0b 4243 while((str=(TObjString*)next())) {
4529acca 4244 buf = str->GetString();
4245 if (buf.Contains(".so") || buf.Contains(".dylib")) {
4246 buf.ReplaceAll(".so", "");
4247 buf.ReplaceAll(".dylib", "");
4248 out << " gSystem->Load(\"" << buf.Data() << "\");" << endl;
4249 }
4250 if (buf.Contains(".par"))
4251 out << " if (!" << setupPar << "(\"" << buf.Data() << "\")) return;" << endl;
6da75e0b 4252 }
715da79c 4253 delete list;
6da75e0b 4254 }
4255 out << endl;
c57f56b7 4256 out << "// analysis source to be compiled at runtime (if any)" << endl;
4257 if (fAnalysisSource.Length()) {
4258 TObjArray *list = fAnalysisSource.Tokenize(" ");
4259 TIter next(list);
4260 TObjString *str;
4261 while((str=(TObjString*)next())) {
4262 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4263 }
4264 if (list) delete list;
4265 }
4266 out << endl;
b3e07543 4267// out << " printf(\"Currently load libraries:\\n\");" << endl;
4268// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 4269 if (fFastReadOption) {
d3339be3 4270 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
4271 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 4272 out << "// fast xrootd reading enabled" << endl;
4273 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4274 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4275 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4276 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4277 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4278 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 4279 }
c57f56b7 4280 out << "// read the analysis manager from file" << endl;
f10e8481 4281 TString analysisFile = fExecutable;
4282 analysisFile.ReplaceAll(".sh", ".root");
105631b5 4283 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4284 << analysisFile << "\");" << endl;
4285 out << " if (!mgr) return;" << endl;
21ca8e59 4286 if (IsLocalTest()) {
4287 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
4288 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 4289 if (fFileForTestMode.IsNull())
4290 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
4291 else
4292 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 4293 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4b5d9e21 4294 if (!fFriendChainName.IsNull())
b9f1efa5 4295 out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\",\"" << fFriendLibs << "\");" << endl;
024dfa52 4296 if (IsUseMCchain())
4297 out << " plugin->SetUseMCchain();" << endl;
715da79c 4298 if (fMCLoop)
4299 out << " plugin->SetMCLoop(kTRUE);" << endl;
21ca8e59 4300 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 4301 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 4302 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 4303 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
4304 } else {
7a366e0b 4305 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 4306 out << " mgr->SetNSysInfo(100);" << endl;
4307 }
21ca8e59 4308 }
c57f56b7 4309 out << " mgr->PrintStatus();" << endl;
52b6a92b 4310 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 4311 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 4312 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 4313 } else {
9a8288ff 4314 if (TestBit(AliAnalysisGrid::kTest))
4315 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4316 else
4317 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 4318 }
4319 }
21ca8e59 4320 if (!IsLocalTest()) {
715da79c 4321 if (fMCLoop) {
4322 out << " mgr->SetCacheSize(0);" << endl;
4323 out << " mgr->EventLoop(" << fNMCevents << ");" << endl;
4324 } else {
4325 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
4326 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
4327 }
21ca8e59 4328 } else {
715da79c 4329 if (fMCLoop) {
4330 out << " mgr->SetCacheSize(0);" << endl;
4331 out << " mgr->EventLoop(" << fNMCevents << ");" << endl;
7435f77f
AG		 4332 } else {
4333 out << " mgr->StartAnalysis(\"localfile\");" << endl;
4334 }
242accb2 4335 }
f7498086 4336 out << " timer.Stop();" << endl;
4337 out << " timer.Print();" << endl;
c57f56b7 4338 out << "}" << endl << endl;
715da79c 4339 if (!IsLocalTest() && !fMCLoop) {
fcc9bb6f 4340 out <<"//________________________________________________________________________________" << endl;
c57f56b7 4341 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
4342 out << "{" << endl;
4343 out << "// Create a chain using url's from xml file" << endl;
242accb2 4344 out << " TString filename;" << endl;
4345 out << " Int_t run = 0;" << endl;
c85cfc0f 4346 if (IsUseMCchain()) {
4347 out << " TString treename = \"TE\";" << endl;
4348 } else {
780ec36d 4349 if (!fTreeName.IsNull()) {
4350 out << " TString treename = \"" << fTreeName << "\";" << endl;
4351 } else {
4352 out << " TString treename = type;" << endl;
4353 out << " treename.ToLower();" << endl;
4354 out << " treename += \"Tree\";" << endl;
4355 }
c85cfc0f 4356 }
e02fee64 4357 out << " printf(\"***************************************\\n\");" << endl;
4358 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
4359 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 4360 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
4361 out << " if (!coll) {" << endl;
4362 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
4363 out << " return NULL;" << endl;
4364 out << " }" << endl;
242accb2 4365 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 4366 out << " TChain *chain = new TChain(treename);" << endl;
06f67114 4367 if(!fFriendChainName.IsNull()) {
4368 out << " TList *friends = new TList();" << endl;
4369 out << " TIter nextfriend(friends);" << endl;
4370 out << " TChain *cfriend = 0;" << endl;
4371 TObjArray *list = fFriendChainName.Tokenize(" ");
4372 TIter next(list);
4373 TObjString *str;
4374 while((str=(TObjString*)next())) {
4375 out << " cfriend = new TChain(treename, \"" << str->GetName() << "\");" << endl;
4376 out << " friends->Add(cfriend);" << endl;
4377 out << " chain->AddFriend(cfriend);" << endl;
4378 }
4379 delete list;
4380// out << " TChain *chainFriend = new TChain(treename);" << endl;
0df6ccf2 4381 }
c57f56b7 4382 out << " coll->Reset();" << endl;
0df6ccf2 4383 out << " while (coll->Next()) {" << endl;
242accb2 4384 out << " filename = coll->GetTURL("");" << endl;
4385 out << " if (mgr) {" << endl;
4386 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
4387 out << " if (nrun && nrun != run) {" << endl;
4388 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
4389 out << " mgr->SetRunFromPath(nrun);" << endl;
4390 out << " run = nrun;" << endl;
4391 out << " }" << endl;
4392 out << " }" << endl;
4393 out << " chain->Add(filename);" << endl;
06f67114 4394 if(!fFriendChainName.IsNull()) {
4395 out << " TString bpath=coll->GetTURL(\"\");" << endl;
4396 out << " if (bpath.Index(\"#\") > -1) bpath.Remove(bpath.Index(\"#\"));" << endl;
4397 out << " bpath = gSystem->DirName(bpath);" << endl;
4398 out << " bpath += \"/\";" << endl;
4399 out << " TString fileFriend;" << endl;
4400 out << " nextfriend.Reset();" << endl;
4401 out << " while ((cfriend=(TChain*)nextfriend())) {" << endl;
4402 out << " fileFriend = bpath;" << endl;
4403 out << " fileFriend += cfriend->GetTitle();" << endl;
4404 out << " TFile *file = TFile::Open(fileFriend);" << endl;
4405 out << " if (file) {" << endl;
4406 out << " file->Close();" << endl;
4407 out << " cfriend->Add(fileFriend.Data());" << endl;
4408 out << " } else {" << endl;
4409 out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
4410 out << " return 0;" << endl;
4411 out << " }" << endl;
4b5d9e21 4412 out << " }" << endl;
0df6ccf2 4413 }
4414 out << " }" << endl;
c57f56b7 4415 out << " if (!chain->GetNtrees()) {" << endl;
4416 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
4417 out << " return NULL;" << endl;
4418 out << " }" << endl;
4419 out << " return chain;" << endl;
fcc9bb6f 4420 out << "}" << endl << endl;
c57f56b7 4421 }
5b273635 4422 if (hasANALYSISalice) {
4423 out <<"//________________________________________________________________________________" << endl;
4424 out << "Bool_t SetupPar(const char *package) {" << endl;
4425 out << "// Compile the package and set it up." << endl;
4426 out << " TString pkgdir = package;" << endl;
4427 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4428 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4429 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4430 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4431 out << " // Check for BUILD.sh and execute" << endl;
4432 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4433 out << " printf(\"*******************************\\n\");" << endl;
4434 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4435 out << " printf(\"*******************************\\n\");" << endl;
4436 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4437 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4438 out << " gSystem->ChangeDirectory(cdir);" << endl;
4439 out << " return kFALSE;" << endl;
4440 out << " }" << endl;
4441 out << " } else {" << endl;
4442 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4443 out << " gSystem->ChangeDirectory(cdir);" << endl;
4444 out << " return kFALSE;" << endl;
4445 out << " }" << endl;
4446 out << " // Check for SETUP.C and execute" << endl;
4447 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4448 out << " printf(\"*******************************\\n\");" << endl;
4449 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4450 out << " printf(\"*******************************\\n\");" << endl;
4451 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4452 out << " } else {" << endl;
4453 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4454 out << " gSystem->ChangeDirectory(cdir);" << endl;
4455 out << " return kFALSE;" << endl;
4456 out << " }" << endl;
4457 out << " // Restore original workdir" << endl;
4458 out << " gSystem->ChangeDirectory(cdir);" << endl;
4459 out << " return kTRUE;" << endl;
4460 out << "}" << endl;
4461 }
c57f56b7 4462 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4463 }
4464 Bool_t copy = kTRUE;
d3b18c4c 4465 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4466 if (copy) {
4467 CdWork();
4468 TString workdir = gGrid->GetHomeDirectory();
4469 workdir += fGridWorkingDir;
4470 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4471 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4472// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4473 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4474 Form("alien://%s/%s", workdir.Data(),
4475 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4476 }
4477}
4478
0f389141 4479//______________________________________________________________________________
4480void AliAnalysisAlien::WriteMergingMacro()
4481{
4482// Write a macro to merge the outputs per master job.
4483 if (!fMergeViaJDL) return;
4484 if (!fOutputFiles.Length()) {
4485 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4486 return;
4487 }
ec5acdce 4488 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
0f389141 4489 TString mergingMacro = fExecutable;
4490 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4491 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4492 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4493 ofstream out;
4494 out.open(mergingMacro.Data(), ios::out);
4495 if (!out.good()) {
4496 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4497 return;
4498 }
5b273635 4499 Bool_t hasSTEERBase = kFALSE;
4500 Bool_t hasESD = kFALSE;
4501 Bool_t hasAOD = kFALSE;
4502 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4503 Bool_t hasOADB = kFALSE;
5b273635 4504 Bool_t hasANALYSISalice = kFALSE;
4505 Bool_t hasCORRFW = kFALSE;
0f389141 4506 TString func = mergingMacro;
4507 TString comment;
4508 func.ReplaceAll(".C", "");
f47d5cb4 4509 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4510 out << "{" << endl;
4511 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4512 out << " TStopwatch timer;" << endl;
4513 out << " timer.Start();" << endl << endl;
b3e07543 4514 // Reset existing include path
4515 out << "// Reset existing include path and add current directory first in the search" << endl;
4516 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4517 if (!fExecutableCommand.Contains("aliroot")) {
4518 out << "// load base root libraries" << endl;
4519 out << " gSystem->Load(\"libTree\");" << endl;
4520 out << " gSystem->Load(\"libGeom\");" << endl;
4521 out << " gSystem->Load(\"libVMC\");" << endl;
4522 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4523 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4524 }
0f389141 4525 if (fAdditionalRootLibs.Length()) {
4526 // in principle libtree /lib geom libvmc etc. can go into this list, too
4527 out << "// Add aditional libraries" << endl;
4528 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4529 TIter next(list);
4530 TObjString *str;
4529acca 4531 TString buf;
0f389141 4532 while((str=(TObjString*)next())) {
4529acca 4533 buf = str->GetString();
4534 if (buf.Contains(".so") || buf.Contains(".dylib")) {
4535 buf.ReplaceAll(".so", "");
4536 buf.ReplaceAll(".dylib", "");
4537 out << " gSystem->Load(\"" << buf.Data() << "\");" << endl;
4538 }
0f389141 4539 }
4540 if (list) delete list;
4541 }
0f389141 4542 out << "// Load analysis framework libraries" << endl;
4543 if (!fPackages) {
7c2cd90a 4544 if (!fExecutableCommand.Contains("aliroot")) {
4545 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4546 out << " gSystem->Load(\"libESD\");" << endl;
4547 out << " gSystem->Load(\"libAOD\");" << endl;
4548 }
0f389141 4549 out << " gSystem->Load(\"libANALYSIS\");" << endl;
4550 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
8a95d6a2 4551 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4552 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4553 } else {
4554 TIter next(fPackages);
4555 TObject *obj;
4556 TString pkgname;
5b273635 4557 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4558 while ((obj=next())) {
4559 pkgname = obj->GetName();
4560 if (pkgname == "STEERBase" ||
4561 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4562 if (pkgname == "ESD" ||
4563 pkgname == "ESD.par") hasESD = kTRUE;
4564 if (pkgname == "AOD" ||
4565 pkgname == "AOD.par") hasAOD = kTRUE;
4566 if (pkgname == "ANALYSIS" ||
4567 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4568 if (pkgname == "OADB" ||
4569 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4570 if (pkgname == "ANALYSISalice" ||
4571 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4572 if (pkgname == "CORRFW" ||
4573 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4574 }
5b273635 4575 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4576 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4577 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4578 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4579 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4580 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4581 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4582 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4583 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4584 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
0f389141 4585 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4586 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
8a95d6a2 4587 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4588 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4589 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4590 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4591 out << "// Compile other par packages" << endl;
4592 next.Reset();
4593 while ((obj=next())) {
4594 pkgname = obj->GetName();
4595 if (pkgname == "STEERBase" ||
4596 pkgname == "STEERBase.par" ||
4597 pkgname == "ESD" ||
4598 pkgname == "ESD.par" ||
4599 pkgname == "AOD" ||
4600 pkgname == "AOD.par" ||
4601 pkgname == "ANALYSIS" ||
4602 pkgname == "ANALYSIS.par" ||
7ea40ac8 4603 pkgname == "OADB" ||
4604 pkgname == "OADB.par" ||
0f389141 4605 pkgname == "ANALYSISalice" ||
4606 pkgname == "ANALYSISalice.par" ||
4607 pkgname == "CORRFW" ||
4608 pkgname == "CORRFW.par") continue;
5b273635 4609 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4610 }
4611 }
b3e07543 4612 out << "// include path" << endl;
4613 // Get the include path from the interpreter and remove entries pointing to AliRoot
4614 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4615 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4616 out << " TIter nextpath(listpaths);" << endl;
4617 out << " TObjString *pname;" << endl;
4618 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4619 out << " TString current = pname->GetName();" << endl;
4620 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4621 out << " gSystem->AddIncludePath(current);" << endl;
4622 out << " }" << endl;
4623 out << " if (listpaths) delete listpaths;" << endl;
4624 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4625 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4626 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0d239a91
AG		 4627 if (fMCLoop && !fGeneratorLibs.IsNull()) {
4628 out << "// MC generator libraries" << endl;
4629 TObjArray *list = fGeneratorLibs.Tokenize(" ");
4630 TIter next(list);
4631 TObjString *str;
4632 while((str=(TObjString*)next())) {
4633 out << " gSystem->Load(\"" << str->GetName() << "\");" << endl;
4634 }
4635 delete list;
4636 }
0f389141 4637 if (fAdditionalLibs.Length()) {
4638 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4639 TString additionalLibs = fAdditionalLibs;
4640 additionalLibs.Strip();
4641 if (additionalLibs.Length() && fFriendLibs.Length())
4642 additionalLibs += " ";
4643 additionalLibs += fFriendLibs;
4644 TObjArray *list = additionalLibs.Tokenize(" ");
0f389141 4645 TIter next(list);
4646 TObjString *str;
4529acca 4647 TString buf;
0f389141 4648 while((str=(TObjString*)next())) {
4529acca 4649 buf = str->GetString();
4650 if (buf.Contains(".so") || buf.Contains(".dylib")) {
4651 buf.ReplaceAll(".so", "");
4652 buf.ReplaceAll(".dylib", "");
4653 out << " gSystem->Load(\"" << buf.Data() << "\");" << endl;
4654 }
0f389141 4655 }
4656 if (list) delete list;
4657 }
4658 out << endl;
4659 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4660 if (fAnalysisSource.Length()) {
4661 TObjArray *list = fAnalysisSource.Tokenize(" ");
4662 TIter next(list);
4663 TObjString *str;
4664 while((str=(TObjString*)next())) {
4665 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4666 }
4667 if (list) delete list;
4668 }
149d288c 4669 out << endl;
4670
0f389141 4671 if (fFastReadOption) {
4672 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4673 out << "// fast xrootd reading enabled" << endl;
4674 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4675 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4676 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4677 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4678 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4679 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4680 }
4681 // Change temp directory to current one
fdee57f1 4682 out << "// Connect to AliEn" << endl;
4683 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
e8b839ab 4684 out << "// Set temporary merging directory to current one" << endl;
4685 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4686 out << "// Set temporary compilation directory to current one" << endl;
4687 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
7c2cd90a 4688 out << " TString outputDir = dir;" << endl;
d87f72d5 4689 if (IsMergeAOD())
4690 out << " TString outputFiles = \"" << GetListOfFiles("outaod") << "\";" << endl;
4691 else
4692 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4693 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4694 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4695 out << " TIter *iter = new TIter(list);" << endl;
4696 out << " TObjString *str;" << endl;
a2f5fc01 4697 out << " TString outputFile;" << endl;
0f389141 4698 out << " Bool_t merged = kTRUE;" << endl;
ec5acdce 4699 TString analysisFile = fExecutable;
4700 analysisFile.ReplaceAll(".sh", ".root");
4701 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4702 << analysisFile << "\");" << endl;
4703 out << " if (!mgr) {" << endl;
4704 out << " printf(\"ERROR: Analysis manager could not be extracted from file \");" << endl;
4705 out << " return;" << endl;
4706 out << " }" << endl;
d66aa2a9 4707 if (IsLocalTest()) {
0c0825e4 4708 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4709 out << " printf(\"Testing merging...\\n\");" << endl;
0c0825e4 4710 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4711 }
0f389141 4712 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4713 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4714 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4715 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4716 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4717 out << " // Skip already merged outputs" << endl;
a2f5fc01 4718 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
d66aa2a9 4719 out << " printf(\"Output file <%s> found. Not merging again.\\n\",outputFile.Data());" << endl;
0f389141 4720 out << " continue;" << endl;
4721 out << " }" << endl;
a2f5fc01 4722 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4723 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4724 out << " if (!merged) {" << endl;
a2f5fc01 4725 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4726 out << " return;" << endl;
0f389141 4727 out << " }" << endl;
4728 out << " }" << endl;
d66aa2a9 4729 if (mgr && mgr->IsCollectThroughput() && !IsLocalTest()) {
ec5acdce 4730 out << " TString infolog = \"" << mgr->GetFileInfoLog() << "\";" << endl;
4731 out << " AliAnalysisAlien::MergeInfo(infolog, outputDir);" << endl;
4732 }
7c2cd90a 4733 out << " // all outputs merged, validate" << endl;
4734 out << " ofstream out;" << endl;
4735 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4736 out << " out.close();" << endl;
4737 out << " // read the analysis manager from file" << endl;
d66aa2a9 4738 if (IsLocalTest()) {
0c0825e4 4739 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4740 out << " printf(\"Testing Terminate()...\\n\");" << endl;
0c0825e4 4741 out << " printf(\"===================================\\n\");" << endl;
d66aa2a9 4742 } else {
4743 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
4744 }
5c4250fc 4745 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4746 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4747 out << " mgr->PrintStatus();" << endl;
ec5acdce 4748 if (mgr) {
4749 if (mgr->GetDebugLevel()>3) {
f866cba5 4750 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4751 } else {
9a8288ff 4752 if (TestBit(AliAnalysisGrid::kTest))
4753 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4754 else
4755 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4756 }
4757 }
9a8288ff 4758 out << " TTree *tree = NULL;" << endl;
4759 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4760 out << "}" << endl << endl;
5b273635 4761 if (hasANALYSISalice) {
4762 out <<"//________________________________________________________________________________" << endl;
4763 out << "Bool_t SetupPar(const char *package) {" << endl;
4764 out << "// Compile the package and set it up." << endl;
4765 out << " TString pkgdir = package;" << endl;
4766 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4767 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4768 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4769 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4770 out << " // Check for BUILD.sh and execute" << endl;
4771 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4772 out << " printf(\"*******************************\\n\");" << endl;
4773 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4774 out << " printf(\"*******************************\\n\");" << endl;
4775 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4776 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4777 out << " gSystem->ChangeDirectory(cdir);" << endl;
4778 out << " return kFALSE;" << endl;
4779 out << " }" << endl;
4780 out << " } else {" << endl;
4781 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4782 out << " gSystem->ChangeDirectory(cdir);" << endl;
4783 out << " return kFALSE;" << endl;
4784 out << " }" << endl;
4785 out << " // Check for SETUP.C and execute" << endl;
4786 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4787 out << " printf(\"*******************************\\n\");" << endl;
4788 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4789 out << " printf(\"*******************************\\n\");" << endl;
4790 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4791 out << " } else {" << endl;
4792 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4793 out << " gSystem->ChangeDirectory(cdir);" << endl;
4794 out << " return kFALSE;" << endl;
4795 out << " }" << endl;
4796 out << " // Restore original workdir" << endl;
4797 out << " gSystem->ChangeDirectory(cdir);" << endl;
4798 out << " return kTRUE;" << endl;
4799 out << "}" << endl;
4800 }
0f389141 4801 }
4802 Bool_t copy = kTRUE;
d3b18c4c 4803 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4804 if (copy) {
4805 CdWork();
4806 TString workdir = gGrid->GetHomeDirectory();
4807 workdir += fGridWorkingDir;
4808 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4809 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4810// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4811 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4812 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4813 }
4814}
4815
4816//______________________________________________________________________________
4817Bool_t AliAnalysisAlien::SetupPar(const char *package)
4818{
205b201f 4819// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4820// Note that for loading the compiled library. The current directory should have precedence in
4821// LD_LIBRARY_PATH
4822 TString pkgdir = package;
4823 pkgdir.ReplaceAll(".par","");
b50a41c3 4824 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4825 TString cdir = gSystem->WorkingDirectory();
4826 gSystem->ChangeDirectory(pkgdir);
4827 // Check for BUILD.sh and execute
4828 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4829 printf("**************************************************\n");
4830 printf("*** Building PAR archive %s\n", package);
4831 printf("**************************************************\n");
4832 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4833 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4834 gSystem->ChangeDirectory(cdir);
4835 return kFALSE;
4836 }
4837 } else {
4838 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4839 gSystem->ChangeDirectory(cdir);
4840 return kFALSE;
4841 }
4842 // Check for SETUP.C and execute
4843 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4844 printf("**************************************************\n");
4845 printf("*** Setup PAR archive %s\n", package);
4846 printf("**************************************************\n");
4847 gROOT->Macro("PROOF-INF/SETUP.C");
4848 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4849 } else {
4850 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4851 gSystem->ChangeDirectory(cdir);
4852 return kFALSE;
4853 }
4854 // Restore original workdir
4855 gSystem->ChangeDirectory(cdir);
4856 return kTRUE;
4857}
4858
c57f56b7 4859//______________________________________________________________________________
4860void AliAnalysisAlien::WriteExecutable()
4861{
4862// Generate the alien executable script.
5aa94dc3 4863 // Patch executable with -x to catch error code
4864 if (fExecutableCommand.Contains("root") &&
4865 fExecutableCommand.Contains("-q") &&
4866 !fExecutableCommand.Contains("-x")) fExecutableCommand += " -x";
c57f56b7 4867 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4868 ofstream out;
4869 out.open(fExecutable.Data(), ios::out);
4870 if (out.bad()) {
5513444a 4871 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4872 return;
4873 }
4874 out << "#!/bin/bash" << endl;
2a4d5166 4875 // Make sure we can properly compile par files
4876 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4877 out << "echo \"=========================================\"" << endl;
4878 out << "echo \"############## PATH : ##############\"" << endl;
4879 out << "echo $PATH" << endl;
4880 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4881 out << "echo $LD_LIBRARY_PATH" << endl;
4882 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4883 out << "echo $ROOTSYS" << endl;
4884 out << "echo \"############## which root : ##############\"" << endl;
4885 out << "which root" << endl;
4886 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4887 out << "echo $ALICE_ROOT" << endl;
4888 out << "echo \"############## which aliroot : ##############\"" << endl;
4889 out << "which aliroot" << endl;
9c5ddadc 4890 out << "echo \"############## system limits : ##############\"" << endl;
4891 out << "ulimit -a" << endl;
4892 out << "echo \"############## memory : ##############\"" << endl;
4893 out << "free -m" << endl;
c57f56b7 4894 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4895 out << fExecutableCommand << " ";
370c9880 4896 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4897 out << "RET=$?" << endl;
4898 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4899 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4900 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4901 out << " let sig=\"$RET - 128\""<<endl;
4902 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4903 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4904 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4905 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4906 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4907 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4908 out << " fi"<<endl;
4909 out << " exit $RET"<< endl;
4910 out << "fi" << endl << endl ;
4911 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4912 out << "echo \"############## memory after: ##############\"" << endl;
4913 out << "free -m" << endl;
c57f56b7 4914 }
4915 Bool_t copy = kTRUE;
d3b18c4c 4916 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4917 if (copy) {
4918 CdWork();
4919 TString workdir = gGrid->GetHomeDirectory();
4920 workdir += fGridWorkingDir;
6da4839e 4921 TString executable = TString::Format("%s/%s", workdir.Data(), fExecutable.Data());
c57f56b7 4922 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4923 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4924 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4925 executable.Data())) Fatal("","Terminating");
c57f56b7 4926 }
4927}
4928
0f389141 4929//______________________________________________________________________________
4930void AliAnalysisAlien::WriteMergeExecutable()
4931{
4932// Generate the alien executable script for the merging job.
4933 if (!fMergeViaJDL) return;
4934 TString mergeExec = fExecutable;
4935 mergeExec.ReplaceAll(".sh", "_merge.sh");
4936 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4937 ofstream out;
4938 out.open(mergeExec.Data(), ios::out);
4939 if (out.bad()) {
4940 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4941 return;
4942 }
4943 out << "#!/bin/bash" << endl;
2a4d5166 4944 // Make sure we can properly compile par files
4945 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4946 out << "echo \"=========================================\"" << endl;
4947 out << "echo \"############## PATH : ##############\"" << endl;
4948 out << "echo $PATH" << endl;
4949 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4950 out << "echo $LD_LIBRARY_PATH" << endl;
4951 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4952 out << "echo $ROOTSYS" << endl;
4953 out << "echo \"############## which root : ##############\"" << endl;
4954 out << "which root" << endl;
4955 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4956 out << "echo $ALICE_ROOT" << endl;
4957 out << "echo \"############## which aliroot : ##############\"" << endl;
4958 out << "which aliroot" << endl;
4959 out << "echo \"############## system limits : ##############\"" << endl;
4960 out << "ulimit -a" << endl;
4961 out << "echo \"############## memory : ##############\"" << endl;
4962 out << "free -m" << endl;
4963 out << "echo \"=========================================\"" << endl << endl;
0f389141 4964 TString mergeMacro = fExecutable;
4965 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4966 if (IsOneStageMerging())
4967 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4968 else
f47d5cb4 4969 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4970 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4971 out << "RET=$?" << endl;
4972 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4973 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4974 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4975 out << " let sig=\"$RET - 128\""<<endl;
4976 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4977 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4978 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4979 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4980 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4981 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4982 out << " fi"<<endl;
4983 out << " exit $RET"<< endl;
4984 out << "fi" << endl << endl ;
0f389141 4985 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4986 out << "echo \"############## memory after: ##############\"" << endl;
4987 out << "free -m" << endl;
0f389141 4988 }
4989 Bool_t copy = kTRUE;
d3b18c4c 4990 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4991 if (copy) {
4992 CdWork();
4993 TString workdir = gGrid->GetHomeDirectory();
0f389141 4994 workdir += fGridWorkingDir;
6da4839e 4995 TString executable = TString::Format("%s/%s", workdir.Data(), mergeExec.Data());
0f389141 4996 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4997 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6da4839e
AG		 4998 if (!copyLocal2Alien("WriteMergeExecutable",mergeExec.Data(),
4999 executable.Data())) Fatal("","Terminating");
0f389141 5000 }
5001}
5002
c57f56b7 5003//______________________________________________________________________________
5513444a 5004void AliAnalysisAlien::WriteProductionFile(const char *filename) const
5005{
5006// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 5007// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 5008// Next lines: full_path_to_dataset XXX (XXX is a string)
5009// To submit, one has to: submit jdl XXX for all lines
5010 ofstream out;
5011 out.open(filename, ios::out);
5012 if (out.bad()) {
5013 Error("WriteProductionFile", "Bad file name: %s", filename);
5014 return;
5015 }
d3b18c4c 5016 TString workdir;
5017 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
5018 workdir = gGrid->GetHomeDirectory();
5513444a 5019 workdir += fGridWorkingDir;
f5e8c702 5020 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 5021 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 5022 out << locjdl << " " << njobspermaster << endl;
5513444a 5023 Int_t nmasterjobs = fInputFiles->GetEntries();
5024 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 5025 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
5026 runOutDir.ReplaceAll(".xml", "");
5027 if (fOutputToRunNo)
5028 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
5029 else
5030 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 5031 }
d3b18c4c 5032 if (gGrid) {
5033 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
5034 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 5035// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
5036 if (!copyLocal2Alien("WriteProductionFile", filename,
5037 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 5038 }
5513444a 5039}
5040
5041//______________________________________________________________________________
0f389141 5042void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 5043{
5044// Generate the alien validation script.
5045 // Generate the validation script
5046 TObjString *os;
d3b18c4c 5047 if (fValidationScript.IsNull()) {
5048 fValidationScript = fExecutable;
5049 fValidationScript.ReplaceAll(".sh", "_validation.sh");
5050 }
5051 TString validationScript = fValidationScript;
5052 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 5053 if (!Connect()) {
5054 Error("WriteValidationScript", "Alien connection required");
5055 return;
5056 }
d3b18c4c 5057 if (!fTerminateFiles.IsNull()) {
5058 fTerminateFiles.Strip();
5059 fTerminateFiles.ReplaceAll(" ",",");
5060 }
a2f5fc01 5061 TString outStream = "";
5062 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 5063 if (!TestBit(AliAnalysisGrid::kSubmit)) {
5064 ofstream out;
0d5d317c 5065 out.open(validationScript, ios::out);
c57f56b7 5066 out << "#!/bin/bash" << endl;
5067 out << "##################################################" << endl;
5068 out << "validateout=`dirname $0`" << endl;
5069 out << "validatetime=`date`" << endl;
5070 out << "validated=\"0\";" << endl;
5071 out << "error=0" << endl;
5072 out << "if [ -z $validateout ]" << endl;
5073 out << "then" << endl;
5074 out << " validateout=\".\"" << endl;
5075 out << "fi" << endl << endl;
5076 out << "cd $validateout;" << endl;
5077 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 5078 out << "echo \"*******************************************************\"" << outStream << endl;
5079 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 5080 out << "" << endl;
a2f5fc01 5081 out << "echo \"* Time: $validatetime \"" << outStream << endl;
5082 out << "echo \"* Dir: $validateout\"" << outStream << endl;
5083 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
5084 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5085 out << "ls -la ./" << outStream << endl;
5086 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 5087 out << "##################################################" << endl;
ebec370a 5088 out << "" << endl;
5089
5090 out << "if [ ! -f stderr ] ; then" << endl;
5091 out << " error=1" << endl;
a2f5fc01 5092 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
5093 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5094 out << "fi" << endl;
5095
b34c9f51 5096 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
5097 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
5098 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
a8ff3ea1 5099 out << "glibcErr=`grep -Ei '\\*\\*\\* glibc detected \\*\\*\\*' stderr`" << endl;
b34c9f51 5100 out << "" << endl;
5101
ebec370a 5102 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
5103 out << " error=1" << endl;
a2f5fc01 5104 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
5105 out << " echo \"$parArch\" " << outStream << endl;
5106 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5107 out << "fi" << endl;
5108
5109 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
5110 out << " error=1" << endl;
a2f5fc01 5111 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
5112 out << " echo \"$segViol\" " << outStream << endl;
5113 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5114 out << "fi" << endl;
5115
5116 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
5117 out << " error=1" << endl;
a2f5fc01 5118 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
5119 out << " echo \"$segFault\" " << outStream << endl;
5120 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 5121 out << "fi" << endl;
5122
b34c9f51 5123 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
5124 out << " error=1" << endl;
a2f5fc01 5125 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
5126 out << " echo \"$glibcErr\" " << outStream << endl;
5127 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 5128 out << "fi" << endl;
5129
ebec370a 5130 // Part dedicated to the specific analyses running into the train
5131
d3b18c4c 5132 TString outputFiles = fOutputFiles;
5133 if (merge && !fTerminateFiles.IsNull()) {
0c0825e4 5134 if (!outputFiles.IsNull()) outputFiles += ",";
d3b18c4c 5135 outputFiles += fTerminateFiles;
5136 }
5137 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 5138 TIter next1(arr);
a2f5fc01 5139 TString outputFile;
bbd88db8 5140 while (!merge && (os=(TObjString*)next1())) {
5141 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 5142 outputFile = os->GetString();
5143 Int_t index = outputFile.Index("@");
5144 if (index > 0) outputFile.Remove(index);
bbd88db8 5145 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 5146 if (outputFile.Contains("*")) continue;
5147 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 5148 out << " error=1" << endl;
7c2cd90a 5149 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
5150 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 5151 out << "fi" << endl;
5152 }
5153 delete arr;
7c2cd90a 5154 out << "if ! [ -f outputs_valid ] ; then" << endl;
5155 out << " error=1" << endl;
5156 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
5157 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
5158 out << "fi" << endl;
923e2ca5 5159
c57f56b7 5160 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 5161 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 5162 if (!IsKeepLogs()) {
5163 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 5164 outStream = "";
149d288c 5165 out << " rm -f std*" << endl;
5166 }
c57f56b7 5167 out << "fi" << endl;
5168
a2f5fc01 5169 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5170 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 5171 out << "cd -" << endl;
5172 out << "exit $error" << endl;
5173 }
5174 Bool_t copy = kTRUE;
d3b18c4c 5175 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 5176 if (copy) {
5177 CdWork();
5178 TString workdir = gGrid->GetHomeDirectory();
5179 workdir += fGridWorkingDir;
d3b18c4c 5180 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 5181 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 5182// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
5183 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
5184 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 5185 }
5186}