Added possibility to declare list of libraries needed to read friends info, extending...
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
c82bb898 51using std::ofstream;
52using std::ifstream;
53using std::ios;
54using std::endl;
c57f56b7 55ClassImp(AliAnalysisAlien)
6c395669 56#if 0
57;
58#endif
c57f56b7 59
6c395669 60namespace {
61 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
62 {
63 TString sl(Form("file:%s", loc));
64 TString sr(Form("alien://%s", rem));
65 Bool_t ret = TFile::Cp(sl, sr);
66 if (!ret) {
67 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
68 }
69 return ret;
70 }
71}
72
c57f56b7 73//______________________________________________________________________________
74AliAnalysisAlien::AliAnalysisAlien()
75 :AliAnalysisGrid(),
76 fGridJDL(NULL),
0f389141 77 fMergingJDL(NULL),
c57f56b7 78 fPrice(0),
79 fTTL(0),
80 fSplitMaxInputFileNumber(0),
81 fMaxInitFailed(0),
82 fMasterResubmitThreshold(0),
bb885a9e 83 fNtestFiles(0),
319593fb 84 fNrunsPerMaster(0),
16a4353c 85 fMaxMergeFiles(0),
b3e07543 86 fMaxMergeStages(0),
d2a409b2 87 fNsubmitted(0),
a3e84053 88 fProductionMode(0),
cd11251e 89 fOutputToRunNo(0),
0f389141 90 fMergeViaJDL(0),
91 fFastReadOption(0),
e1c22e21 92 fOverwriteMode(1),
149d288c 93 fNreplicas(2),
3bdcb562 94 fNproofWorkers(0),
27734f0e 95 fNproofWorkersPerSlave(0),
3bdcb562 96 fProofReset(0),
c57f56b7 97 fRunNumbers(),
98 fExecutable(),
0a1c1f7f 99 fExecutableCommand(),
c57f56b7 100 fArguments(),
631c0b05 101 fExecutableArgs(),
c57f56b7 102 fAnalysisMacro(),
103 fAnalysisSource(),
d3b18c4c 104 fValidationScript(),
d5c6455a 105 fAdditionalRootLibs(),
c57f56b7 106 fAdditionalLibs(),
107 fSplitMode(),
108 fAPIVersion(),
109 fROOTVersion(),
110 fAliROOTVersion(),
648174cf 111 fExternalPackages(),
c57f56b7 112 fUser(),
113 fGridWorkingDir(),
114 fGridDataDir(),
115 fDataPattern(),
116 fGridOutputDir(),
117 fOutputArchive(),
118 fOutputFiles(),
119 fInputFormat(),
e7c71df0 120 fDatasetName(),
c57f56b7 121 fJDLName(),
d3b18c4c 122 fTerminateFiles(),
bb885a9e 123 fMergeExcludes(),
37d21c01 124 fRegisterExcludes(),
f965131e 125 fIncludePath(),
bb885a9e 126 fCloseSE(),
0df6ccf2 127 fFriendChainName(),
c6cb3634 128 fJobTag(),
648174cf 129 fOutputSingle(),
5fce53f4 130 fRunPrefix(),
3bdcb562 131 fProofCluster(),
132 fProofDataSet(),
133 fFileForTestMode(),
3bdcb562 134 fAliRootMode(),
d88e4159 135 fProofProcessOpt(),
f47d5cb4 136 fMergeDirName(),
4e5c5506 137 fInputFiles(0),
28ce6c9c 138 fPackages(0),
77f34eae 139 fModules(0),
d88e4159 140 fProofParam(),
a41b2882 141 fDropToShell(true),
142 fGridJobIDs(""),
143 fGridStages(""),
144 fFriendLibs("")
c57f56b7 145{
146// Dummy ctor.
147 SetDefaults();
148}
149
150//______________________________________________________________________________
151AliAnalysisAlien::AliAnalysisAlien(const char *name)
152 :AliAnalysisGrid(name),
153 fGridJDL(NULL),
0f389141 154 fMergingJDL(NULL),
c57f56b7 155 fPrice(0),
156 fTTL(0),
157 fSplitMaxInputFileNumber(0),
158 fMaxInitFailed(0),
159 fMasterResubmitThreshold(0),
bb885a9e 160 fNtestFiles(0),
319593fb 161 fNrunsPerMaster(0),
16a4353c 162 fMaxMergeFiles(0),
b3e07543 163 fMaxMergeStages(0),
d2a409b2 164 fNsubmitted(0),
a3e84053 165 fProductionMode(0),
cd11251e 166 fOutputToRunNo(0),
0f389141 167 fMergeViaJDL(0),
168 fFastReadOption(0),
e1c22e21 169 fOverwriteMode(1),
149d288c 170 fNreplicas(2),
3bdcb562 171 fNproofWorkers(0),
27734f0e 172 fNproofWorkersPerSlave(0),
3bdcb562 173 fProofReset(0),
c57f56b7 174 fRunNumbers(),
175 fExecutable(),
0a1c1f7f 176 fExecutableCommand(),
c57f56b7 177 fArguments(),
631c0b05 178 fExecutableArgs(),
c57f56b7 179 fAnalysisMacro(),
180 fAnalysisSource(),
d3b18c4c 181 fValidationScript(),
d5c6455a 182 fAdditionalRootLibs(),
c57f56b7 183 fAdditionalLibs(),
184 fSplitMode(),
185 fAPIVersion(),
186 fROOTVersion(),
187 fAliROOTVersion(),
648174cf 188 fExternalPackages(),
c57f56b7 189 fUser(),
190 fGridWorkingDir(),
191 fGridDataDir(),
192 fDataPattern(),
193 fGridOutputDir(),
194 fOutputArchive(),
195 fOutputFiles(),
196 fInputFormat(),
e7c71df0 197 fDatasetName(),
c57f56b7 198 fJDLName(),
d3b18c4c 199 fTerminateFiles(),
bb885a9e 200 fMergeExcludes(),
37d21c01 201 fRegisterExcludes(),
f965131e 202 fIncludePath(),
bb885a9e 203 fCloseSE(),
0df6ccf2 204 fFriendChainName(),
c6cb3634 205 fJobTag(),
648174cf 206 fOutputSingle(),
5fce53f4 207 fRunPrefix(),
3bdcb562 208 fProofCluster(),
209 fProofDataSet(),
210 fFileForTestMode(),
3bdcb562 211 fAliRootMode(),
d88e4159 212 fProofProcessOpt(),
f47d5cb4 213 fMergeDirName(),
4e5c5506 214 fInputFiles(0),
28ce6c9c 215 fPackages(0),
77f34eae 216 fModules(0),
d88e4159 217 fProofParam(),
0ad3ea1f 218 fDropToShell(true),
219 fGridJobIDs(""),
a41b2882 220 fGridStages(""),
221 fFriendLibs("")
c57f56b7 222{
223// Default ctor.
224 SetDefaults();
225}
226
227//______________________________________________________________________________
228AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
229 :AliAnalysisGrid(other),
230 fGridJDL(NULL),
0f389141 231 fMergingJDL(NULL),
c57f56b7 232 fPrice(other.fPrice),
233 fTTL(other.fTTL),
234 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
235 fMaxInitFailed(other.fMaxInitFailed),
236 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 237 fNtestFiles(other.fNtestFiles),
319593fb 238 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 239 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 240 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 241 fNsubmitted(other.fNsubmitted),
a3e84053 242 fProductionMode(other.fProductionMode),
cd11251e 243 fOutputToRunNo(other.fOutputToRunNo),
0f389141 244 fMergeViaJDL(other.fMergeViaJDL),
245 fFastReadOption(other.fFastReadOption),
246 fOverwriteMode(other.fOverwriteMode),
149d288c 247 fNreplicas(other.fNreplicas),
3bdcb562 248 fNproofWorkers(other.fNproofWorkers),
27734f0e 249 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 250 fProofReset(other.fProofReset),
c57f56b7 251 fRunNumbers(other.fRunNumbers),
252 fExecutable(other.fExecutable),
0a1c1f7f 253 fExecutableCommand(other.fExecutableCommand),
c57f56b7 254 fArguments(other.fArguments),
631c0b05 255 fExecutableArgs(other.fExecutableArgs),
c57f56b7 256 fAnalysisMacro(other.fAnalysisMacro),
257 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 258 fValidationScript(other.fValidationScript),
d5c6455a 259 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 260 fAdditionalLibs(other.fAdditionalLibs),
261 fSplitMode(other.fSplitMode),
262 fAPIVersion(other.fAPIVersion),
263 fROOTVersion(other.fROOTVersion),
264 fAliROOTVersion(other.fAliROOTVersion),
648174cf 265 fExternalPackages(other.fExternalPackages),
c57f56b7 266 fUser(other.fUser),
267 fGridWorkingDir(other.fGridWorkingDir),
268 fGridDataDir(other.fGridDataDir),
269 fDataPattern(other.fDataPattern),
270 fGridOutputDir(other.fGridOutputDir),
271 fOutputArchive(other.fOutputArchive),
272 fOutputFiles(other.fOutputFiles),
273 fInputFormat(other.fInputFormat),
e7c71df0 274 fDatasetName(other.fDatasetName),
c57f56b7 275 fJDLName(other.fJDLName),
d3b18c4c 276 fTerminateFiles(other.fTerminateFiles),
bb885a9e 277 fMergeExcludes(other.fMergeExcludes),
37d21c01 278 fRegisterExcludes(other.fRegisterExcludes),
f965131e 279 fIncludePath(other.fIncludePath),
bb885a9e 280 fCloseSE(other.fCloseSE),
0df6ccf2 281 fFriendChainName(other.fFriendChainName),
c6cb3634 282 fJobTag(other.fJobTag),
648174cf 283 fOutputSingle(other.fOutputSingle),
5fce53f4 284 fRunPrefix(other.fRunPrefix),
3bdcb562 285 fProofCluster(other.fProofCluster),
286 fProofDataSet(other.fProofDataSet),
287 fFileForTestMode(other.fFileForTestMode),
3bdcb562 288 fAliRootMode(other.fAliRootMode),
d88e4159 289 fProofProcessOpt(other.fProofProcessOpt),
f47d5cb4 290 fMergeDirName(other.fMergeDirName),
4e5c5506 291 fInputFiles(0),
28ce6c9c 292 fPackages(0),
77f34eae 293 fModules(0),
d88e4159 294 fProofParam(),
0ad3ea1f 295 fDropToShell(other.fDropToShell),
296 fGridJobIDs(other.fGridJobIDs),
a41b2882 297 fGridStages(other.fGridStages),
298 fFriendLibs(other.fFriendLibs)
c57f56b7 299{
300// Copy ctor.
301 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 302 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 303 fRunRange[0] = other.fRunRange[0];
304 fRunRange[1] = other.fRunRange[1];
c57f56b7 305 if (other.fInputFiles) {
306 fInputFiles = new TObjArray();
307 TIter next(other.fInputFiles);
308 TObject *obj;
309 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
310 fInputFiles->SetOwner();
311 }
4e5c5506 312 if (other.fPackages) {
313 fPackages = new TObjArray();
314 TIter next(other.fPackages);
315 TObject *obj;
316 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
317 fPackages->SetOwner();
318 }
77f34eae 319 if (other.fModules) {
320 fModules = new TObjArray();
321 fModules->SetOwner();
322 TIter next(other.fModules);
323 AliAnalysisTaskCfg *mod, *crt;
324 while ((crt=(AliAnalysisTaskCfg*)next())) {
325 mod = new AliAnalysisTaskCfg(*crt);
326 fModules->Add(mod);
327 }
328 }
c57f56b7 329}
330
331//______________________________________________________________________________
332AliAnalysisAlien::~AliAnalysisAlien()
333{
334// Destructor.
77f34eae 335 delete fGridJDL;
336 delete fMergingJDL;
337 delete fInputFiles;
338 delete fPackages;
339 delete fModules;
28ce6c9c 340 fProofParam.DeleteAll();
c57f56b7 341}
342
343//______________________________________________________________________________
344AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
345{
346// Assignment.
347 if (this != &other) {
348 AliAnalysisGrid::operator=(other);
349 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 350 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 351 fPrice = other.fPrice;
352 fTTL = other.fTTL;
353 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
354 fMaxInitFailed = other.fMaxInitFailed;
355 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 356 fNtestFiles = other.fNtestFiles;
a3e84053 357 fNrunsPerMaster = other.fNrunsPerMaster;
358 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 359 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 360 fNsubmitted = other.fNsubmitted;
361 fProductionMode = other.fProductionMode;
cd11251e 362 fOutputToRunNo = other.fOutputToRunNo;
0f389141 363 fMergeViaJDL = other.fMergeViaJDL;
364 fFastReadOption = other.fFastReadOption;
365 fOverwriteMode = other.fOverwriteMode;
149d288c 366 fNreplicas = other.fNreplicas;
3bdcb562 367 fNproofWorkers = other.fNproofWorkers;
27734f0e 368 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 369 fProofReset = other.fProofReset;
c57f56b7 370 fRunNumbers = other.fRunNumbers;
371 fExecutable = other.fExecutable;
0a1c1f7f 372 fExecutableCommand = other.fExecutableCommand;
c57f56b7 373 fArguments = other.fArguments;
631c0b05 374 fExecutableArgs = other.fExecutableArgs;
c57f56b7 375 fAnalysisMacro = other.fAnalysisMacro;
376 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 377 fValidationScript = other.fValidationScript;
d5c6455a 378 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 379 fAdditionalLibs = other.fAdditionalLibs;
380 fSplitMode = other.fSplitMode;
381 fAPIVersion = other.fAPIVersion;
382 fROOTVersion = other.fROOTVersion;
383 fAliROOTVersion = other.fAliROOTVersion;
648174cf 384 fExternalPackages = other.fExternalPackages;
c57f56b7 385 fUser = other.fUser;
386 fGridWorkingDir = other.fGridWorkingDir;
387 fGridDataDir = other.fGridDataDir;
388 fDataPattern = other.fDataPattern;
389 fGridOutputDir = other.fGridOutputDir;
390 fOutputArchive = other.fOutputArchive;
391 fOutputFiles = other.fOutputFiles;
392 fInputFormat = other.fInputFormat;
e7c71df0 393 fDatasetName = other.fDatasetName;
c57f56b7 394 fJDLName = other.fJDLName;
d3b18c4c 395 fTerminateFiles = other.fTerminateFiles;
bb885a9e 396 fMergeExcludes = other.fMergeExcludes;
37d21c01 397 fRegisterExcludes = other.fRegisterExcludes;
f965131e 398 fIncludePath = other.fIncludePath;
bb885a9e 399 fCloseSE = other.fCloseSE;
0df6ccf2 400 fFriendChainName = other.fFriendChainName;
c6cb3634 401 fJobTag = other.fJobTag;
648174cf 402 fOutputSingle = other.fOutputSingle;
5fce53f4 403 fRunPrefix = other.fRunPrefix;
3bdcb562 404 fProofCluster = other.fProofCluster;
405 fProofDataSet = other.fProofDataSet;
406 fFileForTestMode = other.fFileForTestMode;
3bdcb562 407 fAliRootMode = other.fAliRootMode;
d88e4159 408 fProofProcessOpt = other.fProofProcessOpt;
f47d5cb4 409 fMergeDirName = other.fMergeDirName;
d88e4159 410 fDropToShell = other.fDropToShell;
0ad3ea1f 411 fGridJobIDs = other.fGridJobIDs;
412 fGridStages = other.fGridStages;
a41b2882 413 fFriendLibs = other.fFriendLibs;
c57f56b7 414 if (other.fInputFiles) {
415 fInputFiles = new TObjArray();
416 TIter next(other.fInputFiles);
417 TObject *obj;
418 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
419 fInputFiles->SetOwner();
420 }
4e5c5506 421 if (other.fPackages) {
422 fPackages = new TObjArray();
423 TIter next(other.fPackages);
424 TObject *obj;
425 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
426 fPackages->SetOwner();
427 }
77f34eae 428 if (other.fModules) {
429 fModules = new TObjArray();
430 fModules->SetOwner();
431 TIter next(other.fModules);
432 AliAnalysisTaskCfg *mod, *crt;
433 while ((crt=(AliAnalysisTaskCfg*)next())) {
434 mod = new AliAnalysisTaskCfg(*crt);
435 fModules->Add(mod);
436 }
437 }
c57f56b7 438 }
439 return *this;
440}
441
442//______________________________________________________________________________
37d21c01 443void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
444{
445// Add a single additional library to be loaded. Extension must be present.
446 TString lib(name);
447 if (!lib.Contains(".")) {
448 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
449 return;
450 }
451 if (fAdditionalLibs.Contains(name)) {
452 Warning("AddAdditionalLibrary", "Library %s already added.", name);
453 return;
454 }
455 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
456 fAdditionalLibs += lib;
457}
458
459//______________________________________________________________________________
77f34eae 460void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
461{
462// Adding a module. Checks if already existing. Becomes owned by this.
463 if (!module) return;
464 if (GetModule(module->GetName())) {
465 Error("AddModule", "A module having the same name %s already added", module->GetName());
466 return;
467 }
468 if (!fModules) {
469 fModules = new TObjArray();
470 fModules->SetOwner();
471 }
472 fModules->Add(module);
473}
474
475//______________________________________________________________________________
476void AliAnalysisAlien::AddModules(TObjArray *list)
477{
478// Adding a list of modules. Checks if already existing. Becomes owned by this.
479 TIter next(list);
480 AliAnalysisTaskCfg *module;
481 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
482}
483
484//______________________________________________________________________________
485Bool_t AliAnalysisAlien::CheckDependencies()
486{
487// Check if all dependencies are satisfied. Reorder modules if needed.
488 Int_t nmodules = GetNmodules();
489 if (!nmodules) {
490 Warning("CheckDependencies", "No modules added yet to check their dependencies");
491 return kTRUE;
492 }
493 AliAnalysisTaskCfg *mod = 0;
494 AliAnalysisTaskCfg *dep = 0;
495 TString depname;
496 Int_t i, j, k;
497 for (i=0; i<nmodules; i++) {
498 mod = (AliAnalysisTaskCfg*) fModules->At(i);
499 Int_t ndeps = mod->GetNdeps();
500 Int_t istart = i;
501 for (j=0; j<ndeps; j++) {
502 depname = mod->GetDependency(j);
503 dep = GetModule(depname);
504 if (!dep) {
505 Error("CheckDependencies","Dependency %s not added for module %s",
506 depname.Data(), mod->GetName());
507 return kFALSE;
508 }
509 if (dep->NeedsDependency(mod->GetName())) {
510 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
511 mod->GetName(), dep->GetName());
512 return kFALSE;
513 }
514 Int_t idep = fModules->IndexOf(dep);
515 // The dependency task must come first
516 if (idep>i) {
517 // Remove at idep and move all objects below up one slot
518 // down to index i included.
519 fModules->RemoveAt(idep);
3e40fd4c 520 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 521 fModules->AddAt(dep, i++);
522 }
523 //Redo from istart if dependencies were inserted
524 if (i>istart) i=istart-1;
525 }
526 }
527 return kTRUE;
528}
529
530//______________________________________________________________________________
3e40fd4c 531AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
532{
533// Create the analysis manager and optionally execute the macro in filename.
534 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
535 if (mgr) return mgr;
536 mgr = new AliAnalysisManager(name);
537 mgr->SetGridHandler((AliAnalysisGrid*)this);
538 if (strlen(filename)) {
539 TString line = gSystem->ExpandPathName(filename);
540 line.Prepend(".x ");
541 gROOT->ProcessLine(line.Data());
542 }
543 return mgr;
544}
545
546//______________________________________________________________________________
77f34eae 547Int_t AliAnalysisAlien::GetNmodules() const
548{
549// Get number of modules.
550 if (!fModules) return 0;
551 return fModules->GetEntries();
552}
553
554//______________________________________________________________________________
555AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
556{
557// Get a module by name.
558 if (!fModules) return 0;
559 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
560}
561
562//______________________________________________________________________________
563Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
564{
565// Load a given module.
566 if (mod->IsLoaded()) return kTRUE;
192ddca2 567 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
568 if (!mgr) {
569 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
570 return kFALSE;
571 }
77f34eae 572 Int_t ndeps = mod->GetNdeps();
573 TString depname;
574 for (Int_t j=0; j<ndeps; j++) {
575 depname = mod->GetDependency(j);
576 AliAnalysisTaskCfg *dep = GetModule(depname);
577 if (!dep) {
578 Error("LoadModule","Dependency %s not existing for module %s",
579 depname.Data(), mod->GetName());
580 return kFALSE;
581 }
582 if (!LoadModule(dep)) {
583 Error("LoadModule","Dependency %s for module %s could not be loaded",
584 depname.Data(), mod->GetName());
585 return kFALSE;
586 }
587 }
588 // Load libraries for the module
589 if (!mod->CheckLoadLibraries()) {
590 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
591 return kFALSE;
592 }
192ddca2 593 // Check if a custom file name was requested
594 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
595
596 // Check if a custom terminate file name was requested
597 if (strlen(mod->GetTerminateFileName())) {
598 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
599 fTerminateFiles += mod->GetTerminateFileName();
600 }
601
77f34eae 602 // Execute the macro
603 if (mod->ExecuteMacro()<0) {
604 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
605 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
606 return kFALSE;
607 }
608 // Configure dependencies
609 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
610 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
611 mod->GetConfigMacro()->GetTitle(), mod->GetName());
612 return kFALSE;
613 }
5e8c7c22 614 // Adjust extra libraries
615 Int_t nlibs = mod->GetNlibs();
616 TString lib;
617 for (Int_t i=0; i<nlibs; i++) {
618 lib = mod->GetLibrary(i);
5e8c7c22 619 lib = Form("lib%s.so", lib.Data());
c9d6bd12 620 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 621 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
622 fAdditionalLibs += lib;
623 }
77f34eae 624 return kTRUE;
625}
626
627//______________________________________________________________________________
139fe6ba 628Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
629{
630// Generate the full train.
631 fAdditionalLibs = "";
632 if (!LoadModules()) return kFALSE;
633 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
634 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 635 mgr->RunLocalInit();
139fe6ba 636 mgr->PrintStatus();
637 Int_t productionMode = fProductionMode;
638 SetProductionMode();
639 TString macro = fAnalysisMacro;
640 TString executable = fExecutable;
641 TString validation = fValidationScript;
642 TString execCommand = fExecutableCommand;
643 SetAnalysisMacro(Form("%s.C", name));
644 SetExecutable(Form("%s.sh", name));
a8e31fb0 645// SetExecutableCommand("aliroot -b -q ");
139fe6ba 646 SetValidationScript(Form("%s_validation.sh", name));
647 StartAnalysis();
648 SetProductionMode(productionMode);
649 fAnalysisMacro = macro;
650 fExecutable = executable;
651 fExecutableCommand = execCommand;
652 fValidationScript = validation;
653 return kTRUE;
654}
655
656//______________________________________________________________________________
21ca8e59 657Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 658{
659// Generate test macros for a single module or for the full train.
5e8c7c22 660 fAdditionalLibs = "";
77f34eae 661 if (strlen(modname)) {
21ca8e59 662 if (!CheckDependencies()) return kFALSE;
663 AliAnalysisTaskCfg *mod = GetModule(modname);
664 if (!mod) {
665 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
666 return kFALSE;
667 }
668 if (!LoadModule(mod)) return kFALSE;
a41b2882 669 if (!LoadFriendLibs()) return kFALSE;
21ca8e59 670 } else if (!LoadModules()) return kFALSE;
671 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
672 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 673 mgr->RunLocalInit();
21ca8e59 674 mgr->PrintStatus();
675 SetLocalTest(kTRUE);
676 Int_t productionMode = fProductionMode;
677 SetProductionMode();
678 TString macro = fAnalysisMacro;
679 TString executable = fExecutable;
680 TString validation = fValidationScript;
681 TString execCommand = fExecutableCommand;
682 SetAnalysisMacro(Form("%s.C", name));
683 SetExecutable(Form("%s.sh", name));
d66aa2a9 684 fOutputFiles = GetListOfFiles("outaod");
685 // Add extra files registered to the analysis manager
686 TString extra = GetListOfFiles("ext");
687 if (!extra.IsNull()) {
688 extra.ReplaceAll(".root", "*.root");
689 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
690 fOutputFiles += extra;
691 }
a8e31fb0 692// SetExecutableCommand("aliroot -b -q ");
21ca8e59 693 SetValidationScript(Form("%s_validation.sh", name));
694 WriteAnalysisFile();
695 WriteAnalysisMacro();
696 WriteExecutable();
697 WriteValidationScript();
ffbe06b9 698 WriteMergingMacro();
699 WriteMergeExecutable();
700 WriteValidationScript(kTRUE);
21ca8e59 701 SetLocalTest(kFALSE);
702 SetProductionMode(productionMode);
703 fAnalysisMacro = macro;
704 fExecutable = executable;
705 fExecutableCommand = execCommand;
706 fValidationScript = validation;
77f34eae 707 return kTRUE;
708}
709
710//______________________________________________________________________________
711Bool_t AliAnalysisAlien::LoadModules()
712{
713// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 714 fAdditionalLibs = "";
21ca8e59 715 Int_t nmodules = GetNmodules();
716 if (!nmodules) {
717 Warning("LoadModules", "No module to be loaded");
718 return kTRUE;
719 }
3e40fd4c 720 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
721 if (!mgr) {
722 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
723 return kFALSE;
724 }
77f34eae 725 if (!CheckDependencies()) return kFALSE;
21ca8e59 726 nmodules = GetNmodules();
77f34eae 727 AliAnalysisTaskCfg *mod;
21ca8e59 728 for (Int_t imod=0; imod<nmodules; imod++) {
729 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 730 if (!LoadModule(mod)) return kFALSE;
731 }
a41b2882 732 // Load additional friend libraries
733 return LoadFriendLibs();
77f34eae 734}
735
736//______________________________________________________________________________
a41b2882 737Bool_t AliAnalysisAlien::LoadFriendLibs() const
738{
739// Load libraries required for reading friends.
740 if (fFriendLibs.Length()) {
741 TObjArray *list = 0;
742 TString lib;
743 if (fFriendLibs.Contains(",")) list = fFriendLibs.Tokenize(",");
744 else list = fFriendLibs.Tokenize(" ");
745 for (Int_t ilib=0; ilib<list->GetEntriesFast(); ilib++) {
746 lib = list->At(ilib)->GetName();
747 lib.ReplaceAll(".so","");
748 lib.ReplaceAll(" ","");
749 if (lib.BeginsWith("lib")) lib.Remove(0, 3);
750 lib.Prepend("lib");
751 Int_t loaded = strlen(gSystem->GetLibraries(lib,"",kFALSE));
752 if (!loaded) loaded = gSystem->Load(lib);
753 if (loaded < 0) {
754 Error("LoadModules", "Cannot load library for friends %s", lib.Data());
755 return kFALSE;
756 }
757 }
758 delete list;
759 }
760 return kTRUE;
761}
762
763//______________________________________________________________________________
74013535 764void AliAnalysisAlien::SetRunPrefix(const char *prefix)
765{
766// Set the run number format. Can be a prefix or a format like "%09d"
767 fRunPrefix = prefix;
768 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
769}
770
771//______________________________________________________________________________
f965131e 772void AliAnalysisAlien::AddIncludePath(const char *path)
773{
774// Add include path in the remote analysis macro.
775 TString p(path);
776 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
777 else fIncludePath += Form("-I%s ", path);
778}
779
780//______________________________________________________________________________
c57f56b7 781void AliAnalysisAlien::AddRunNumber(Int_t run)
782{
783// Add a run number to the list of runs to be processed.
784 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 785 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 786}
787
788//______________________________________________________________________________
98690895 789void AliAnalysisAlien::AddRunList(const char* runList)
790{
791// Add several runs into the list of runs; they are expected to be separated by a blank character.
792 TString sList = runList;
793 TObjArray *list = sList.Tokenize(" ");
794 Int_t n = list->GetEntries();
795 for (Int_t i = 0; i < n; i++) {
796 TObjString *os = (TObjString*)list->At(i);
797 AddRunNumber(os->GetString().Atoi());
798 }
799 delete list;
800}
801
802//______________________________________________________________________________
ee75cfc3 803void AliAnalysisAlien::AddRunNumber(const char* run)
804{
805// Add a run number to the list of runs to be processed.
7aa5d539 806 TString runs = run;
807 TObjString *os;
808 TObjArray *arr = runs.Tokenize(" ");
809 TIter next(arr);
810 TString prefix;
811 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
812 while ((os=(TObjString*)next())){
813 if (fRunNumbers.Length()) fRunNumbers += " ";
814 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
815 }
816 delete arr;
ee75cfc3 817}
818
819//______________________________________________________________________________
c57f56b7 820void AliAnalysisAlien::AddDataFile(const char *lfn)
821{
822// Adds a data file to the input to be analysed. The file should be a valid LFN
823// or point to an existing file in the alien workdir.
824 if (!fInputFiles) fInputFiles = new TObjArray();
825 fInputFiles->Add(new TObjString(lfn));
826}
648174cf 827
828//______________________________________________________________________________
829void AliAnalysisAlien::AddExternalPackage(const char *package)
830{
831// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
832 if (fExternalPackages) fExternalPackages += " ";
833 fExternalPackages += package;
834}
835
c57f56b7 836//______________________________________________________________________________
837Bool_t AliAnalysisAlien::Connect()
838{
839// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
840 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 841 if (fProductionMode) return kTRUE;
c57f56b7 842 if (!gGrid) {
843 Info("Connect", "Trying to connect to AliEn ...");
844 TGrid::Connect("alien://");
845 }
846 if (!gGrid || !gGrid->IsConnected()) {
847 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
848 return kFALSE;
849 }
850 fUser = gGrid->GetUser();
851 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
852 return kTRUE;
853}
854
855//______________________________________________________________________________
856void AliAnalysisAlien::CdWork()
857{
858// Check validity of alien workspace. Create directory if possible.
859 if (!Connect()) {
860 Error("CdWork", "Alien connection required");
861 return;
862 }
863 TString homedir = gGrid->GetHomeDirectory();
864 TString workdir = homedir + fGridWorkingDir;
923e2ca5 865 if (DirectoryExists(workdir)) {
866 gGrid->Cd(workdir);
867 return;
868 }
869 // Work directory not existing - create it
870 gGrid->Cd(homedir);
b93f8109 871 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 872 gGrid->Cd(fGridWorkingDir);
d3b18c4c 873 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 874 } else {
d3b18c4c 875 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 876 workdir.Data(), homedir.Data());
877 fGridWorkingDir = "";
878 }
c57f56b7 879}
880
881//______________________________________________________________________________
348be253 882Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
883{
884// Check if file copying is possible.
d3b18c4c 885 if (fProductionMode) return kTRUE;
b5ae4fe0 886 TString salienpath(alienpath);
887 if (salienpath.Contains(" ")) {
888 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
889 return kFALSE;
890 }
348be253 891 if (!Connect()) {
892 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
893 return kFALSE;
894 }
d3339be3 895 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
896 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 897 // Check if alien_CLOSE_SE is defined
898 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
899 if (!closeSE.IsNull()) {
900 Info("CheckFileCopy", "Your current close storage is pointing to: \
901 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
902 } else {
903 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
904 }
905 // Check if grid directory exists.
906 if (!DirectoryExists(alienpath)) {
907 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
908 return kFALSE;
909 }
b5ae4fe0 910 TString stest = "plugin_test_copy";
911 TFile f(stest, "RECREATE");
348be253 912 // User may not have write permissions to current directory
913 if (f.IsZombie()) {
914 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
915 gSystem->WorkingDirectory());
916 return kFALSE;
917 }
918 f.Close();
b5ae4fe0 919 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
920 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 921 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 922 \n# 1. Make sure you have write permissions there. If this is the case: \
923 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
924 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
925 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
926 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 927 gSystem->Unlink(stest.Data());
348be253 928 return kFALSE;
929 }
b5ae4fe0 930 gSystem->Unlink(stest.Data());
931 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 932 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 933 return kTRUE;
934}
935
936//______________________________________________________________________________
c57f56b7 937Bool_t AliAnalysisAlien::CheckInputData()
938{
939// Check validity of input data. If necessary, create xml files.
d3b18c4c 940 if (fProductionMode) return kTRUE;
a8739e8a 941 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 942 if (!fGridDataDir.Length()) {
943 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
944 return kFALSE;
945 }
9f020952 946 if (fMergeViaJDL) {
947 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
948 return kFALSE;
949 }
d2a409b2 950 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 951 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
952 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 953 return kTRUE;
c57f56b7 954 }
955 // Process declared files
a2f5fc01 956 Bool_t isCollection = kFALSE;
957 Bool_t isXml = kFALSE;
958 Bool_t useTags = kFALSE;
c57f56b7 959 Bool_t checked = kFALSE;
d3b18c4c 960 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 961 TString file;
962 TString workdir = gGrid->GetHomeDirectory();
963 workdir += fGridWorkingDir;
964 if (fInputFiles) {
965 TObjString *objstr;
966 TIter next(fInputFiles);
967 while ((objstr=(TObjString*)next())) {
968 file = workdir;
969 file += "/";
970 file += objstr->GetString();
971 // Store full lfn path
972 if (FileExists(file)) objstr->SetString(file);
973 else {
974 file = objstr->GetName();
975 if (!FileExists(objstr->GetName())) {
976 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
977 objstr->GetName(), workdir.Data());
978 return kFALSE;
979 }
980 }
981 Bool_t iscoll, isxml, usetags;
982 CheckDataType(file, iscoll, isxml, usetags);
983 if (!checked) {
984 checked = kTRUE;
a2f5fc01 985 isCollection = iscoll;
986 isXml = isxml;
987 useTags = usetags;
988 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 989 } else {
a2f5fc01 990 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 991 Error("CheckInputData", "Some conflict was found in the types of inputs");
992 return kFALSE;
993 }
994 }
995 }
996 }
997 // Process requested run numbers
a8739e8a 998 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 999 // Check validity of alien data directory
1000 if (!fGridDataDir.Length()) {
1001 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
1002 return kFALSE;
1003 }
923e2ca5 1004 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 1005 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
1006 return kFALSE;
1007 }
a2f5fc01 1008 if (isCollection) {
c57f56b7 1009 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
1010 return kFALSE;
1011 }
1012
a2f5fc01 1013 if (checked && !isXml) {
c57f56b7 1014 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
1015 return kFALSE;
1016 }
1017 // Check validity of run number(s)
1018 TObjArray *arr;
1019 TObjString *os;
74013535 1020 TString format;
319593fb 1021 Int_t nruns = 0;
904f9f5f 1022 TString schunk, schunk2;
c57f56b7 1023 TString path;
1024 if (!checked) {
1025 checked = kTRUE;
a2f5fc01 1026 useTags = fDataPattern.Contains("tag");
1027 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1028 }
a2f5fc01 1029 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 1030 Error("CheckInputData", "Cannot mix input files using/not using tags");
1031 return kFALSE;
1032 }
1033 if (fRunNumbers.Length()) {
a8739e8a 1034 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 1035 arr = fRunNumbers.Tokenize(" ");
1036 TIter next(arr);
1037 while ((os=(TObjString*)next())) {
1038 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1039 if (!DirectoryExists(path)) {
1040 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 1041 continue;
c57f56b7 1042 }
1043 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
1044 TString msg = "\n##### file: ";
1045 msg += path;
1046 msg += " type: xml_collection;";
a2f5fc01 1047 if (useTags) msg += " using_tags: Yes";
c57f56b7 1048 else msg += " using_tags: No";
23329835 1049 Info("CheckDataType", "%s", msg.Data());
319593fb 1050 if (fNrunsPerMaster<2) {
d2a409b2 1051 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 1052 } else {
1053 nruns++;
1054 if (((nruns-1)%fNrunsPerMaster) == 0) {
1055 schunk = os->GetString();
1056 }
1057 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1058 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1059 AddDataFile(schunk);
319593fb 1060 }
c57f56b7 1061 }
1062 delete arr;
a8739e8a 1063 } else {
1064 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1065 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1066 format = Form("%%s/%s ", fRunPrefix.Data());
1067 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1068 if (!DirectoryExists(path)) {
a8739e8a 1069 continue;
1070 }
74013535 1071 format = Form("%%s/%s.xml", fRunPrefix.Data());
1072 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1073 TString msg = "\n##### file: ";
1074 msg += path;
1075 msg += " type: xml_collection;";
a2f5fc01 1076 if (useTags) msg += " using_tags: Yes";
a8739e8a 1077 else msg += " using_tags: No";
23329835 1078 Info("CheckDataType", "%s", msg.Data());
319593fb 1079 if (fNrunsPerMaster<2) {
74013535 1080 format = Form("%s.xml", fRunPrefix.Data());
1081 AddDataFile(Form(format.Data(),irun));
319593fb 1082 } else {
1083 nruns++;
1084 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1085 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1086 }
74013535 1087 format = Form("_%s.xml", fRunPrefix.Data());
1088 schunk2 = Form(format.Data(), irun);
319593fb 1089 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1090 schunk += schunk2;
d2a409b2 1091 AddDataFile(schunk);
319593fb 1092 }
a8739e8a 1093 }
904f9f5f 1094 if (!fInputFiles) {
1095 schunk += schunk2;
1096 AddDataFile(schunk);
1097 }
c57f56b7 1098 }
1099 return kTRUE;
1100}
1101
1102//______________________________________________________________________________
4a8b667c 1103Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1104{
1105// Copy data from the given grid directory according a pattern and make a local
1106// dataset.
2ba4c287 1107// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1108 if (!Connect()) {
1109 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
4a8b667c 1110 return 0;
21ca8e59 1111 }
1112 if (!DirectoryExists(griddir)) {
1113 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
4a8b667c 1114 return 0;
21ca8e59 1115 }
1116 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1117 printf("Running command: %s\n", command.Data());
1118 TGridResult *res = gGrid->Command(command);
1119 Int_t nfound = res->GetEntries();
1120 if (!nfound) {
1121 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
4a8b667c 1122 return 0;
21ca8e59 1123 }
1124 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1125
1126 // archives
1127 TObjArray* additionalArchives = 0;
1128 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1129 additionalArchives = TString(archivefile).Tokenize(";");
1130 archivefile = additionalArchives->At(0)->GetName();
1131 additionalArchives->RemoveAt(0);
1132 additionalArchives->Compress();
1133 }
1134
21ca8e59 1135 // Copy files locally
1136 ofstream out;
1137 out.open(output, ios::out);
1138 TMap *map;
1139 TString turl, dirname, filename, temp;
1140 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1141 gSystem->MakeDirectory(outputdir);
1142 gSystem->ChangeDirectory(outputdir);
4a8b667c 1143 Int_t ncopied = 0;
21ca8e59 1144 for (Int_t i=0; i<nfound; i++) {
1145 map = (TMap*)res->At(i);
1146 turl = map->GetValue("turl")->GetName();
1147 filename = gSystem->BaseName(turl.Data());
1148 dirname = gSystem->DirName(turl.Data());
1149 dirname = gSystem->BaseName(dirname.Data());
1150 gSystem->MakeDirectory(dirname);
43b807f6 1151
1152 TString source(turl);
1153 TString targetFileName(filename);
1154
1155 if (strlen(archivefile) > 0) {
4a8b667c 1156// TODO here the archive in which the file resides should be determined
1157// however whereis returns only a guid, and guid2lfn does not work
1158// Therefore we use the one provided as argument for now
1159 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1160 targetFileName = archivefile;
43b807f6 1161 }
1162 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
4a8b667c 1163 Bool_t success = kTRUE;
1164 if (additionalArchives) {
1165 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1166 TString target;
1167 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1168 gSystem->MakeDirectory(gSystem->DirName(target));
1169 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1170 }
1171 }
2ba4c287 1172
4a8b667c 1173 if (success) {
1174 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1175 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1176 ncopied++;
1177 }
21ca8e59 1178 }
1179 }
1180 gSystem->ChangeDirectory(cdir);
1181 delete res;
4a8b667c 1182 delete additionalArchives;
1183 return ncopied;
21ca8e59 1184}
1185
1186//______________________________________________________________________________
c57f56b7 1187Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1188{
1189// Create dataset for the grid data directory + run number.
3af861ff 1190 const Int_t gMaxEntries = 15000;
d3b18c4c 1191 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1192 if (!Connect()) {
1193 Error("CreateDataset", "Cannot create dataset with no grid connection");
1194 return kFALSE;
1195 }
1196
1197 // Cd workspace
d3b18c4c 1198 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1199 TString workdir = gGrid->GetHomeDirectory();
1200 workdir += fGridWorkingDir;
1201
1202 // Compose the 'find' command arguments
74013535 1203 TString format;
c57f56b7 1204 TString command;
1205 TString options = "-x collection ";
bb885a9e 1206 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1207 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1208 TString conditions = "";
3af861ff 1209 Int_t nstart = 0;
1210 Int_t ncount = 0;
1211 Int_t stage = 0;
c57f56b7 1212 TString file;
1213 TString path;
319593fb 1214 Int_t nruns = 0;
904f9f5f 1215 TString schunk, schunk2;
ab254fd1 1216 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1217 if (!fRunNumbers.Length() && !fRunRange[0]) {
1218 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1219 // Make a single data collection from data directory.
1220 path = fGridDataDir;
923e2ca5 1221 if (!DirectoryExists(path)) {
d2a409b2 1222 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1223 return kFALSE;
1224 }
923e2ca5 1225// CdWork();
d2a409b2 1226 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1227 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1228 while (1) {
1229 ncount = 0;
1230 stage++;
1231 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1232 command = "find ";
1233 command += Form("%s -o %d ",options.Data(), nstart);
1234 command += path;
1235 command += " ";
1236 command += pattern;
1237 command += conditions;
1238 printf("command: %s\n", command.Data());
1239 TGridResult *res = gGrid->Command(command);
1240 if (res) delete res;
1241 // Write standard output to file
1242 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1243 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1244 Bool_t nullFile = kFALSE;
1245 if (!hasGrep) {
1246 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1247 } else {
1248 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1249 if (nullFile) {
1250 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1251 gSystem->Exec("rm -f __tmp*");
1252 return kFALSE;
1253 }
1254 TString line;
1255 ifstream in;
1256 in.open("__tmp__");
1257 in >> line;
1258 in.close();
1259 gSystem->Exec("rm -f __tmp__");
1260 ncount = line.Atoi();
1261 }
1262 }
1263 if (ncount == gMaxEntries) {
1264 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1265 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1266 if (!cbase) cbase = cadd;
1267 else {
1268 cbase->Add(cadd);
1269 delete cadd;
defd7a3a 1270 }
3af861ff 1271 nstart += ncount;
1272 } else {
1273 if (cbase) {
1274 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1275 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1276 cbase->Add(cadd);
1277 delete cadd;
1278 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1279 delete cbase; cbase = 0;
1280 } else {
1281 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1282 }
1283 gSystem->Exec("rm -f __tmp*");
1284 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1285 break;
1286 }
0f389141 1287 }
1288 Bool_t fileExists = FileExists(file);
1289 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1290 // Copy xml file to alien space
0f389141 1291 if (fileExists) gGrid->Rm(file);
d2a409b2 1292 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1293 if (!FileExists(file)) {
1294 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1295 return kFALSE;
1296 }
1297 // Update list of files to be processed.
1298 }
1299 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1300 return kTRUE;
1301 }
c57f56b7 1302 // Several runs
a2f5fc01 1303 Bool_t nullResult = kTRUE;
a8739e8a 1304 if (fRunNumbers.Length()) {
1305 TObjArray *arr = fRunNumbers.Tokenize(" ");
1306 TObjString *os;
1307 TIter next(arr);
1308 while ((os=(TObjString*)next())) {
3af861ff 1309 nstart = 0;
1310 stage = 0;
dc030a24 1311 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1312 if (!DirectoryExists(path)) continue;
1313// CdWork();
a8739e8a 1314 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1315 else file = Form("%s.xml", os->GetString().Data());
319593fb 1316 // If local collection file does not exist, create it via 'find' command.
3af861ff 1317 while (1) {
1318 ncount = 0;
1319 stage++;
1320 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1321 command = "find ";
1322 command += Form("%s -o %d ",options.Data(), nstart);
1323 command += path;
1324 command += pattern;
1325 command += conditions;
1326 TGridResult *res = gGrid->Command(command);
1327 if (res) delete res;
1328 // Write standard output to file
1329 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1330 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1331 Bool_t nullFile = kFALSE;
1332 if (!hasGrep) {
1333 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1334 } else {
1335 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1336 if (nullFile) {
1337 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1338 gSystem->Exec("rm -f __tmp*");
1339 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1340 break;
3af861ff 1341 }
1342 TString line;
1343 ifstream in;
1344 in.open("__tmp__");
1345 in >> line;
1346 in.close();
1347 gSystem->Exec("rm -f __tmp__");
1348 ncount = line.Atoi();
1349 }
1350 nullResult = kFALSE;
1351 }
1352 if (ncount == gMaxEntries) {
1353 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1354 if (fNrunsPerMaster > 1) {
1355 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1356 file.Data(),gMaxEntries);
1357 return kFALSE;
1358 }
1359 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1360 if (!cbase) cbase = cadd;
1361 else {
1362 cbase->Add(cadd);
1363 delete cadd;
defd7a3a 1364 }
3af861ff 1365 nstart += ncount;
1366 } else {
c89a84bc 1367 if (cbase && fNrunsPerMaster<2) {
3af861ff 1368 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1369 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1370 cbase->Add(cadd);
1371 delete cadd;
1372 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1373 delete cbase; cbase = 0;
1374 } else {
1375 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1376 }
1377 gSystem->Exec("rm -f __tmp*");
1378 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1379 break;
d3339be3 1380 }
3af861ff 1381 }
a8739e8a 1382 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1383 // Check if there is one run per master job.
1384 if (fNrunsPerMaster<2) {
1385 if (FileExists(file)) {
0f389141 1386 if (fOverwriteMode) gGrid->Rm(file);
1387 else {
1388 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1389 continue;
1390 }
319593fb 1391 }
1392 // Copy xml file to alien space
1393 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1394 if (!FileExists(file)) {
1395 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1396 delete arr;
1397 return kFALSE;
1398 }
1399 } else {
1400 nruns++;
1401 if (((nruns-1)%fNrunsPerMaster) == 0) {
1402 schunk = os->GetString();
1403 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1404 } else {
1405 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1406 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1407 cbase->Add(cadd);
1408 delete cadd;
1409 }
1410 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1411 continue;
1412 }
1413 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1414 if (FileExists(schunk)) {
1415 if (fOverwriteMode) gGrid->Rm(file);
1416 else {
1417 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1418 continue;
1419 }
319593fb 1420 }
84fcd93f 1421 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1422 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1423 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1424 if (!FileExists(schunk)) {
1425 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1426 delete arr;
1427 return kFALSE;
1428 }
d3339be3 1429 }
a8739e8a 1430 }
1431 delete arr;
a2f5fc01 1432 if (nullResult) {
d3339be3 1433 Error("CreateDataset", "No valid dataset corresponding to the query!");
1434 return kFALSE;
1435 }
a8739e8a 1436 } else {
1437 // Process a full run range.
1438 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1439 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1440 nstart = 0;
1441 stage = 0;
74013535 1442 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1443 if (!DirectoryExists(path)) continue;
1444// CdWork();
74013535 1445 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1446 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1447 else file = Form(format.Data(), irun);
0f389141 1448 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1449 if (fOverwriteMode) gGrid->Rm(file);
1450 else {
1451 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1452 continue;
1453 }
a8739e8a 1454 }
319593fb 1455 // If local collection file does not exist, create it via 'find' command.
3af861ff 1456 while (1) {
1457 ncount = 0;
1458 stage++;
1459 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1460 command = "find ";
1461 command += Form("%s -o %d ",options.Data(), nstart);
1462 command += path;
1463 command += pattern;
1464 command += conditions;
1465 TGridResult *res = gGrid->Command(command);
1466 if (res) delete res;
1467 // Write standard output to file
1468 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1469 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1470 Bool_t nullFile = kFALSE;
1471 if (!hasGrep) {
1472 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1473 } else {
1474 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1475 if (nullFile) {
1476 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1477 gSystem->Exec("rm -f __tmp*");
1604c9ed 1478 break;
3af861ff 1479 }
1480 TString line;
1481 ifstream in;
1482 in.open("__tmp__");
1483 in >> line;
1484 in.close();
1485 gSystem->Exec("rm -f __tmp__");
1486 ncount = line.Atoi();
1487 }
1488 nullResult = kFALSE;
1489 }
1490 if (ncount == gMaxEntries) {
1491 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1492 if (fNrunsPerMaster > 1) {
1493 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1494 file.Data(),gMaxEntries);
1495 return kFALSE;
1496 }
1497 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1498 if (!cbase) cbase = cadd;
1499 else {
1500 cbase->Add(cadd);
1501 delete cadd;
defd7a3a 1502 }
3af861ff 1503 nstart += ncount;
1504 } else {
c89a84bc 1505 if (cbase && fNrunsPerMaster<2) {
3af861ff 1506 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1507 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1508 cbase->Add(cadd);
1509 delete cadd;
1510 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1511 delete cbase; cbase = 0;
1512 } else {
1513 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1514 }
1515 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1516 break;
d3339be3 1517 }
319593fb 1518 }
a8739e8a 1519 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1520 // Check if there is one run per master job.
1521 if (fNrunsPerMaster<2) {
1522 if (FileExists(file)) {
0f389141 1523 if (fOverwriteMode) gGrid->Rm(file);
1524 else {
1525 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1526 continue;
1527 }
319593fb 1528 }
1529 // Copy xml file to alien space
1530 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1531 if (!FileExists(file)) {
1532 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1533 return kFALSE;
1534 }
1535 } else {
1536 nruns++;
95e5b448 1537 // Check if the collection for the chunk exist locally.
1538 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1539 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1540 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1541 else continue;
1542 }
84fcd93f 1543 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1544 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1545 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1546 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1547 } else {
1548 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1549 cbase->Add(cadd);
1550 delete cadd;
1551 }
74013535 1552 format = Form("%%s_%s.xml", fRunPrefix.Data());
1553 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1554 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1555 continue;
1556 }
904f9f5f 1557 schunk = schunk2;
319593fb 1558 if (FileExists(schunk)) {
0f389141 1559 if (fOverwriteMode) gGrid->Rm(schunk);
1560 else {
1561 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1562 continue;
1563 }
319593fb 1564 }
84fcd93f 1565 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1566 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1567 if (FileExists(schunk)) {
0f389141 1568 if (fOverwriteMode) gGrid->Rm(schunk);
1569 else {
1570 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1571 continue;
1572 }
95e5b448 1573 }
319593fb 1574 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1575 if (!FileExists(schunk)) {
1576 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1577 return kFALSE;
1578 }
1579 }
c57f56b7 1580 }
a2f5fc01 1581 if (nullResult) {
d3339be3 1582 Error("CreateDataset", "No valid dataset corresponding to the query!");
1583 return kFALSE;
1584 }
a8739e8a 1585 }
c57f56b7 1586 return kTRUE;
1587}
1588
1589//______________________________________________________________________________
1590Bool_t AliAnalysisAlien::CreateJDL()
1591{
1592// Generate a JDL file according to current settings. The name of the file is
1593// specified by fJDLName.
ec5acdce 1594 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
c57f56b7 1595 Bool_t error = kFALSE;
1596 TObjArray *arr = 0;
1597 Bool_t copy = kTRUE;
d3b18c4c 1598 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1599 Bool_t generate = kTRUE;
1600 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1601 if (!Connect()) {
1602 Error("CreateJDL", "Alien connection required");
1603 return kFALSE;
1604 }
1605 // Check validity of alien workspace
d3b18c4c 1606 TString workdir;
1607 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1608 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1609 workdir += fGridWorkingDir;
1610 if (generate) {
1611 TObjString *os;
1612 if (!fInputFiles) {
1613 Error("CreateJDL()", "Define some input files for your analysis.");
1614 error = kTRUE;
1615 }
1616 // Compose list of input files
1617 // Check if output files were defined
1618 if (!fOutputFiles.Length()) {
1619 Error("CreateJDL", "You must define at least one output file");
1620 error = kTRUE;
1621 }
1622 // Check if an output directory was defined and valid
1623 if (!fGridOutputDir.Length()) {
1624 Error("CreateJDL", "You must define AliEn output directory");
1625 error = kTRUE;
1626 } else {
d3b18c4c 1627 if (!fProductionMode) {
1628 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1629 if (!DirectoryExists(fGridOutputDir)) {
1630 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1631 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1632 } else {
1633 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1634 // error = kTRUE;
1635 }
67a3b254 1636 } else {
1637 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1638 }
1639 gGrid->Cd(workdir);
1640 }
c57f56b7 1641 }
1642 // Exit if any error up to now
1643 if (error) return kFALSE;
1644 // Set JDL fields
0f389141 1645 if (!fUser.IsNull()) {
1646 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1647 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1648 }
1649 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1650 TString mergeExec = fExecutable;
1651 mergeExec.ReplaceAll(".sh", "_merge.sh");
1652 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1653 mergeExec.ReplaceAll(".sh", ".C");
1654 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1655 if (!fArguments.IsNull())
1656 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1657 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1658 else {
1659 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1660 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1661 }
b3e07543 1662
1f0d1ca2 1663 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1664 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1665 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1666 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1667
0f389141 1668 if (fMaxInitFailed > 0) {
c57f56b7 1669 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1670 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1671 }
1672 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1673 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1674 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1675 }
1676 if (!IsOneStageMerging()) {
f47d5cb4 1677 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1678 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1679 }
1680 if (fSplitMode.Length()) {
c57f56b7 1681 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1682 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1683 }
1684 fMergingJDL->SetValue("Split", "\"se\"");
1685 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1686 if (!fAliROOTVersion.IsNull()) {
1687 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1688 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1689 }
1690 if (!fROOTVersion.IsNull()) {
c57f56b7 1691 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1692 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1693 }
1694 if (!fAPIVersion.IsNull()) {
c57f56b7 1695 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1696 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1697 }
648174cf 1698 if (!fExternalPackages.IsNull()) {
1699 arr = fExternalPackages.Tokenize(" ");
1700 TIter next(arr);
1701 while ((os=(TObjString*)next())) {
1702 TString pkgname = os->GetString();
1703 Int_t index = pkgname.Index("::");
1704 TString pkgversion = pkgname(index+2, pkgname.Length());
1705 pkgname.Remove(index);
1706 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1707 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1708 }
1709 delete arr;
1710 }
0f389141 1711 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1712 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1713 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1714 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1715 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1716 TString analysisFile = fExecutable;
1717 analysisFile.ReplaceAll(".sh", ".root");
1718 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1719 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1720 if (fAdditionalLibs.Length()) {
1721 arr = fAdditionalLibs.Tokenize(" ");
1722 TIter next(arr);
1723 while ((os=(TObjString*)next())) {
1724 if (os->GetString().Contains(".so")) continue;
1725 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1726 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1727 }
1728 delete arr;
1729 }
4e5c5506 1730 if (fPackages) {
1731 TIter next(fPackages);
1732 TObject *obj;
0f389141 1733 while ((obj=next())) {
4e5c5506 1734 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1735 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1736 }
4e5c5506 1737 }
95e16f01 1738 const char *comment = "List of output files and archives";
c57f56b7 1739 if (fOutputArchive.Length()) {
e4d9a069 1740 TString outputArchive = fOutputArchive;
1741 if (!fRegisterExcludes.IsNull()) {
1742 arr = fRegisterExcludes.Tokenize(" ");
1743 TIter next1(arr);
1744 while ((os=(TObjString*)next1())) {
1745 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1746 outputArchive.ReplaceAll(os->GetString(),"");
1747 }
1748 delete arr;
1749 }
1750 arr = outputArchive.Tokenize(" ");
c57f56b7 1751 TIter next(arr);
0f389141 1752 Bool_t first = kTRUE;
0f389141 1753 while ((os=(TObjString*)next())) {
0f389141 1754 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1755 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
0f389141 1756 else
95e16f01 1757 fGridJDL->AddToSet("Output", os->GetString());
1758 if (first) fGridJDL->AddToSetDescription("Output", comment);
0f389141 1759 first = kFALSE;
1760 }
c57f56b7 1761 delete arr;
f790bc1b 1762 // Output archive for the merging jdl
f790bc1b 1763 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1764 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1765 // Add normal output files, extra files + terminate files
1766 TString files = GetListOfFiles("outextter");
37d21c01 1767 // Do not register files in fRegisterExcludes
1768 if (!fRegisterExcludes.IsNull()) {
1769 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1770 TIter next1(arr);
1771 while ((os=(TObjString*)next1())) {
1772 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1773 files.ReplaceAll(os->GetString(),"");
1774 }
66d71516 1775 delete arr;
0f389141 1776 }
f790bc1b 1777 files.ReplaceAll(".root", "*.root");
ec5acdce 1778
1779 if (mgr->IsCollectThroughput())
95e16f01 1780 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",files.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 1781 else
1782 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1783 } else {
95f84779 1784 TString files = fOutputArchive;
1785 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1786 outputArchive = files;
f790bc1b 1787 }
0f389141 1788 arr = outputArchive.Tokenize(" ");
705adb3e 1789 TIter next2(arr);
0f389141 1790 first = kTRUE;
705adb3e 1791 while ((os=(TObjString*)next2())) {
0cdf65a8 1792 TString currentfile = os->GetString();
0cdf65a8 1793 if (!currentfile.Contains("@") && fCloseSE.Length())
95e16f01 1794 fMergingJDL->AddToSet("Output", Form("%s@%s",currentfile.Data(), fCloseSE.Data()));
0f389141 1795 else
95e16f01 1796 fMergingJDL->AddToSet("Output", currentfile);
1797 if (first) fMergingJDL->AddToSetDescription("Output", comment);
0f389141 1798 first = kFALSE;
1799 }
1800 delete arr;
c57f56b7 1801 }
149d288c 1802 arr = fOutputFiles.Tokenize(",");
c57f56b7 1803 TIter next(arr);
0f389141 1804 Bool_t first = kTRUE;
43da816a 1805 while ((os=(TObjString*)next())) {
e1eaf596 1806 // Ignore ouputs in jdl that are also in outputarchive
1807 TString sout = os->GetString();
f790bc1b 1808 sout.ReplaceAll("*", "");
1809 sout.ReplaceAll(".root", "");
e1eaf596 1810 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1811 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1812 // Ignore fRegisterExcludes
1813 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1814 if (!first) comment = NULL;
43da816a 1815 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1816 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
43da816a 1817 else
95e16f01 1818 fGridJDL->AddToSet("Output", os->GetString());
1819 if (first) fGridJDL->AddToSetDescription("Output", comment);
f790bc1b 1820 if (fMergeExcludes.Contains(sout)) continue;
1821 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1822 fMergingJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
f790bc1b 1823 else
95e16f01 1824 fMergingJDL->AddToSet("Output", os->GetString());
1825 if (first) fMergingJDL->AddToSetDescription("Output", comment);
1826 first = kFALSE;
43da816a 1827 }
c57f56b7 1828 delete arr;
0f389141 1829 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1830 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1831 TString validationScript = fValidationScript;
0f389141 1832 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1833 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1834 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1835 if (fMasterResubmitThreshold) {
1836 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1837 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1838 }
d2a409b2 1839 // Write a jdl with 2 input parameters: collection name and output dir name.
1840 WriteJDL(copy);
c57f56b7 1841 }
1842 // Copy jdl to grid workspace
a8739e8a 1843 if (copy) {
b5e4aaa7 1844 // Check if an output directory was defined and valid
1845 if (!fGridOutputDir.Length()) {
1846 Error("CreateJDL", "You must define AliEn output directory");
1847 return kFALSE;
1848 } else {
1849 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1850 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1851 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1852 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1853 } else {
1854 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1855 return kFALSE;
1856 }
1857 }
1858 gGrid->Cd(workdir);
1859 }
648174cf 1860 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1861 TString mergeJDLName = fExecutable;
1862 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1863 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1864 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1865 if (fProductionMode) {
648174cf 1866 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1867 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1868 }
648174cf 1869 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1870 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1871 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1872 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1873 Fatal("","Terminating");
1874// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1875 if (fMergeViaJDL) {
1876 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1877// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1878 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1879 Fatal("","Terminating");
0f389141 1880 }
648174cf 1881 }
c57f56b7 1882 if (fAdditionalLibs.Length()) {
e7c71df0 1883 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1884 TObjString *os;
1885 TIter next(arr);
1886 while ((os=(TObjString*)next())) {
c57f56b7 1887 if (os->GetString().Contains(".so")) continue;
4e5c5506 1888 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1889 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1890// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1891 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1892 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1893 Fatal("","Terminating");
c57f56b7 1894 }
1895 delete arr;
1896 }
4e5c5506 1897 if (fPackages) {
1898 TIter next(fPackages);
1899 TObject *obj;
1900 while ((obj=next())) {
fdbbc7be 1901 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1902 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1903// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1904 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1905 Form("%s/%s", workdir.Data(), obj->GetName())))
1906 Fatal("","Terminating");
4e5c5506 1907 }
1908 }
c57f56b7 1909 }
1910 return kTRUE;
1911}
1912
1913//______________________________________________________________________________
d2a409b2 1914Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1915{
1916// Writes one or more JDL's corresponding to findex. If findex is negative,
1917// all run numbers are considered in one go (jdl). For non-negative indices
1918// they correspond to the indices in the array fInputFiles.
1919 if (!fInputFiles) return kFALSE;
d3b18c4c 1920 TObject *os;
1921 TString workdir;
1922 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1923 workdir += fGridWorkingDir;
f47d5cb4 1924 TString stageName = "$2";
1925 if (fProductionMode) stageName = "$4";
1926 if (!fMergeDirName.IsNull()) {
1927 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1928 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1929 } else {
1930 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1931 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1932 }
d3b18c4c 1933 if (fProductionMode) {
a8739e8a 1934 TIter next(fInputFiles);
b3e07543 1935 while ((os=next())) {
d3b18c4c 1936 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1937 }
56754bf6 1938 if (!fOutputToRunNo)
1939 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1940 else
1941 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1942 } else {
1943 if (!fRunNumbers.Length() && !fRunRange[0]) {
1944 // One jdl with no parameters in case input data is specified by name.
1945 TIter next(fInputFiles);
1946 while ((os=next()))
1947 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1948 if (!fOutputSingle.IsNull())
1949 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1950 else {
1951 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1952 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1953 }
1954 } else {
1955 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1956 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1957 if (!fOutputSingle.IsNull()) {
1958 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1959 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1960 } else {
1961 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1962 }
1963 }
a8739e8a 1964 }
1965
a8739e8a 1966 // Generate the JDL as a string
1967 TString sjdl = fGridJDL->Generate();
0f389141 1968 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1969 // Final merge jdl
1970 if (!fMergeDirName.IsNull()) {
1971 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1972 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1973 } else {
1974 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1975 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1976 }
b3e07543 1977 TString sjdl2 = fMergingJDL->Generate();
1978 Int_t index, index1;
95e16f01 1979 sjdl.ReplaceAll("\",\"", "\",\n \"");
a8739e8a 1980 sjdl.ReplaceAll("(member", "\n (member");
1981 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1982 sjdl.ReplaceAll("{", "{\n ");
1983 sjdl.ReplaceAll("};", "\n};");
1984 sjdl.ReplaceAll("{\n \n", "{\n");
1985 sjdl.ReplaceAll("\n\n", "\n");
1986 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 1987 sjdl1.ReplaceAll("\",\"", "\",\n \"");
0f389141 1988 sjdl1.ReplaceAll("(member", "\n (member");
1989 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1990 sjdl1.ReplaceAll("{", "{\n ");
1991 sjdl1.ReplaceAll("};", "\n};");
1992 sjdl1.ReplaceAll("{\n \n", "{\n");
1993 sjdl1.ReplaceAll("\n\n", "\n");
1994 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 1995 sjdl2.ReplaceAll("\",\"", "\",\n \"");
b3e07543 1996 sjdl2.ReplaceAll("(member", "\n (member");
1997 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1998 sjdl2.ReplaceAll("{", "{\n ");
1999 sjdl2.ReplaceAll("};", "\n};");
2000 sjdl2.ReplaceAll("{\n \n", "{\n");
2001 sjdl2.ReplaceAll("\n\n", "\n");
2002 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 2003 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 2004 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 2005 index = sjdl.Index("JDLVariables");
2006 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 2007 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2008 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 2009 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 2010 index = fJobTag.Index(":");
2011 if (index < 0) index = fJobTag.Length();
2012 TString jobTag = fJobTag;
f47d5cb4 2013 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 2014 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 2015 if (fProductionMode) {
2016 sjdl1.Prepend("# Generated merging jdl (production mode) \
2017 \n# $1 = full alien path to output directory to be merged \
2018 \n# $2 = train number \
2019 \n# $3 = production (like LHC10b) \
2020 \n# $4 = merging stage \
2021 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2022 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2023 sjdl2.Prepend("# Generated merging jdl \
2024 \n# $1 = full alien path to output directory to be merged \
2025 \n# $2 = train number \
2026 \n# $3 = production (like LHC10b) \
2027 \n# $4 = merging stage \
2028 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2029 } else {
2030 sjdl1.Prepend("# Generated merging jdl \
2031 \n# $1 = full alien path to output directory to be merged \
2032 \n# $2 = merging stage \
03335324 2033 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2034 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2035 sjdl2.Prepend("# Generated merging jdl \
2036 \n# $1 = full alien path to output directory to be merged \
2037 \n# $2 = merging stage \
03335324 2038 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2039 }
0f389141 2040 index = sjdl1.Index("JDLVariables");
2041 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 2042 index = sjdl2.Index("JDLVariables");
2043 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 2044 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2045 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
2046 index = sjdl2.Index("Split =");
2047 if (index>=0) {
2048 index1 = sjdl2.Index("\n", index);
2049 sjdl2.Remove(index, index1-index+1);
2050 }
2051 index = sjdl2.Index("SplitMaxInputFileNumber");
2052 if (index>=0) {
2053 index1 = sjdl2.Index("\n", index);
2054 sjdl2.Remove(index, index1-index+1);
2055 }
2056 index = sjdl2.Index("InputDataCollection");
2057 if (index>=0) {
2058 index1 = sjdl2.Index(";", index);
2059 sjdl2.Remove(index, index1-index+1);
2060 }
2061 index = sjdl2.Index("InputDataListFormat");
2062 if (index>=0) {
2063 index1 = sjdl2.Index("\n", index);
2064 sjdl2.Remove(index, index1-index+1);
2065 }
2066 index = sjdl2.Index("InputDataList");
2067 if (index>=0) {
2068 index1 = sjdl2.Index("\n", index);
2069 sjdl2.Remove(index, index1-index+1);
2070 }
f47d5cb4 2071 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2072 // Write jdl to file
a8739e8a 2073 ofstream out;
d2a409b2 2074 out.open(fJDLName.Data(), ios::out);
a8739e8a 2075 if (out.bad()) {
d3b18c4c 2076 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2077 return kFALSE;
2078 }
2079 out << sjdl << endl;
b3e07543 2080 out.close();
0f389141 2081 TString mergeJDLName = fExecutable;
2082 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2083 if (fMergeViaJDL) {
2084 ofstream out1;
2085 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2086 if (out1.bad()) {
d3b18c4c 2087 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2088 return kFALSE;
2089 }
2090 out1 << sjdl1 << endl;
b3e07543 2091 out1.close();
2092 ofstream out2;
2093 TString finalJDL = mergeJDLName;
2094 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2095 out2.open(finalJDL.Data(), ios::out);
2096 if (out2.bad()) {
2097 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2098 return kFALSE;
2099 }
2100 out2 << sjdl2 << endl;
2101 out2.close();
0f389141 2102 }
a8739e8a 2103
2104 // Copy jdl to grid workspace
2105 if (!copy) {
d3b18c4c 2106 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2107 } else {
d2a409b2 2108 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2109 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2110 TString finalJDL = mergeJDLName;
2111 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2112 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2113 if (fProductionMode) {
b5fe9cba 2114 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2115 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2116 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2117 }
d2a409b2 2118 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2119 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2120 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2121 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2122// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2123 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2124 Fatal("","Terminating");
0f389141 2125 if (fMergeViaJDL) {
b3e07543 2126 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2127// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2128// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2129 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2130 Fatal("","Terminating");
2131 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2132 Fatal("","Terminating");
0f389141 2133 }
a8739e8a 2134 }
2135 return kTRUE;
2136}
2137
2138//______________________________________________________________________________
5513444a 2139Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2140{
2141// Returns true if file exists.
5513444a 2142 if (!gGrid) return kFALSE;
2ea55496 2143 TString slfn = lfn;
2144 slfn.ReplaceAll("alien://","");
2145 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2146 if (!res) return kFALSE;
2147 TMap *map = dynamic_cast<TMap*>(res->At(0));
2148 if (!map) {
2149 delete res;
2150 return kFALSE;
2151 }
2152 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2153 if (!objs || !objs->GetString().Length()) {
2154 delete res;
2155 return kFALSE;
2156 }
2157 delete res;
2158 return kTRUE;
2159}
2160
2161//______________________________________________________________________________
923e2ca5 2162Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2163{
2164// Returns true if directory exists. Can be also a path.
2165 if (!gGrid) return kFALSE;
2166 // Check if dirname is a path
2167 TString dirstripped = dirname;
2168 dirstripped = dirstripped.Strip();
2169 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2170 TString dir = gSystem->BaseName(dirstripped);
2171 dir += "/";
2172 TString path = gSystem->DirName(dirstripped);
2173 TGridResult *res = gGrid->Ls(path, "-F");
2174 if (!res) return kFALSE;
2175 TIter next(res);
2176 TMap *map;
2177 TObject *obj;
2178 while ((map=dynamic_cast<TMap*>(next()))) {
2179 obj = map->GetValue("name");
2180 if (!obj) break;
2181 if (dir == obj->GetName()) {
2182 delete res;
2183 return kTRUE;
2184 }
2185 }
2186 delete res;
2187 return kFALSE;
2188}
2189
2190//______________________________________________________________________________
a2f5fc01 2191void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2192{
2193// Check input data type.
a2f5fc01 2194 isCollection = kFALSE;
2195 isXml = kFALSE;
2196 useTags = kFALSE;
c57f56b7 2197 if (!gGrid) {
2198 Error("CheckDataType", "No connection to grid");
2199 return;
2200 }
a2f5fc01 2201 isCollection = IsCollection(lfn);
c57f56b7 2202 TString msg = "\n##### file: ";
2203 msg += lfn;
a2f5fc01 2204 if (isCollection) {
c57f56b7 2205 msg += " type: raw_collection;";
2206 // special treatment for collections
a2f5fc01 2207 isXml = kFALSE;
c57f56b7 2208 // check for tag files in the collection
2209 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2210 if (!res) {
2211 msg += " using_tags: No (unknown)";
23329835 2212 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2213 return;
2214 }
2215 const char* typeStr = res->GetKey(0, "origLFN");
2216 if (!typeStr || !strlen(typeStr)) {
2217 msg += " using_tags: No (unknown)";
23329835 2218 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2219 return;
2220 }
2221 TString file = typeStr;
a2f5fc01 2222 useTags = file.Contains(".tag");
2223 if (useTags) msg += " using_tags: Yes";
c57f56b7 2224 else msg += " using_tags: No";
23329835 2225 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2226 return;
2227 }
2228 TString slfn(lfn);
2229 slfn.ToLower();
a2f5fc01 2230 isXml = slfn.Contains(".xml");
2231 if (isXml) {
c57f56b7 2232 // Open xml collection and check if there are tag files inside
2233 msg += " type: xml_collection;";
2234 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2235 if (!coll) {
2236 msg += " using_tags: No (unknown)";
23329835 2237 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2238 return;
2239 }
2240 TMap *map = coll->Next();
2241 if (!map) {
2242 msg += " using_tags: No (unknown)";
23329835 2243 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2244 return;
2245 }
2246 map = (TMap*)map->GetValue("");
2247 TString file;
2248 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2249 useTags = file.Contains(".tag");
c57f56b7 2250 delete coll;
a2f5fc01 2251 if (useTags) msg += " using_tags: Yes";
c57f56b7 2252 else msg += " using_tags: No";
23329835 2253 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2254 return;
2255 }
a2f5fc01 2256 useTags = slfn.Contains(".tag");
c57f56b7 2257 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2258 else msg += " type: unknown file;";
a2f5fc01 2259 if (useTags) msg += " using_tags: Yes";
c57f56b7 2260 else msg += " using_tags: No";
23329835 2261 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2262}
2263
2264//______________________________________________________________________________
4e5c5506 2265void AliAnalysisAlien::EnablePackage(const char *package)
2266{
2267// Enables a par file supposed to exist in the current directory.
2268 TString pkg(package);
2269 pkg.ReplaceAll(".par", "");
2270 pkg += ".par";
2271 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2272 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2273 return;
2274 }
2275 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2276 Info("EnablePackage", "AliEn plugin will use .par packages");
2277 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2278 if (!fPackages) {
2279 fPackages = new TObjArray();
2280 fPackages->SetOwner();
2281 }
2282 fPackages->Add(new TObjString(pkg));
2283}
2284
2285//______________________________________________________________________________
3bdcb562 2286TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2287{
2288// Make a tree from files having the location specified in fFileForTestMode.
2289// Inspired from JF's CreateESDChain.
2290 if (fFileForTestMode.IsNull()) {
2291 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2292 return NULL;
2293 }
2294 if (gSystem->AccessPathName(fFileForTestMode)) {
2295 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2296 return NULL;
2297 }
2298 // Open the file
2299 ifstream in;
2300 in.open(fFileForTestMode);
2301 Int_t count = 0;
2302 // Read the input list of files and add them to the chain
b505f1f8 2303 TString line;
d6c60197 2304 TString streeName(treeName);
2305 if (IsUseMCchain()) streeName = "TE";
2306 TChain *chain = new TChain(streeName);
b505f1f8 2307 TChain *chainFriend = 0;
d6c60197 2308 if (!fFriendChainName.IsNull()) chainFriend = new TChain(streeName);
b505f1f8 2309 while (in.good())
2310 {
3bdcb562 2311 in >> line;
ffbe06b9 2312 if (line.IsNull() || line.BeginsWith("#")) continue;
3bdcb562 2313 if (count++ == fNtestFiles) break;
2314 TString esdFile(line);
2315 TFile *file = TFile::Open(esdFile);
b505f1f8 2316 if (file && !file->IsZombie()) {
2317 chain->Add(esdFile);
3bdcb562 2318 file->Close();
b505f1f8 2319 if (!fFriendChainName.IsNull()) {
4b5d9e21 2320 if (esdFile.Index("#") > -1)
2321 esdFile.Remove(esdFile.Index("#"));
2322 esdFile = gSystem->DirName(esdFile);
2323 esdFile += "/" + fFriendChainName;
b505f1f8 2324 file = TFile::Open(esdFile);
2325 if (file && !file->IsZombie()) {
2326 file->Close();
2327 chainFriend->Add(esdFile);
2328 } else {
2329 Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data());
2330 return 0;
2331 }
4b5d9e21 2332 }
3bdcb562 2333 } else {
2334 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2335 }
b505f1f8 2336 }
2337 in.close();
2338 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2339 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2340 delete chain;
b505f1f8 2341 delete chainFriend;
3bdcb562 2342 return NULL;
b505f1f8 2343 }
3bdcb562 2344// chain->ls();
b505f1f8 2345 if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend);
2346 return chain;
3bdcb562 2347}
2348
2349//______________________________________________________________________________
d2a409b2 2350const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2351{
2352// Get job status for all jobs with jobid>jobidstart.
2353 static char mstatus[20];
2354 mstatus[0] = '\0';
2355 nrunning = 0;
2356 nwaiting = 0;
2357 nerror = 0;
2358 ndone = 0;
2359 TGridJobStatusList *list = gGrid->Ps("");
2360 if (!list) return mstatus;
2361 Int_t nentries = list->GetSize();
2362 TGridJobStatus *status;
2363 Int_t pid;
2364 for (Int_t ijob=0; ijob<nentries; ijob++) {
2365 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2366 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2367 if (pid<jobidstart) continue;
2368 if (pid == lastid) {
8ddad121 2369 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2370 }
2371 switch (status->GetStatus()) {
2372 case TGridJobStatus::kWAITING:
2373 nwaiting++; break;
2374 case TGridJobStatus::kRUNNING:
2375 nrunning++; break;
2376 case TGridJobStatus::kABORTED:
2377 case TGridJobStatus::kFAIL:
2378 case TGridJobStatus::kUNKNOWN:
2379 nerror++; break;
2380 case TGridJobStatus::kDONE:
2381 ndone++;
2382 }
2383 }
2384 list->Delete();
2385 delete list;
2386 return mstatus;
2387}
2388
2389//______________________________________________________________________________
c57f56b7 2390Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2391{
2392// Returns true if file is a collection. Functionality duplicated from
2393// TAlien::Type() because we don't want to directly depend on TAlien.
2394 if (!gGrid) {
2395 Error("IsCollection", "No connection to grid");
2396 return kFALSE;
2397 }
2398 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2399 if (!res) return kFALSE;
2400 const char* typeStr = res->GetKey(0, "type");
2401 if (!typeStr || !strlen(typeStr)) return kFALSE;
2402 if (!strcmp(typeStr, "collection")) return kTRUE;
2403 delete res;
2404 return kFALSE;
2405}
2406
2407//______________________________________________________________________________
fe2d7fc2 2408Bool_t AliAnalysisAlien::IsSingleOutput() const
2409{
2410// Check if single-ouput option is on.
2411 return (!fOutputSingle.IsNull());
2412}
2413
2414//______________________________________________________________________________
16a4353c 2415void AliAnalysisAlien::Print(Option_t *) const
2416{
2417// Print current plugin settings.
84fcd93f 2418 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2419 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2420 if (mgr && mgr->IsProofMode()) {
2421 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2422 if (TestBit(AliAnalysisGrid::kTest))
2423 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2424 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2425 if (!fProofDataSet.IsNull())
2426 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2427 if (fProofReset==1)
2428 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2429 if (fProofReset>1)
2430 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
874de7ea 2431 if (!fROOTVersion.IsNull())
2432 printf("= ROOT version requested________________________ %s\n", fROOTVersion.Data());
d3b18c4c 2433 else
2434 printf("= ROOT version requested________________________ default\n");
2435 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2436 if (!fAliRootMode.IsNull())
2437 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2438 if (fNproofWorkers)
2439 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2440 if (fNproofWorkersPerSlave)
2441 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2442 if (TestSpecialBit(kClearPackages))
2443 printf("= ClearPackages requested...\n");
2444 if (fIncludePath.Data())
2445 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2446 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2447 if (fPackages && fPackages->GetEntries()) {
2448 TIter next(fPackages);
2449 TObject *obj;
2450 TString list;
2451 while ((obj=next())) list += obj->GetName();
2452 printf("= Par files to be used: ________________________ %s\n", list.Data());
2453 }
2454 if (TestSpecialBit(kProofConnectGrid))
2455 printf("= Requested PROOF connection to grid\n");
2456 return;
2457 }
e1c22e21 2458 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2459 if (fOverwriteMode) {
2460 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2461 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2462 }
348be253 2463 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2464 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2465 printf("= Production mode:______________________________ %d\n", fProductionMode);
2466 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2467 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2468 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2469 if (fUser.Length())
84fcd93f 2470 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2471 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2472 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2473 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2474 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2475 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2476 if (fRunNumbers.Length())
84fcd93f 2477 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2478 if (fRunRange[0])
74013535 2479 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2480 if (!fRunRange[0] && !fRunNumbers.Length()) {
2481 TIter next(fInputFiles);
2482 TObject *obj;
2483 TString list;
2484 while ((obj=next())) list += obj->GetName();
84fcd93f 2485 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2486 }
2487 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2488 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2489 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2490 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2491 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2492 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2493 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2494 printf("=====================================================================\n");
2495 printf("= Job price: ___________________________________ %d\n", fPrice);
2496 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2497 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2498 if (fMaxInitFailed>0)
84fcd93f 2499 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2500 if (fMasterResubmitThreshold>0)
84fcd93f 2501 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2502 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2503 if (fNrunsPerMaster>0)
84fcd93f 2504 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2505 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2506 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2507 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2508 if (fArguments.Length())
84fcd93f 2509 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2510 if (fExecutableArgs.Length())
2511 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2512 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2513 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2514 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2515 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2516 if (fDatasetName)
84fcd93f 2517 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2518 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2519 if (fIncludePath.Data())
84fcd93f 2520 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2521 if (fCloseSE.Length())
84fcd93f 2522 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2523 if (fFriendChainName.Length())
84fcd93f 2524 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2525 if (fPackages && fPackages->GetEntries()) {
16a4353c 2526 TIter next(fPackages);
2527 TObject *obj;
2528 TString list;
2529 while ((obj=next())) list += obj->GetName();
84fcd93f 2530 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2531 }
2532}
2533
2534//______________________________________________________________________________
c57f56b7 2535void AliAnalysisAlien::SetDefaults()
2536{
2537// Set default values for everything. What cannot be filled will be left empty.
2538 if (fGridJDL) delete fGridJDL;
2539 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2540 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2541 fPrice = 1;
2542 fTTL = 30000;
2543 fSplitMaxInputFileNumber = 100;
2544 fMaxInitFailed = 0;
2545 fMasterResubmitThreshold = 0;
bb885a9e 2546 fNtestFiles = 10;
149d288c 2547 fNreplicas = 2;
a8739e8a 2548 fRunRange[0] = 0;
2549 fRunRange[1] = 0;
8e3bd06a 2550 fRunPrefix = "%d";
319593fb 2551 fNrunsPerMaster = 1;
16a4353c 2552 fMaxMergeFiles = 100;
c57f56b7 2553 fRunNumbers = "";
2554 fExecutable = "analysis.sh";
5aa94dc3 2555 fExecutableCommand = "root -b -q -x";
c57f56b7 2556 fArguments = "";
631c0b05 2557 fExecutableArgs = "";
c57f56b7 2558 fAnalysisMacro = "myAnalysis.C";
2559 fAnalysisSource = "";
2560 fAdditionalLibs = "";
2561 fSplitMode = "se";
2562 fAPIVersion = "";
2563 fROOTVersion = "";
2564 fAliROOTVersion = "";
2565 fUser = ""; // Your alien user name
2566 fGridWorkingDir = "";
2567 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2568 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2569 fFriendChainName = "";
c57f56b7 2570 fGridOutputDir = "output";
b83f84f0 2571 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2572 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2573 fInputFormat = "xml-single";
2574 fJDLName = "analysis.jdl";
c6cb3634 2575 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2576 fMergeExcludes = "";
0f389141 2577 fMergeViaJDL = 0;
348be253 2578 SetUseCopy(kTRUE);
2579 SetCheckCopy(kTRUE);
149d288c 2580 SetDefaultOutputs(kTRUE);
e1c22e21 2581 fOverwriteMode = 1;
c57f56b7 2582}
2583
2584//______________________________________________________________________________
a41b2882 2585void AliAnalysisAlien::SetFriendChainName(const char *name, const char *libnames)
2586{
2587 // Set file name for the chain of friends and optionally additional libs to be loaded.
2588 // Libs should be separated by blancs.
2589 fFriendChainName = name;
2590 fFriendLibs = libnames;
2591 if (!fFriendLibs.Contains(".so")) {
2592 Fatal("SetFriendChainName()", "You should provide explicit library names (with extension)");
2593 }
2594}
2595
2596//______________________________________________________________________________
874de7ea 2597void AliAnalysisAlien::SetRootVersionForProof(const char *version)
2598{
2599// Obsolete method. Use SetROOTVersion instead
2600 Warning("SetRootVersionForProof", "Obsolete. Use SetROOTVersion instead");
95e16f01 2601 if (fROOTVersion.IsNull()) SetROOTVersion(version);
2602 else Error("SetRootVersionForProof", "ROOT version already set to %s", fROOTVersion.Data());
874de7ea 2603}
2604
2605//______________________________________________________________________________
b3e07543 2606Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2607{
b3e07543 2608// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2609 // First check if the result is already in the output directory.
2610 if (FileExists(Form("%s/%s",aliendir,filename))) {
2611 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2612 return kFALSE;
2613 }
b3e07543 2614 // Now check the last stage done.
2615 Int_t stage = 0;
2616 while (1) {
2617 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2618 stage++;
2619 }
2620 // Next stage of merging
2621 stage++;
2622 TString pattern = "*root_archive.zip";
2623 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2624 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2625 if (res) delete res;
2626 // Write standard output to file
2627 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2628 // Count the number of files inside
2629 ifstream ifile;
2630 ifile.open(Form("Stage_%d.xml",stage));
2631 if (!ifile.good()) {
2632 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2633 return kFALSE;
b3e07543 2634 }
2635 TString line;
2636 Int_t nfiles = 0;
2637 while (!ifile.eof()) {
2638 ifile >> line;
2639 if (line.Contains("/event")) nfiles++;
7c2cd90a 2640 }
b3e07543 2641 ifile.close();
2642 if (!nfiles) {
2643 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2644 return kFALSE;
2645 } else {
2646 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2647 }
b3e07543 2648 // Copy the file in the output directory
2649 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2650// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2651 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2652 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2653 // Check if this is the last stage to be done.
2654 Bool_t laststage = (nfiles<nperchunk);
2655 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
0ad3ea1f 2656 Int_t jobId = 0;
b3e07543 2657 if (laststage) {
2658 printf("### Submiting final merging stage %d\n", stage);
2659 TString finalJDL = jdl;
2660 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2661 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
0ad3ea1f 2662 jobId = SubmitSingleJob(query);
b3e07543 2663 } else {
2664 printf("### Submiting merging stage %d\n", stage);
03335324 2665 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
0ad3ea1f 2666 jobId = SubmitSingleJob(query);
7c2cd90a 2667 }
0ad3ea1f 2668 if (!jobId) return kFALSE;
2669
2670 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
2671 fGridJobIDs.Append(Form("%d", jobId));
2672 if (!fGridStages.IsNull()) fGridStages.Append(" ");
2673 fGridStages.Append(Form("%s_merge_stage%d",
2674 laststage ? "final" : "partial", stage));
2675
b3e07543 2676 return kTRUE;
2677}
7c2cd90a 2678
2679//______________________________________________________________________________
105631b5 2680AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2681{
2682// Loat the analysis manager from a file.
2683 TFile *file = TFile::Open(fname);
2684 if (!file) {
2685 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2686 return 0;
2687 }
2688 TIter nextkey(file->GetListOfKeys());
2689 AliAnalysisManager *mgr = 0;
2690 TKey *key;
2691 while ((key=(TKey*)nextkey())) {
2692 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2693 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2694 }
2695 if (!mgr)
2696 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2697 return mgr;
2698}
2699
2700//______________________________________________________________________________
7c2cd90a 2701Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2702{
2703// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2704 if (!gGrid) return 0;
2705 printf("=> %s ------> ",query);
2706 TGridResult *res = gGrid->Command(query);
2707 if (!res) return 0;
2708 TString jobId = res->GetKey(0,"jobId");
2709 delete res;
2710 if (jobId.IsNull()) {
2711 printf("submission failed. Reason:\n");
2712 gGrid->Stdout();
2713 gGrid->Stderr();
2714 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2715 return 0;
2716 }
0ad3ea1f 2717 Int_t ijobId = jobId.Atoi();
2718 printf(" Job id: '%s' (%d)\n", jobId.Data(), ijobId);
2719 return ijobId;
7c2cd90a 2720}
2721
2722//______________________________________________________________________________
ec5acdce 2723Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2724{
2725// Merges a collection of output files using concatenation.
d66aa2a9 2726 TString scoll(collection);
2727 if (!scoll.Contains(".xml")) return kFALSE;
ec5acdce 2728 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2729 if (!coll) {
2730 ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2731 return kFALSE;
2732 }
2733 // Iterate grid collection
2734 TString outtmp;
2735 Bool_t merged = kFALSE;
2736 Int_t ifile = 0;
2737 while (coll->Next()) {
2738 TString fname = gSystem->DirName(coll->GetTURL());
2739 fname += "/";
2740 fname += output;
2741 outtmp = Form("%d_%s", ifile, output);
2742 if (!TFile::Cp(fname, outtmp)) {
2743 ::Error("MergeInfo", "Could not copy %s", fname.Data());
2744 continue;
2745 }
2746 ifile++;
2747 if (ifile<2) {
2748 gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2749 continue;
2750 }
2751 gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2752 gSystem->Exec("cp tempmerged lastmerged");
2753 }
2754 if (ifile) {
2755 gSystem->Exec(Form("cp lastmerged %s", output));
2756 gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2757 merged = kTRUE;
2758 }
2759 return merged;
2760}
2761
2762//______________________________________________________________________________
b3e07543 2763Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2764{
b3e07543 2765// Merge given output files from basedir. Basedir can be an alien output directory
2766// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2767// files in a group (ignored for xml input). Merging can be done in stages:
2768// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2769// stage=1 : works with an xml of all root_archive.zip in the output directory
2770// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2771 TString outputFile = output;
0f389141 2772 TString command;
a2f5fc01 2773 TString outputChunk;
2774 TString previousChunk = "";
b3e07543 2775 TObjArray *listoffiles = new TObjArray();
2776// listoffiles->SetOwner();
a2f5fc01 2777 Int_t countChunk = 0;
2778 Int_t countZero = nmaxmerge;
0f389141 2779 Bool_t merged = kTRUE;
d66aa2a9 2780 Bool_t isGrid = kTRUE;
a2f5fc01 2781 Int_t index = outputFile.Index("@");
2782 if (index > 0) outputFile.Remove(index);
7c2cd90a 2783 TString inputFile = outputFile;
b3e07543 2784 TString sbasedir = basedir;
2785 if (sbasedir.Contains(".xml")) {
2786 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2787 nmaxmerge = 9999999;
2788 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2789 if (!coll) {
2790 ::Error("MergeOutput", "Input XML collection empty.");
2791 return kFALSE;
2792 }
2793 // Iterate grid collection
2794 while (coll->Next()) {
2795 TString fname = gSystem->DirName(coll->GetTURL());
2796 fname += "/";
2797 fname += inputFile;
2798 listoffiles->Add(new TNamed(fname.Data(),""));
2799 }
ffbe06b9 2800 } else if (sbasedir.Contains(".txt")) {
d66aa2a9 2801 // The file having the .txt extension is expected to contain a list of
2802 // folders where the output files will be looked. For alien folders,
2803 // the full folder LFN is expected (starting with alien://)
ffbe06b9 2804 // Assume lfn's on each line
2805 TString line;
2806 ifstream in;
2807 in.open(sbasedir);
d66aa2a9 2808 if (in.fail()) {
2809 ::Error("MergeOutput", "File %s cannot be opened. Merging stopped." ,sbasedir.Data());
2810 return kTRUE;
2811 }
ffbe06b9 2812 Int_t nfiles = 0;
2813 while (in.good()) {
2814 in >> line;
2815 if (line.IsNull() || line.BeginsWith("#")) continue;
d66aa2a9 2816 line.Strip();
2817 if (!line.Contains("alien:")) isGrid = kFALSE;
2818 line += "/";
2819 line += outputFile;
ffbe06b9 2820 nfiles++;
2821 listoffiles->Add(new TNamed(line.Data(),""));
2822 }
d66aa2a9 2823 in.close();
ffbe06b9 2824 if (!nfiles) {
2825 ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data());
2826 delete listoffiles;
2827 return kFALSE;
2828 }
b3e07543 2829 } else {
2830 command = Form("find %s/ *%s", basedir, inputFile.Data());
2831 printf("command: %s\n", command.Data());
2832 TGridResult *res = gGrid->Command(command);
2833 if (!res) {
2834 ::Error("MergeOutput","No result for the find command\n");
2835 delete listoffiles;
2836 return kFALSE;
2837 }
2838 TIter nextmap(res);
2839 TMap *map = 0;
2840 while ((map=(TMap*)nextmap())) {
2841 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2842 if (!objs || !objs->GetString().Length()) {
2843 // Nothing found - skip this output
2844 delete res;
2845 delete listoffiles;
2846 return kFALSE;
2847 }
2848 listoffiles->Add(new TNamed(objs->GetName(),""));
2849 }
2850 delete res;
2851 }
2852 if (!listoffiles->GetEntries()) {
7c2cd90a 2853 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2854 delete listoffiles;
0f389141 2855 return kFALSE;
2856 }
2857
2858 TFileMerger *fm = 0;
b3e07543 2859 TIter next0(listoffiles);
2860 TObjArray *listoffilestmp = new TObjArray();
2861 listoffilestmp->SetOwner();
2862 TObject *nextfile;
2863 TString snextfile;
2864 // Keep only the files at upper level
2865 Int_t countChar = 0;
2866 while ((nextfile=next0())) {
2867 snextfile = nextfile->GetName();
2868 Int_t crtCount = snextfile.CountChar('/');
2869 if (nextfile == listoffiles->First()) countChar = crtCount;
2870 if (crtCount < countChar) countChar = crtCount;
2871 }
2872 next0.Reset();
2873 while ((nextfile=next0())) {
2874 snextfile = nextfile->GetName();
2875 Int_t crtCount = snextfile.CountChar('/');
2876 if (crtCount > countChar) {
2877 delete nextfile;
2878 continue;
2879 }
2880 listoffilestmp->Add(nextfile);
2881 }
2882 delete listoffiles;
2883 listoffiles = listoffilestmp; // Now contains 'good' files
2884 listoffiles->Print();
2885 TIter next(listoffiles);
7c2cd90a 2886 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2887 outputChunk = outputFile;
2888 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2889 // Check for existent temporary merge files
e1c22e21 2890 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2891 // Preserve old merging functionality for stage 0.
2892 if (stage==0) {
2893 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2894 while (1) {
2895 // Skip as many input files as in a chunk
2ea55496 2896 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2897 nextfile = next();
2898 if (!nextfile) {
2ea55496 2899 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2900 delete listoffiles;
2ea55496 2901 return kFALSE;
2902 }
b3e07543 2903 snextfile = nextfile->GetName();
7c2cd90a 2904 }
2905 outputChunk = outputFile;
2906 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2907 countChunk++;
2908 if (gSystem->AccessPathName(outputChunk)) continue;
2909 // Merged file with chunks up to <countChunk> found
2910 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2911 previousChunk = outputChunk;
2912 break;
2913 }
2914 }
2915 countZero = nmaxmerge;
2916
b3e07543 2917 while ((nextfile=next())) {
2918 snextfile = nextfile->GetName();
2ea55496 2919 // Loop 'find' results and get next LFN
7c2cd90a 2920 if (countZero == nmaxmerge) {
2921 // First file in chunk - create file merger and add previous chunk if any.
d66aa2a9 2922 fm = new TFileMerger(isGrid);
7c2cd90a 2923 fm->SetFastMethod(kTRUE);
2924 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2925 outputChunk = outputFile;
2926 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2927 }
2928 // If last file found, put merged results in the output file
b3e07543 2929 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2930 // Add file to be merged and decrement chunk counter.
b3e07543 2931 fm->AddFile(snextfile);
7c2cd90a 2932 countZero--;
b3e07543 2933 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2934 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2935 // Nothing found - skip this output
2936 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2937 merged = kFALSE;
2938 break;
7c2cd90a 2939 }
2940 fm->OutputFile(outputChunk);
2941 // Merge the outputs, then go to next chunk
2942 if (!fm->Merge()) {
2943 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2944 merged = kFALSE;
2945 break;
7c2cd90a 2946 } else {
2947 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2948 gSystem->Unlink(previousChunk);
2949 }
b3e07543 2950 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2951 countChunk++;
2952 countZero = nmaxmerge;
2953 previousChunk = outputChunk;
0f389141 2954 }
0f389141 2955 }
b3e07543 2956 delete listoffiles;
2957 delete fm;
7c2cd90a 2958 return merged;
2959 }
2960 // Merging stage different than 0.
2961 // Move to the begining of the requested chunk.
d66aa2a9 2962 fm = new TFileMerger(isGrid);
7c2cd90a 2963 fm->SetFastMethod(kTRUE);
b3e07543 2964 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2965 delete listoffiles;
7c2cd90a 2966 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2967 // Nothing found - skip this output
2968 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2969 delete fm;
2970 return kFALSE;
2971 }
b3e07543 2972 fm->OutputFile(outputFile);
7c2cd90a 2973 // Merge the outputs
2974 if (!fm->Merge()) {
2975 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2976 delete fm;
2977 return kFALSE;
2978 } else {
b3e07543 2979 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2980 }