Fix from Salvatore for new framework
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
c82bb898 51using std::ofstream;
52using std::ifstream;
53using std::ios;
54using std::endl;
c57f56b7 55ClassImp(AliAnalysisAlien)
6c395669 56#if 0
57;
58#endif
c57f56b7 59
6c395669 60namespace {
61 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
62 {
63 TString sl(Form("file:%s", loc));
64 TString sr(Form("alien://%s", rem));
65 Bool_t ret = TFile::Cp(sl, sr);
66 if (!ret) {
67 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
68 }
69 return ret;
70 }
71}
72
c57f56b7 73//______________________________________________________________________________
74AliAnalysisAlien::AliAnalysisAlien()
75 :AliAnalysisGrid(),
76 fGridJDL(NULL),
0f389141 77 fMergingJDL(NULL),
c57f56b7 78 fPrice(0),
79 fTTL(0),
80 fSplitMaxInputFileNumber(0),
81 fMaxInitFailed(0),
82 fMasterResubmitThreshold(0),
bb885a9e 83 fNtestFiles(0),
319593fb 84 fNrunsPerMaster(0),
16a4353c 85 fMaxMergeFiles(0),
b3e07543 86 fMaxMergeStages(0),
d2a409b2 87 fNsubmitted(0),
a3e84053 88 fProductionMode(0),
cd11251e 89 fOutputToRunNo(0),
0f389141 90 fMergeViaJDL(0),
91 fFastReadOption(0),
e1c22e21 92 fOverwriteMode(1),
149d288c 93 fNreplicas(2),
3bdcb562 94 fNproofWorkers(0),
27734f0e 95 fNproofWorkersPerSlave(0),
3bdcb562 96 fProofReset(0),
c57f56b7 97 fRunNumbers(),
98 fExecutable(),
0a1c1f7f 99 fExecutableCommand(),
c57f56b7 100 fArguments(),
631c0b05 101 fExecutableArgs(),
c57f56b7 102 fAnalysisMacro(),
103 fAnalysisSource(),
d3b18c4c 104 fValidationScript(),
d5c6455a 105 fAdditionalRootLibs(),
c57f56b7 106 fAdditionalLibs(),
107 fSplitMode(),
108 fAPIVersion(),
109 fROOTVersion(),
110 fAliROOTVersion(),
648174cf 111 fExternalPackages(),
c57f56b7 112 fUser(),
113 fGridWorkingDir(),
114 fGridDataDir(),
115 fDataPattern(),
116 fGridOutputDir(),
117 fOutputArchive(),
118 fOutputFiles(),
119 fInputFormat(),
e7c71df0 120 fDatasetName(),
c57f56b7 121 fJDLName(),
d3b18c4c 122 fTerminateFiles(),
bb885a9e 123 fMergeExcludes(),
37d21c01 124 fRegisterExcludes(),
f965131e 125 fIncludePath(),
bb885a9e 126 fCloseSE(),
0df6ccf2 127 fFriendChainName(),
c6cb3634 128 fJobTag(),
648174cf 129 fOutputSingle(),
5fce53f4 130 fRunPrefix(),
3bdcb562 131 fProofCluster(),
132 fProofDataSet(),
133 fFileForTestMode(),
3bdcb562 134 fAliRootMode(),
d88e4159 135 fProofProcessOpt(),
f47d5cb4 136 fMergeDirName(),
4e5c5506 137 fInputFiles(0),
28ce6c9c 138 fPackages(0),
77f34eae 139 fModules(0),
d88e4159 140 fProofParam(),
a41b2882 141 fDropToShell(true),
142 fGridJobIDs(""),
143 fGridStages(""),
780ec36d 144 fFriendLibs(""),
145 fTreeName()
c57f56b7 146{
147// Dummy ctor.
148 SetDefaults();
149}
150
151//______________________________________________________________________________
152AliAnalysisAlien::AliAnalysisAlien(const char *name)
153 :AliAnalysisGrid(name),
154 fGridJDL(NULL),
0f389141 155 fMergingJDL(NULL),
c57f56b7 156 fPrice(0),
157 fTTL(0),
158 fSplitMaxInputFileNumber(0),
159 fMaxInitFailed(0),
160 fMasterResubmitThreshold(0),
bb885a9e 161 fNtestFiles(0),
319593fb 162 fNrunsPerMaster(0),
16a4353c 163 fMaxMergeFiles(0),
b3e07543 164 fMaxMergeStages(0),
d2a409b2 165 fNsubmitted(0),
a3e84053 166 fProductionMode(0),
cd11251e 167 fOutputToRunNo(0),
0f389141 168 fMergeViaJDL(0),
169 fFastReadOption(0),
e1c22e21 170 fOverwriteMode(1),
149d288c 171 fNreplicas(2),
3bdcb562 172 fNproofWorkers(0),
27734f0e 173 fNproofWorkersPerSlave(0),
3bdcb562 174 fProofReset(0),
c57f56b7 175 fRunNumbers(),
176 fExecutable(),
0a1c1f7f 177 fExecutableCommand(),
c57f56b7 178 fArguments(),
631c0b05 179 fExecutableArgs(),
c57f56b7 180 fAnalysisMacro(),
181 fAnalysisSource(),
d3b18c4c 182 fValidationScript(),
d5c6455a 183 fAdditionalRootLibs(),
c57f56b7 184 fAdditionalLibs(),
185 fSplitMode(),
186 fAPIVersion(),
187 fROOTVersion(),
188 fAliROOTVersion(),
648174cf 189 fExternalPackages(),
c57f56b7 190 fUser(),
191 fGridWorkingDir(),
192 fGridDataDir(),
193 fDataPattern(),
194 fGridOutputDir(),
195 fOutputArchive(),
196 fOutputFiles(),
197 fInputFormat(),
e7c71df0 198 fDatasetName(),
c57f56b7 199 fJDLName(),
d3b18c4c 200 fTerminateFiles(),
bb885a9e 201 fMergeExcludes(),
37d21c01 202 fRegisterExcludes(),
f965131e 203 fIncludePath(),
bb885a9e 204 fCloseSE(),
0df6ccf2 205 fFriendChainName(),
c6cb3634 206 fJobTag(),
648174cf 207 fOutputSingle(),
5fce53f4 208 fRunPrefix(),
3bdcb562 209 fProofCluster(),
210 fProofDataSet(),
211 fFileForTestMode(),
3bdcb562 212 fAliRootMode(),
d88e4159 213 fProofProcessOpt(),
f47d5cb4 214 fMergeDirName(),
4e5c5506 215 fInputFiles(0),
28ce6c9c 216 fPackages(0),
77f34eae 217 fModules(0),
d88e4159 218 fProofParam(),
0ad3ea1f 219 fDropToShell(true),
220 fGridJobIDs(""),
a41b2882 221 fGridStages(""),
780ec36d 222 fFriendLibs(""),
223 fTreeName()
c57f56b7 224{
225// Default ctor.
226 SetDefaults();
227}
228
229//______________________________________________________________________________
230AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
231 :AliAnalysisGrid(other),
232 fGridJDL(NULL),
0f389141 233 fMergingJDL(NULL),
c57f56b7 234 fPrice(other.fPrice),
235 fTTL(other.fTTL),
236 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
237 fMaxInitFailed(other.fMaxInitFailed),
238 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 239 fNtestFiles(other.fNtestFiles),
319593fb 240 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 241 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 242 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 243 fNsubmitted(other.fNsubmitted),
a3e84053 244 fProductionMode(other.fProductionMode),
cd11251e 245 fOutputToRunNo(other.fOutputToRunNo),
0f389141 246 fMergeViaJDL(other.fMergeViaJDL),
247 fFastReadOption(other.fFastReadOption),
248 fOverwriteMode(other.fOverwriteMode),
149d288c 249 fNreplicas(other.fNreplicas),
3bdcb562 250 fNproofWorkers(other.fNproofWorkers),
27734f0e 251 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 252 fProofReset(other.fProofReset),
c57f56b7 253 fRunNumbers(other.fRunNumbers),
254 fExecutable(other.fExecutable),
0a1c1f7f 255 fExecutableCommand(other.fExecutableCommand),
c57f56b7 256 fArguments(other.fArguments),
631c0b05 257 fExecutableArgs(other.fExecutableArgs),
c57f56b7 258 fAnalysisMacro(other.fAnalysisMacro),
259 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 260 fValidationScript(other.fValidationScript),
d5c6455a 261 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 262 fAdditionalLibs(other.fAdditionalLibs),
263 fSplitMode(other.fSplitMode),
264 fAPIVersion(other.fAPIVersion),
265 fROOTVersion(other.fROOTVersion),
266 fAliROOTVersion(other.fAliROOTVersion),
648174cf 267 fExternalPackages(other.fExternalPackages),
c57f56b7 268 fUser(other.fUser),
269 fGridWorkingDir(other.fGridWorkingDir),
270 fGridDataDir(other.fGridDataDir),
271 fDataPattern(other.fDataPattern),
272 fGridOutputDir(other.fGridOutputDir),
273 fOutputArchive(other.fOutputArchive),
274 fOutputFiles(other.fOutputFiles),
275 fInputFormat(other.fInputFormat),
e7c71df0 276 fDatasetName(other.fDatasetName),
c57f56b7 277 fJDLName(other.fJDLName),
d3b18c4c 278 fTerminateFiles(other.fTerminateFiles),
bb885a9e 279 fMergeExcludes(other.fMergeExcludes),
37d21c01 280 fRegisterExcludes(other.fRegisterExcludes),
f965131e 281 fIncludePath(other.fIncludePath),
bb885a9e 282 fCloseSE(other.fCloseSE),
0df6ccf2 283 fFriendChainName(other.fFriendChainName),
c6cb3634 284 fJobTag(other.fJobTag),
648174cf 285 fOutputSingle(other.fOutputSingle),
5fce53f4 286 fRunPrefix(other.fRunPrefix),
3bdcb562 287 fProofCluster(other.fProofCluster),
288 fProofDataSet(other.fProofDataSet),
289 fFileForTestMode(other.fFileForTestMode),
3bdcb562 290 fAliRootMode(other.fAliRootMode),
d88e4159 291 fProofProcessOpt(other.fProofProcessOpt),
f47d5cb4 292 fMergeDirName(other.fMergeDirName),
4e5c5506 293 fInputFiles(0),
28ce6c9c 294 fPackages(0),
77f34eae 295 fModules(0),
d88e4159 296 fProofParam(),
0ad3ea1f 297 fDropToShell(other.fDropToShell),
298 fGridJobIDs(other.fGridJobIDs),
a41b2882 299 fGridStages(other.fGridStages),
780ec36d 300 fFriendLibs(other.fFriendLibs),
301 fTreeName(other.fTreeName)
c57f56b7 302{
303// Copy ctor.
304 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 305 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 306 fRunRange[0] = other.fRunRange[0];
307 fRunRange[1] = other.fRunRange[1];
c57f56b7 308 if (other.fInputFiles) {
309 fInputFiles = new TObjArray();
310 TIter next(other.fInputFiles);
311 TObject *obj;
312 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
313 fInputFiles->SetOwner();
314 }
4e5c5506 315 if (other.fPackages) {
316 fPackages = new TObjArray();
317 TIter next(other.fPackages);
318 TObject *obj;
319 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
320 fPackages->SetOwner();
321 }
77f34eae 322 if (other.fModules) {
323 fModules = new TObjArray();
324 fModules->SetOwner();
325 TIter next(other.fModules);
326 AliAnalysisTaskCfg *mod, *crt;
327 while ((crt=(AliAnalysisTaskCfg*)next())) {
328 mod = new AliAnalysisTaskCfg(*crt);
329 fModules->Add(mod);
330 }
331 }
c57f56b7 332}
333
334//______________________________________________________________________________
335AliAnalysisAlien::~AliAnalysisAlien()
336{
337// Destructor.
77f34eae 338 delete fGridJDL;
339 delete fMergingJDL;
340 delete fInputFiles;
341 delete fPackages;
342 delete fModules;
28ce6c9c 343 fProofParam.DeleteAll();
c57f56b7 344}
345
346//______________________________________________________________________________
347AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
348{
349// Assignment.
350 if (this != &other) {
351 AliAnalysisGrid::operator=(other);
352 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 353 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 354 fPrice = other.fPrice;
355 fTTL = other.fTTL;
356 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
357 fMaxInitFailed = other.fMaxInitFailed;
358 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 359 fNtestFiles = other.fNtestFiles;
a3e84053 360 fNrunsPerMaster = other.fNrunsPerMaster;
361 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 362 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 363 fNsubmitted = other.fNsubmitted;
364 fProductionMode = other.fProductionMode;
cd11251e 365 fOutputToRunNo = other.fOutputToRunNo;
0f389141 366 fMergeViaJDL = other.fMergeViaJDL;
367 fFastReadOption = other.fFastReadOption;
368 fOverwriteMode = other.fOverwriteMode;
149d288c 369 fNreplicas = other.fNreplicas;
3bdcb562 370 fNproofWorkers = other.fNproofWorkers;
27734f0e 371 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 372 fProofReset = other.fProofReset;
c57f56b7 373 fRunNumbers = other.fRunNumbers;
374 fExecutable = other.fExecutable;
0a1c1f7f 375 fExecutableCommand = other.fExecutableCommand;
c57f56b7 376 fArguments = other.fArguments;
631c0b05 377 fExecutableArgs = other.fExecutableArgs;
c57f56b7 378 fAnalysisMacro = other.fAnalysisMacro;
379 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 380 fValidationScript = other.fValidationScript;
d5c6455a 381 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 382 fAdditionalLibs = other.fAdditionalLibs;
383 fSplitMode = other.fSplitMode;
384 fAPIVersion = other.fAPIVersion;
385 fROOTVersion = other.fROOTVersion;
386 fAliROOTVersion = other.fAliROOTVersion;
648174cf 387 fExternalPackages = other.fExternalPackages;
c57f56b7 388 fUser = other.fUser;
389 fGridWorkingDir = other.fGridWorkingDir;
390 fGridDataDir = other.fGridDataDir;
391 fDataPattern = other.fDataPattern;
392 fGridOutputDir = other.fGridOutputDir;
393 fOutputArchive = other.fOutputArchive;
394 fOutputFiles = other.fOutputFiles;
395 fInputFormat = other.fInputFormat;
e7c71df0 396 fDatasetName = other.fDatasetName;
c57f56b7 397 fJDLName = other.fJDLName;
d3b18c4c 398 fTerminateFiles = other.fTerminateFiles;
bb885a9e 399 fMergeExcludes = other.fMergeExcludes;
37d21c01 400 fRegisterExcludes = other.fRegisterExcludes;
f965131e 401 fIncludePath = other.fIncludePath;
bb885a9e 402 fCloseSE = other.fCloseSE;
0df6ccf2 403 fFriendChainName = other.fFriendChainName;
c6cb3634 404 fJobTag = other.fJobTag;
648174cf 405 fOutputSingle = other.fOutputSingle;
5fce53f4 406 fRunPrefix = other.fRunPrefix;
3bdcb562 407 fProofCluster = other.fProofCluster;
408 fProofDataSet = other.fProofDataSet;
409 fFileForTestMode = other.fFileForTestMode;
3bdcb562 410 fAliRootMode = other.fAliRootMode;
d88e4159 411 fProofProcessOpt = other.fProofProcessOpt;
f47d5cb4 412 fMergeDirName = other.fMergeDirName;
d88e4159 413 fDropToShell = other.fDropToShell;
0ad3ea1f 414 fGridJobIDs = other.fGridJobIDs;
415 fGridStages = other.fGridStages;
a41b2882 416 fFriendLibs = other.fFriendLibs;
780ec36d 417 fTreeName = other.fTreeName;
c57f56b7 418 if (other.fInputFiles) {
419 fInputFiles = new TObjArray();
420 TIter next(other.fInputFiles);
421 TObject *obj;
422 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
423 fInputFiles->SetOwner();
424 }
4e5c5506 425 if (other.fPackages) {
426 fPackages = new TObjArray();
427 TIter next(other.fPackages);
428 TObject *obj;
429 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
430 fPackages->SetOwner();
431 }
77f34eae 432 if (other.fModules) {
433 fModules = new TObjArray();
434 fModules->SetOwner();
435 TIter next(other.fModules);
436 AliAnalysisTaskCfg *mod, *crt;
437 while ((crt=(AliAnalysisTaskCfg*)next())) {
438 mod = new AliAnalysisTaskCfg(*crt);
439 fModules->Add(mod);
440 }
441 }
c57f56b7 442 }
443 return *this;
444}
445
446//______________________________________________________________________________
37d21c01 447void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
448{
449// Add a single additional library to be loaded. Extension must be present.
450 TString lib(name);
451 if (!lib.Contains(".")) {
452 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
453 return;
454 }
455 if (fAdditionalLibs.Contains(name)) {
456 Warning("AddAdditionalLibrary", "Library %s already added.", name);
457 return;
458 }
459 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
460 fAdditionalLibs += lib;
461}
462
463//______________________________________________________________________________
77f34eae 464void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
465{
466// Adding a module. Checks if already existing. Becomes owned by this.
467 if (!module) return;
468 if (GetModule(module->GetName())) {
469 Error("AddModule", "A module having the same name %s already added", module->GetName());
470 return;
471 }
472 if (!fModules) {
473 fModules = new TObjArray();
474 fModules->SetOwner();
475 }
476 fModules->Add(module);
477}
478
479//______________________________________________________________________________
480void AliAnalysisAlien::AddModules(TObjArray *list)
481{
482// Adding a list of modules. Checks if already existing. Becomes owned by this.
483 TIter next(list);
484 AliAnalysisTaskCfg *module;
485 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
486}
487
488//______________________________________________________________________________
489Bool_t AliAnalysisAlien::CheckDependencies()
490{
491// Check if all dependencies are satisfied. Reorder modules if needed.
492 Int_t nmodules = GetNmodules();
493 if (!nmodules) {
494 Warning("CheckDependencies", "No modules added yet to check their dependencies");
495 return kTRUE;
496 }
497 AliAnalysisTaskCfg *mod = 0;
498 AliAnalysisTaskCfg *dep = 0;
499 TString depname;
500 Int_t i, j, k;
501 for (i=0; i<nmodules; i++) {
502 mod = (AliAnalysisTaskCfg*) fModules->At(i);
503 Int_t ndeps = mod->GetNdeps();
504 Int_t istart = i;
505 for (j=0; j<ndeps; j++) {
506 depname = mod->GetDependency(j);
507 dep = GetModule(depname);
508 if (!dep) {
509 Error("CheckDependencies","Dependency %s not added for module %s",
510 depname.Data(), mod->GetName());
511 return kFALSE;
512 }
513 if (dep->NeedsDependency(mod->GetName())) {
514 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
515 mod->GetName(), dep->GetName());
516 return kFALSE;
517 }
518 Int_t idep = fModules->IndexOf(dep);
519 // The dependency task must come first
520 if (idep>i) {
521 // Remove at idep and move all objects below up one slot
522 // down to index i included.
523 fModules->RemoveAt(idep);
3e40fd4c 524 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 525 fModules->AddAt(dep, i++);
526 }
527 //Redo from istart if dependencies were inserted
528 if (i>istart) i=istart-1;
529 }
530 }
531 return kTRUE;
532}
533
534//______________________________________________________________________________
3e40fd4c 535AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
536{
537// Create the analysis manager and optionally execute the macro in filename.
538 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
539 if (mgr) return mgr;
540 mgr = new AliAnalysisManager(name);
541 mgr->SetGridHandler((AliAnalysisGrid*)this);
542 if (strlen(filename)) {
543 TString line = gSystem->ExpandPathName(filename);
544 line.Prepend(".x ");
545 gROOT->ProcessLine(line.Data());
546 }
547 return mgr;
548}
549
550//______________________________________________________________________________
77f34eae 551Int_t AliAnalysisAlien::GetNmodules() const
552{
553// Get number of modules.
554 if (!fModules) return 0;
555 return fModules->GetEntries();
556}
557
558//______________________________________________________________________________
559AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
560{
561// Get a module by name.
562 if (!fModules) return 0;
563 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
564}
565
566//______________________________________________________________________________
567Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
568{
569// Load a given module.
570 if (mod->IsLoaded()) return kTRUE;
192ddca2 571 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
572 if (!mgr) {
573 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
574 return kFALSE;
575 }
77f34eae 576 Int_t ndeps = mod->GetNdeps();
577 TString depname;
578 for (Int_t j=0; j<ndeps; j++) {
579 depname = mod->GetDependency(j);
580 AliAnalysisTaskCfg *dep = GetModule(depname);
581 if (!dep) {
582 Error("LoadModule","Dependency %s not existing for module %s",
583 depname.Data(), mod->GetName());
584 return kFALSE;
585 }
586 if (!LoadModule(dep)) {
587 Error("LoadModule","Dependency %s for module %s could not be loaded",
588 depname.Data(), mod->GetName());
589 return kFALSE;
590 }
591 }
592 // Load libraries for the module
593 if (!mod->CheckLoadLibraries()) {
594 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
595 return kFALSE;
596 }
192ddca2 597 // Check if a custom file name was requested
598 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
599
600 // Check if a custom terminate file name was requested
601 if (strlen(mod->GetTerminateFileName())) {
602 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
603 fTerminateFiles += mod->GetTerminateFileName();
604 }
605
77f34eae 606 // Execute the macro
607 if (mod->ExecuteMacro()<0) {
608 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
609 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
610 return kFALSE;
611 }
612 // Configure dependencies
613 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
614 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
615 mod->GetConfigMacro()->GetTitle(), mod->GetName());
616 return kFALSE;
617 }
5e8c7c22 618 // Adjust extra libraries
619 Int_t nlibs = mod->GetNlibs();
620 TString lib;
621 for (Int_t i=0; i<nlibs; i++) {
622 lib = mod->GetLibrary(i);
5e8c7c22 623 lib = Form("lib%s.so", lib.Data());
c9d6bd12 624 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 625 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
626 fAdditionalLibs += lib;
627 }
77f34eae 628 return kTRUE;
629}
630
631//______________________________________________________________________________
139fe6ba 632Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
633{
634// Generate the full train.
635 fAdditionalLibs = "";
636 if (!LoadModules()) return kFALSE;
637 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
638 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 639 mgr->RunLocalInit();
139fe6ba 640 mgr->PrintStatus();
641 Int_t productionMode = fProductionMode;
642 SetProductionMode();
643 TString macro = fAnalysisMacro;
644 TString executable = fExecutable;
645 TString validation = fValidationScript;
646 TString execCommand = fExecutableCommand;
647 SetAnalysisMacro(Form("%s.C", name));
648 SetExecutable(Form("%s.sh", name));
a8e31fb0 649// SetExecutableCommand("aliroot -b -q ");
139fe6ba 650 SetValidationScript(Form("%s_validation.sh", name));
651 StartAnalysis();
652 SetProductionMode(productionMode);
653 fAnalysisMacro = macro;
654 fExecutable = executable;
655 fExecutableCommand = execCommand;
656 fValidationScript = validation;
657 return kTRUE;
658}
659
660//______________________________________________________________________________
21ca8e59 661Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 662{
663// Generate test macros for a single module or for the full train.
5e8c7c22 664 fAdditionalLibs = "";
77f34eae 665 if (strlen(modname)) {
21ca8e59 666 if (!CheckDependencies()) return kFALSE;
667 AliAnalysisTaskCfg *mod = GetModule(modname);
668 if (!mod) {
669 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
670 return kFALSE;
671 }
672 if (!LoadModule(mod)) return kFALSE;
a41b2882 673 if (!LoadFriendLibs()) return kFALSE;
21ca8e59 674 } else if (!LoadModules()) return kFALSE;
675 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
676 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 677 mgr->RunLocalInit();
21ca8e59 678 mgr->PrintStatus();
679 SetLocalTest(kTRUE);
680 Int_t productionMode = fProductionMode;
681 SetProductionMode();
682 TString macro = fAnalysisMacro;
683 TString executable = fExecutable;
684 TString validation = fValidationScript;
685 TString execCommand = fExecutableCommand;
686 SetAnalysisMacro(Form("%s.C", name));
687 SetExecutable(Form("%s.sh", name));
d66aa2a9 688 fOutputFiles = GetListOfFiles("outaod");
689 // Add extra files registered to the analysis manager
690 TString extra = GetListOfFiles("ext");
691 if (!extra.IsNull()) {
692 extra.ReplaceAll(".root", "*.root");
693 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
694 fOutputFiles += extra;
695 }
a8e31fb0 696// SetExecutableCommand("aliroot -b -q ");
21ca8e59 697 SetValidationScript(Form("%s_validation.sh", name));
698 WriteAnalysisFile();
699 WriteAnalysisMacro();
700 WriteExecutable();
701 WriteValidationScript();
ffbe06b9 702 WriteMergingMacro();
703 WriteMergeExecutable();
704 WriteValidationScript(kTRUE);
21ca8e59 705 SetLocalTest(kFALSE);
706 SetProductionMode(productionMode);
707 fAnalysisMacro = macro;
708 fExecutable = executable;
709 fExecutableCommand = execCommand;
710 fValidationScript = validation;
77f34eae 711 return kTRUE;
712}
713
714//______________________________________________________________________________
715Bool_t AliAnalysisAlien::LoadModules()
716{
717// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 718 fAdditionalLibs = "";
21ca8e59 719 Int_t nmodules = GetNmodules();
720 if (!nmodules) {
721 Warning("LoadModules", "No module to be loaded");
722 return kTRUE;
723 }
3e40fd4c 724 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
725 if (!mgr) {
726 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
727 return kFALSE;
728 }
77f34eae 729 if (!CheckDependencies()) return kFALSE;
21ca8e59 730 nmodules = GetNmodules();
77f34eae 731 AliAnalysisTaskCfg *mod;
21ca8e59 732 for (Int_t imod=0; imod<nmodules; imod++) {
733 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 734 if (!LoadModule(mod)) return kFALSE;
735 }
a41b2882 736 // Load additional friend libraries
737 return LoadFriendLibs();
77f34eae 738}
739
740//______________________________________________________________________________
a41b2882 741Bool_t AliAnalysisAlien::LoadFriendLibs() const
742{
743// Load libraries required for reading friends.
744 if (fFriendLibs.Length()) {
745 TObjArray *list = 0;
746 TString lib;
747 if (fFriendLibs.Contains(",")) list = fFriendLibs.Tokenize(",");
748 else list = fFriendLibs.Tokenize(" ");
749 for (Int_t ilib=0; ilib<list->GetEntriesFast(); ilib++) {
750 lib = list->At(ilib)->GetName();
751 lib.ReplaceAll(".so","");
752 lib.ReplaceAll(" ","");
753 if (lib.BeginsWith("lib")) lib.Remove(0, 3);
754 lib.Prepend("lib");
755 Int_t loaded = strlen(gSystem->GetLibraries(lib,"",kFALSE));
756 if (!loaded) loaded = gSystem->Load(lib);
757 if (loaded < 0) {
758 Error("LoadModules", "Cannot load library for friends %s", lib.Data());
759 return kFALSE;
760 }
761 }
762 delete list;
763 }
764 return kTRUE;
765}
766
767//______________________________________________________________________________
74013535 768void AliAnalysisAlien::SetRunPrefix(const char *prefix)
769{
770// Set the run number format. Can be a prefix or a format like "%09d"
771 fRunPrefix = prefix;
772 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
773}
774
775//______________________________________________________________________________
f965131e 776void AliAnalysisAlien::AddIncludePath(const char *path)
777{
778// Add include path in the remote analysis macro.
779 TString p(path);
780 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
781 else fIncludePath += Form("-I%s ", path);
782}
783
784//______________________________________________________________________________
c57f56b7 785void AliAnalysisAlien::AddRunNumber(Int_t run)
786{
787// Add a run number to the list of runs to be processed.
788 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 789 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 790}
791
792//______________________________________________________________________________
98690895 793void AliAnalysisAlien::AddRunList(const char* runList)
794{
795// Add several runs into the list of runs; they are expected to be separated by a blank character.
796 TString sList = runList;
797 TObjArray *list = sList.Tokenize(" ");
798 Int_t n = list->GetEntries();
799 for (Int_t i = 0; i < n; i++) {
800 TObjString *os = (TObjString*)list->At(i);
801 AddRunNumber(os->GetString().Atoi());
802 }
803 delete list;
804}
805
806//______________________________________________________________________________
ee75cfc3 807void AliAnalysisAlien::AddRunNumber(const char* run)
808{
809// Add a run number to the list of runs to be processed.
7aa5d539 810 TString runs = run;
811 TObjString *os;
812 TObjArray *arr = runs.Tokenize(" ");
813 TIter next(arr);
814 TString prefix;
815 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
816 while ((os=(TObjString*)next())){
817 if (fRunNumbers.Length()) fRunNumbers += " ";
818 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
819 }
820 delete arr;
ee75cfc3 821}
822
823//______________________________________________________________________________
c57f56b7 824void AliAnalysisAlien::AddDataFile(const char *lfn)
825{
826// Adds a data file to the input to be analysed. The file should be a valid LFN
827// or point to an existing file in the alien workdir.
828 if (!fInputFiles) fInputFiles = new TObjArray();
829 fInputFiles->Add(new TObjString(lfn));
830}
648174cf 831
832//______________________________________________________________________________
833void AliAnalysisAlien::AddExternalPackage(const char *package)
834{
835// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
836 if (fExternalPackages) fExternalPackages += " ";
837 fExternalPackages += package;
838}
839
c57f56b7 840//______________________________________________________________________________
841Bool_t AliAnalysisAlien::Connect()
842{
843// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
844 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 845 if (fProductionMode) return kTRUE;
c57f56b7 846 if (!gGrid) {
847 Info("Connect", "Trying to connect to AliEn ...");
848 TGrid::Connect("alien://");
849 }
850 if (!gGrid || !gGrid->IsConnected()) {
851 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
852 return kFALSE;
853 }
854 fUser = gGrid->GetUser();
855 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
856 return kTRUE;
857}
858
859//______________________________________________________________________________
860void AliAnalysisAlien::CdWork()
861{
862// Check validity of alien workspace. Create directory if possible.
863 if (!Connect()) {
864 Error("CdWork", "Alien connection required");
865 return;
866 }
867 TString homedir = gGrid->GetHomeDirectory();
868 TString workdir = homedir + fGridWorkingDir;
923e2ca5 869 if (DirectoryExists(workdir)) {
870 gGrid->Cd(workdir);
871 return;
872 }
873 // Work directory not existing - create it
874 gGrid->Cd(homedir);
b93f8109 875 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 876 gGrid->Cd(fGridWorkingDir);
d3b18c4c 877 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 878 } else {
d3b18c4c 879 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 880 workdir.Data(), homedir.Data());
881 fGridWorkingDir = "";
882 }
c57f56b7 883}
884
885//______________________________________________________________________________
348be253 886Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
887{
888// Check if file copying is possible.
d3b18c4c 889 if (fProductionMode) return kTRUE;
b5ae4fe0 890 TString salienpath(alienpath);
891 if (salienpath.Contains(" ")) {
892 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
893 return kFALSE;
894 }
348be253 895 if (!Connect()) {
896 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
897 return kFALSE;
898 }
d3339be3 899 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
900 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 901 // Check if alien_CLOSE_SE is defined
902 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
903 if (!closeSE.IsNull()) {
904 Info("CheckFileCopy", "Your current close storage is pointing to: \
905 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
906 } else {
907 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
908 }
909 // Check if grid directory exists.
910 if (!DirectoryExists(alienpath)) {
911 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
912 return kFALSE;
913 }
b5ae4fe0 914 TString stest = "plugin_test_copy";
915 TFile f(stest, "RECREATE");
348be253 916 // User may not have write permissions to current directory
917 if (f.IsZombie()) {
918 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
919 gSystem->WorkingDirectory());
920 return kFALSE;
921 }
922 f.Close();
b5ae4fe0 923 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
924 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 925 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 926 \n# 1. Make sure you have write permissions there. If this is the case: \
927 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
928 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
929 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
930 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 931 gSystem->Unlink(stest.Data());
348be253 932 return kFALSE;
933 }
b5ae4fe0 934 gSystem->Unlink(stest.Data());
935 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 936 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 937 return kTRUE;
938}
939
940//______________________________________________________________________________
c57f56b7 941Bool_t AliAnalysisAlien::CheckInputData()
942{
943// Check validity of input data. If necessary, create xml files.
d3b18c4c 944 if (fProductionMode) return kTRUE;
a8739e8a 945 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 946 if (!fGridDataDir.Length()) {
947 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
948 return kFALSE;
949 }
9f020952 950 if (fMergeViaJDL) {
951 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
952 return kFALSE;
953 }
d2a409b2 954 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 955 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
956 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 957 return kTRUE;
c57f56b7 958 }
959 // Process declared files
a2f5fc01 960 Bool_t isCollection = kFALSE;
961 Bool_t isXml = kFALSE;
962 Bool_t useTags = kFALSE;
c57f56b7 963 Bool_t checked = kFALSE;
d3b18c4c 964 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 965 TString file;
966 TString workdir = gGrid->GetHomeDirectory();
967 workdir += fGridWorkingDir;
968 if (fInputFiles) {
969 TObjString *objstr;
970 TIter next(fInputFiles);
971 while ((objstr=(TObjString*)next())) {
972 file = workdir;
973 file += "/";
974 file += objstr->GetString();
975 // Store full lfn path
976 if (FileExists(file)) objstr->SetString(file);
977 else {
978 file = objstr->GetName();
979 if (!FileExists(objstr->GetName())) {
980 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
981 objstr->GetName(), workdir.Data());
982 return kFALSE;
983 }
984 }
985 Bool_t iscoll, isxml, usetags;
986 CheckDataType(file, iscoll, isxml, usetags);
987 if (!checked) {
988 checked = kTRUE;
a2f5fc01 989 isCollection = iscoll;
990 isXml = isxml;
991 useTags = usetags;
992 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 993 } else {
a2f5fc01 994 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 995 Error("CheckInputData", "Some conflict was found in the types of inputs");
996 return kFALSE;
997 }
998 }
999 }
1000 }
1001 // Process requested run numbers
a8739e8a 1002 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 1003 // Check validity of alien data directory
1004 if (!fGridDataDir.Length()) {
1005 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
1006 return kFALSE;
1007 }
923e2ca5 1008 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 1009 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
1010 return kFALSE;
1011 }
a2f5fc01 1012 if (isCollection) {
c57f56b7 1013 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
1014 return kFALSE;
1015 }
1016
a2f5fc01 1017 if (checked && !isXml) {
c57f56b7 1018 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
1019 return kFALSE;
1020 }
1021 // Check validity of run number(s)
1022 TObjArray *arr;
1023 TObjString *os;
74013535 1024 TString format;
319593fb 1025 Int_t nruns = 0;
904f9f5f 1026 TString schunk, schunk2;
c57f56b7 1027 TString path;
1028 if (!checked) {
1029 checked = kTRUE;
a2f5fc01 1030 useTags = fDataPattern.Contains("tag");
1031 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1032 }
a2f5fc01 1033 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 1034 Error("CheckInputData", "Cannot mix input files using/not using tags");
1035 return kFALSE;
1036 }
1037 if (fRunNumbers.Length()) {
a8739e8a 1038 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 1039 arr = fRunNumbers.Tokenize(" ");
1040 TIter next(arr);
1041 while ((os=(TObjString*)next())) {
1042 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1043 if (!DirectoryExists(path)) {
1044 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 1045 continue;
c57f56b7 1046 }
1047 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
1048 TString msg = "\n##### file: ";
1049 msg += path;
1050 msg += " type: xml_collection;";
a2f5fc01 1051 if (useTags) msg += " using_tags: Yes";
c57f56b7 1052 else msg += " using_tags: No";
23329835 1053 Info("CheckDataType", "%s", msg.Data());
319593fb 1054 if (fNrunsPerMaster<2) {
d2a409b2 1055 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 1056 } else {
1057 nruns++;
1058 if (((nruns-1)%fNrunsPerMaster) == 0) {
1059 schunk = os->GetString();
1060 }
1061 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1062 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1063 AddDataFile(schunk);
319593fb 1064 }
c57f56b7 1065 }
1066 delete arr;
a8739e8a 1067 } else {
1068 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1069 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1070 format = Form("%%s/%s ", fRunPrefix.Data());
1071 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1072 if (!DirectoryExists(path)) {
a8739e8a 1073 continue;
1074 }
74013535 1075 format = Form("%%s/%s.xml", fRunPrefix.Data());
1076 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1077 TString msg = "\n##### file: ";
1078 msg += path;
1079 msg += " type: xml_collection;";
a2f5fc01 1080 if (useTags) msg += " using_tags: Yes";
a8739e8a 1081 else msg += " using_tags: No";
23329835 1082 Info("CheckDataType", "%s", msg.Data());
319593fb 1083 if (fNrunsPerMaster<2) {
74013535 1084 format = Form("%s.xml", fRunPrefix.Data());
1085 AddDataFile(Form(format.Data(),irun));
319593fb 1086 } else {
1087 nruns++;
1088 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1089 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1090 }
74013535 1091 format = Form("_%s.xml", fRunPrefix.Data());
1092 schunk2 = Form(format.Data(), irun);
319593fb 1093 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1094 schunk += schunk2;
d2a409b2 1095 AddDataFile(schunk);
319593fb 1096 }
a8739e8a 1097 }
904f9f5f 1098 if (!fInputFiles) {
1099 schunk += schunk2;
1100 AddDataFile(schunk);
1101 }
c57f56b7 1102 }
1103 return kTRUE;
1104}
1105
1106//______________________________________________________________________________
4a8b667c 1107Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1108{
1109// Copy data from the given grid directory according a pattern and make a local
1110// dataset.
2ba4c287 1111// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1112 if (!Connect()) {
1113 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
4a8b667c 1114 return 0;
21ca8e59 1115 }
1116 if (!DirectoryExists(griddir)) {
1117 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
4a8b667c 1118 return 0;
21ca8e59 1119 }
1120 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1121 printf("Running command: %s\n", command.Data());
1122 TGridResult *res = gGrid->Command(command);
1123 Int_t nfound = res->GetEntries();
1124 if (!nfound) {
1125 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
4a8b667c 1126 return 0;
21ca8e59 1127 }
1128 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1129
1130 // archives
1131 TObjArray* additionalArchives = 0;
1132 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1133 additionalArchives = TString(archivefile).Tokenize(";");
1134 archivefile = additionalArchives->At(0)->GetName();
1135 additionalArchives->RemoveAt(0);
1136 additionalArchives->Compress();
1137 }
1138
21ca8e59 1139 // Copy files locally
1140 ofstream out;
1141 out.open(output, ios::out);
1142 TMap *map;
1143 TString turl, dirname, filename, temp;
1144 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1145 gSystem->MakeDirectory(outputdir);
1146 gSystem->ChangeDirectory(outputdir);
4a8b667c 1147 Int_t ncopied = 0;
21ca8e59 1148 for (Int_t i=0; i<nfound; i++) {
1149 map = (TMap*)res->At(i);
1150 turl = map->GetValue("turl")->GetName();
1151 filename = gSystem->BaseName(turl.Data());
1152 dirname = gSystem->DirName(turl.Data());
1153 dirname = gSystem->BaseName(dirname.Data());
1154 gSystem->MakeDirectory(dirname);
43b807f6 1155
1156 TString source(turl);
1157 TString targetFileName(filename);
1158
1159 if (strlen(archivefile) > 0) {
4a8b667c 1160// TODO here the archive in which the file resides should be determined
1161// however whereis returns only a guid, and guid2lfn does not work
1162// Therefore we use the one provided as argument for now
1163 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1164 targetFileName = archivefile;
43b807f6 1165 }
1166 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
4a8b667c 1167 Bool_t success = kTRUE;
1168 if (additionalArchives) {
1169 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1170 TString target;
1171 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1172 gSystem->MakeDirectory(gSystem->DirName(target));
1173 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1174 }
1175 }
2ba4c287 1176
4a8b667c 1177 if (success) {
1178 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1179 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1180 ncopied++;
1181 }
21ca8e59 1182 }
1183 }
1184 gSystem->ChangeDirectory(cdir);
1185 delete res;
4a8b667c 1186 delete additionalArchives;
1187 return ncopied;
21ca8e59 1188}
1189
1190//______________________________________________________________________________
c57f56b7 1191Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1192{
1193// Create dataset for the grid data directory + run number.
3af861ff 1194 const Int_t gMaxEntries = 15000;
d3b18c4c 1195 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1196 if (!Connect()) {
1197 Error("CreateDataset", "Cannot create dataset with no grid connection");
1198 return kFALSE;
1199 }
1200
1201 // Cd workspace
d3b18c4c 1202 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1203 TString workdir = gGrid->GetHomeDirectory();
1204 workdir += fGridWorkingDir;
1205
1206 // Compose the 'find' command arguments
74013535 1207 TString format;
c57f56b7 1208 TString command;
780ec36d 1209 TString delimiter = pattern;
1210 delimiter.Strip();
1211 if (delimiter.Contains(" ")) delimiter = "";
1212 else delimiter = " ";
c57f56b7 1213 TString options = "-x collection ";
bb885a9e 1214 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1215 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1216 TString conditions = "";
3af861ff 1217 Int_t nstart = 0;
1218 Int_t ncount = 0;
1219 Int_t stage = 0;
c57f56b7 1220 TString file;
1221 TString path;
319593fb 1222 Int_t nruns = 0;
904f9f5f 1223 TString schunk, schunk2;
ab254fd1 1224 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1225 if (!fRunNumbers.Length() && !fRunRange[0]) {
1226 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1227 // Make a single data collection from data directory.
1228 path = fGridDataDir;
923e2ca5 1229 if (!DirectoryExists(path)) {
d2a409b2 1230 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1231 return kFALSE;
780ec36d 1232 }
923e2ca5 1233// CdWork();
d2a409b2 1234 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1235 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1236 while (1) {
1237 ncount = 0;
1238 stage++;
1239 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1240 command = "find ";
1241 command += Form("%s -o %d ",options.Data(), nstart);
1242 command += path;
780ec36d 1243 command += delimiter;
3af861ff 1244 command += pattern;
1245 command += conditions;
1246 printf("command: %s\n", command.Data());
1247 TGridResult *res = gGrid->Command(command);
1248 if (res) delete res;
1249 // Write standard output to file
1250 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1251 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1252 Bool_t nullFile = kFALSE;
1253 if (!hasGrep) {
1254 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1255 } else {
1256 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1257 if (nullFile) {
1258 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1259 gSystem->Exec("rm -f __tmp*");
1260 return kFALSE;
1261 }
1262 TString line;
1263 ifstream in;
1264 in.open("__tmp__");
1265 in >> line;
1266 in.close();
1267 gSystem->Exec("rm -f __tmp__");
1268 ncount = line.Atoi();
1269 }
1270 }
1271 if (ncount == gMaxEntries) {
1272 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1273 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1274 if (!cbase) cbase = cadd;
1275 else {
1276 cbase->Add(cadd);
1277 delete cadd;
defd7a3a 1278 }
3af861ff 1279 nstart += ncount;
1280 } else {
1281 if (cbase) {
1282 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1283 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1284 cbase->Add(cadd);
1285 delete cadd;
1286 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1287 delete cbase; cbase = 0;
1288 } else {
1289 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1290 }
1291 gSystem->Exec("rm -f __tmp*");
1292 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1293 break;
1294 }
0f389141 1295 }
1296 Bool_t fileExists = FileExists(file);
1297 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1298 // Copy xml file to alien space
0f389141 1299 if (fileExists) gGrid->Rm(file);
d2a409b2 1300 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1301 if (!FileExists(file)) {
1302 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1303 return kFALSE;
1304 }
1305 // Update list of files to be processed.
1306 }
1307 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1308 return kTRUE;
1309 }
c57f56b7 1310 // Several runs
a2f5fc01 1311 Bool_t nullResult = kTRUE;
a8739e8a 1312 if (fRunNumbers.Length()) {
1313 TObjArray *arr = fRunNumbers.Tokenize(" ");
1314 TObjString *os;
1315 TIter next(arr);
1316 while ((os=(TObjString*)next())) {
3af861ff 1317 nstart = 0;
1318 stage = 0;
780ec36d 1319 path = Form("%s/%s/", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1320 if (!DirectoryExists(path)) continue;
1321// CdWork();
a8739e8a 1322 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1323 else file = Form("%s.xml", os->GetString().Data());
319593fb 1324 // If local collection file does not exist, create it via 'find' command.
3af861ff 1325 while (1) {
1326 ncount = 0;
1327 stage++;
1328 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1329 command = "find ";
1330 command += Form("%s -o %d ",options.Data(), nstart);
1331 command += path;
780ec36d 1332 command += delimiter;
3af861ff 1333 command += pattern;
1334 command += conditions;
1335 TGridResult *res = gGrid->Command(command);
1336 if (res) delete res;
1337 // Write standard output to file
1338 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1339 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1340 Bool_t nullFile = kFALSE;
1341 if (!hasGrep) {
1342 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1343 } else {
1344 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1345 if (nullFile) {
1346 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1347 gSystem->Exec("rm -f __tmp*");
1348 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1349 break;
3af861ff 1350 }
1351 TString line;
1352 ifstream in;
1353 in.open("__tmp__");
1354 in >> line;
1355 in.close();
1356 gSystem->Exec("rm -f __tmp__");
1357 ncount = line.Atoi();
1358 }
1359 nullResult = kFALSE;
1360 }
1361 if (ncount == gMaxEntries) {
1362 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1363 if (fNrunsPerMaster > 1) {
1364 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1365 file.Data(),gMaxEntries);
1366 return kFALSE;
1367 }
1368 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1369 if (!cbase) cbase = cadd;
1370 else {
1371 cbase->Add(cadd);
1372 delete cadd;
defd7a3a 1373 }
3af861ff 1374 nstart += ncount;
1375 } else {
c89a84bc 1376 if (cbase && fNrunsPerMaster<2) {
3af861ff 1377 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1378 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1379 cbase->Add(cadd);
1380 delete cadd;
1381 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1382 delete cbase; cbase = 0;
1383 } else {
1384 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1385 }
1386 gSystem->Exec("rm -f __tmp*");
1387 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1388 break;
d3339be3 1389 }
3af861ff 1390 }
a8739e8a 1391 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1392 // Check if there is one run per master job.
1393 if (fNrunsPerMaster<2) {
1394 if (FileExists(file)) {
0f389141 1395 if (fOverwriteMode) gGrid->Rm(file);
1396 else {
1397 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1398 continue;
1399 }
319593fb 1400 }
1401 // Copy xml file to alien space
1402 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1403 if (!FileExists(file)) {
1404 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1405 delete arr;
1406 return kFALSE;
1407 }
1408 } else {
1409 nruns++;
1410 if (((nruns-1)%fNrunsPerMaster) == 0) {
1411 schunk = os->GetString();
1412 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1413 } else {
1414 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1415 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1416 cbase->Add(cadd);
1417 delete cadd;
1418 }
1419 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1420 continue;
1421 }
1422 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1423 if (FileExists(schunk)) {
1424 if (fOverwriteMode) gGrid->Rm(file);
1425 else {
1426 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1427 continue;
1428 }
319593fb 1429 }
84fcd93f 1430 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1431 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1432 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1433 if (!FileExists(schunk)) {
1434 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1435 delete arr;
1436 return kFALSE;
1437 }
d3339be3 1438 }
a8739e8a 1439 }
1440 delete arr;
a2f5fc01 1441 if (nullResult) {
d3339be3 1442 Error("CreateDataset", "No valid dataset corresponding to the query!");
1443 return kFALSE;
1444 }
a8739e8a 1445 } else {
1446 // Process a full run range.
1447 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
780ec36d 1448 format = Form("%%s/%s/", fRunPrefix.Data());
3af861ff 1449 nstart = 0;
1450 stage = 0;
74013535 1451 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1452 if (!DirectoryExists(path)) continue;
1453// CdWork();
74013535 1454 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1455 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1456 else file = Form(format.Data(), irun);
0f389141 1457 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1458 if (fOverwriteMode) gGrid->Rm(file);
1459 else {
1460 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1461 continue;
1462 }
a8739e8a 1463 }
319593fb 1464 // If local collection file does not exist, create it via 'find' command.
3af861ff 1465 while (1) {
1466 ncount = 0;
1467 stage++;
1468 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1469 command = "find ";
1470 command += Form("%s -o %d ",options.Data(), nstart);
1471 command += path;
780ec36d 1472 command += delimiter;
3af861ff 1473 command += pattern;
1474 command += conditions;
1475 TGridResult *res = gGrid->Command(command);
1476 if (res) delete res;
1477 // Write standard output to file
1478 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1479 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1480 Bool_t nullFile = kFALSE;
1481 if (!hasGrep) {
1482 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1483 } else {
1484 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1485 if (nullFile) {
1486 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1487 gSystem->Exec("rm -f __tmp*");
1604c9ed 1488 break;
3af861ff 1489 }
1490 TString line;
1491 ifstream in;
1492 in.open("__tmp__");
1493 in >> line;
1494 in.close();
1495 gSystem->Exec("rm -f __tmp__");
1496 ncount = line.Atoi();
1497 }
1498 nullResult = kFALSE;
1499 }
1500 if (ncount == gMaxEntries) {
1501 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1502 if (fNrunsPerMaster > 1) {
1503 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1504 file.Data(),gMaxEntries);
1505 return kFALSE;
1506 }
1507 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1508 if (!cbase) cbase = cadd;
1509 else {
1510 cbase->Add(cadd);
1511 delete cadd;
defd7a3a 1512 }
3af861ff 1513 nstart += ncount;
1514 } else {
c89a84bc 1515 if (cbase && fNrunsPerMaster<2) {
3af861ff 1516 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1517 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1518 cbase->Add(cadd);
1519 delete cadd;
1520 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1521 delete cbase; cbase = 0;
1522 } else {
1523 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1524 }
1525 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1526 break;
d3339be3 1527 }
319593fb 1528 }
a8739e8a 1529 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1530 // Check if there is one run per master job.
1531 if (fNrunsPerMaster<2) {
1532 if (FileExists(file)) {
0f389141 1533 if (fOverwriteMode) gGrid->Rm(file);
1534 else {
1535 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1536 continue;
1537 }
319593fb 1538 }
1539 // Copy xml file to alien space
1540 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1541 if (!FileExists(file)) {
1542 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1543 return kFALSE;
1544 }
1545 } else {
1546 nruns++;
95e5b448 1547 // Check if the collection for the chunk exist locally.
1548 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1549 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1550 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1551 else continue;
1552 }
84fcd93f 1553 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1554 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1555 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1556 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1557 } else {
1558 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1559 cbase->Add(cadd);
1560 delete cadd;
1561 }
74013535 1562 format = Form("%%s_%s.xml", fRunPrefix.Data());
1563 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1564 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1565 continue;
1566 }
904f9f5f 1567 schunk = schunk2;
319593fb 1568 if (FileExists(schunk)) {
0f389141 1569 if (fOverwriteMode) gGrid->Rm(schunk);
1570 else {
1571 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1572 continue;
1573 }
319593fb 1574 }
84fcd93f 1575 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1576 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1577 if (FileExists(schunk)) {
0f389141 1578 if (fOverwriteMode) gGrid->Rm(schunk);
1579 else {
1580 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1581 continue;
1582 }
95e5b448 1583 }
319593fb 1584 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1585 if (!FileExists(schunk)) {
1586 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1587 return kFALSE;
1588 }
1589 }
c57f56b7 1590 }
a2f5fc01 1591 if (nullResult) {
d3339be3 1592 Error("CreateDataset", "No valid dataset corresponding to the query!");
1593 return kFALSE;
1594 }
a8739e8a 1595 }
c57f56b7 1596 return kTRUE;
1597}
1598
1599//______________________________________________________________________________
1600Bool_t AliAnalysisAlien::CreateJDL()
1601{
1602// Generate a JDL file according to current settings. The name of the file is
1603// specified by fJDLName.
ec5acdce 1604 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
c57f56b7 1605 Bool_t error = kFALSE;
1606 TObjArray *arr = 0;
1607 Bool_t copy = kTRUE;
d3b18c4c 1608 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1609 Bool_t generate = kTRUE;
1610 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1611 if (!Connect()) {
1612 Error("CreateJDL", "Alien connection required");
1613 return kFALSE;
1614 }
1615 // Check validity of alien workspace
d3b18c4c 1616 TString workdir;
1617 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1618 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1619 workdir += fGridWorkingDir;
1620 if (generate) {
1621 TObjString *os;
1622 if (!fInputFiles) {
1623 Error("CreateJDL()", "Define some input files for your analysis.");
1624 error = kTRUE;
1625 }
1626 // Compose list of input files
1627 // Check if output files were defined
1628 if (!fOutputFiles.Length()) {
1629 Error("CreateJDL", "You must define at least one output file");
1630 error = kTRUE;
1631 }
1632 // Check if an output directory was defined and valid
1633 if (!fGridOutputDir.Length()) {
1634 Error("CreateJDL", "You must define AliEn output directory");
1635 error = kTRUE;
1636 } else {
d3b18c4c 1637 if (!fProductionMode) {
1638 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1639 if (!DirectoryExists(fGridOutputDir)) {
1640 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1641 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1642 } else {
1643 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1644 // error = kTRUE;
1645 }
67a3b254 1646 } else {
1647 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1648 }
1649 gGrid->Cd(workdir);
1650 }
c57f56b7 1651 }
1652 // Exit if any error up to now
1653 if (error) return kFALSE;
1654 // Set JDL fields
0f389141 1655 if (!fUser.IsNull()) {
1656 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1657 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1658 }
1659 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1660 TString mergeExec = fExecutable;
1661 mergeExec.ReplaceAll(".sh", "_merge.sh");
1662 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1663 mergeExec.ReplaceAll(".sh", ".C");
1664 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1665 if (!fArguments.IsNull())
1666 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1667 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1668 else {
1669 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1670 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1671 }
b3e07543 1672
1f0d1ca2 1673 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1674 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1675 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1676 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1677
0f389141 1678 if (fMaxInitFailed > 0) {
c57f56b7 1679 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1680 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1681 }
1682 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1683 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1684 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1685 }
1686 if (!IsOneStageMerging()) {
f47d5cb4 1687 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1688 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1689 }
1690 if (fSplitMode.Length()) {
c57f56b7 1691 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1692 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1693 }
1694 fMergingJDL->SetValue("Split", "\"se\"");
1695 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1696 if (!fAliROOTVersion.IsNull()) {
1697 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1698 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1699 }
1700 if (!fROOTVersion.IsNull()) {
c57f56b7 1701 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1702 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1703 }
1704 if (!fAPIVersion.IsNull()) {
c57f56b7 1705 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1706 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1707 }
648174cf 1708 if (!fExternalPackages.IsNull()) {
1709 arr = fExternalPackages.Tokenize(" ");
1710 TIter next(arr);
1711 while ((os=(TObjString*)next())) {
1712 TString pkgname = os->GetString();
1713 Int_t index = pkgname.Index("::");
1714 TString pkgversion = pkgname(index+2, pkgname.Length());
1715 pkgname.Remove(index);
1716 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1717 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1718 }
1719 delete arr;
1720 }
0f389141 1721 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1722 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1723 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1724 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1725 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1726 TString analysisFile = fExecutable;
1727 analysisFile.ReplaceAll(".sh", ".root");
1728 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1729 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1730 if (fAdditionalLibs.Length()) {
1731 arr = fAdditionalLibs.Tokenize(" ");
1732 TIter next(arr);
1733 while ((os=(TObjString*)next())) {
1734 if (os->GetString().Contains(".so")) continue;
1735 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1736 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1737 }
1738 delete arr;
1739 }
4e5c5506 1740 if (fPackages) {
1741 TIter next(fPackages);
1742 TObject *obj;
0f389141 1743 while ((obj=next())) {
4e5c5506 1744 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1745 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1746 }
4e5c5506 1747 }
95e16f01 1748 const char *comment = "List of output files and archives";
c57f56b7 1749 if (fOutputArchive.Length()) {
e4d9a069 1750 TString outputArchive = fOutputArchive;
1751 if (!fRegisterExcludes.IsNull()) {
1752 arr = fRegisterExcludes.Tokenize(" ");
1753 TIter next1(arr);
1754 while ((os=(TObjString*)next1())) {
1755 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1756 outputArchive.ReplaceAll(os->GetString(),"");
1757 }
1758 delete arr;
1759 }
1760 arr = outputArchive.Tokenize(" ");
c57f56b7 1761 TIter next(arr);
0f389141 1762 Bool_t first = kTRUE;
0f389141 1763 while ((os=(TObjString*)next())) {
0f389141 1764 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1765 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
0f389141 1766 else
95e16f01 1767 fGridJDL->AddToSet("Output", os->GetString());
1768 if (first) fGridJDL->AddToSetDescription("Output", comment);
0f389141 1769 first = kFALSE;
1770 }
c57f56b7 1771 delete arr;
f790bc1b 1772 // Output archive for the merging jdl
f790bc1b 1773 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1774 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1775 // Add normal output files, extra files + terminate files
1776 TString files = GetListOfFiles("outextter");
37d21c01 1777 // Do not register files in fRegisterExcludes
1778 if (!fRegisterExcludes.IsNull()) {
1779 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1780 TIter next1(arr);
1781 while ((os=(TObjString*)next1())) {
1782 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1783 files.ReplaceAll(os->GetString(),"");
1784 }
66d71516 1785 delete arr;
0f389141 1786 }
f790bc1b 1787 files.ReplaceAll(".root", "*.root");
ec5acdce 1788
1789 if (mgr->IsCollectThroughput())
95e16f01 1790 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",files.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 1791 else
1792 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1793 } else {
95f84779 1794 TString files = fOutputArchive;
1795 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1796 outputArchive = files;
f790bc1b 1797 }
0f389141 1798 arr = outputArchive.Tokenize(" ");
705adb3e 1799 TIter next2(arr);
0f389141 1800 first = kTRUE;
705adb3e 1801 while ((os=(TObjString*)next2())) {
0cdf65a8 1802 TString currentfile = os->GetString();
0cdf65a8 1803 if (!currentfile.Contains("@") && fCloseSE.Length())
95e16f01 1804 fMergingJDL->AddToSet("Output", Form("%s@%s",currentfile.Data(), fCloseSE.Data()));
0f389141 1805 else
95e16f01 1806 fMergingJDL->AddToSet("Output", currentfile);
1807 if (first) fMergingJDL->AddToSetDescription("Output", comment);
0f389141 1808 first = kFALSE;
1809 }
1810 delete arr;
c57f56b7 1811 }
149d288c 1812 arr = fOutputFiles.Tokenize(",");
c57f56b7 1813 TIter next(arr);
0f389141 1814 Bool_t first = kTRUE;
43da816a 1815 while ((os=(TObjString*)next())) {
e1eaf596 1816 // Ignore ouputs in jdl that are also in outputarchive
1817 TString sout = os->GetString();
f790bc1b 1818 sout.ReplaceAll("*", "");
1819 sout.ReplaceAll(".root", "");
e1eaf596 1820 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1821 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1822 // Ignore fRegisterExcludes
1823 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1824 if (!first) comment = NULL;
43da816a 1825 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1826 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
43da816a 1827 else
95e16f01 1828 fGridJDL->AddToSet("Output", os->GetString());
1829 if (first) fGridJDL->AddToSetDescription("Output", comment);
f790bc1b 1830 if (fMergeExcludes.Contains(sout)) continue;
1831 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1832 fMergingJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
f790bc1b 1833 else
95e16f01 1834 fMergingJDL->AddToSet("Output", os->GetString());
1835 if (first) fMergingJDL->AddToSetDescription("Output", comment);
1836 first = kFALSE;
43da816a 1837 }
c57f56b7 1838 delete arr;
0f389141 1839 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1840 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1841 TString validationScript = fValidationScript;
0f389141 1842 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1843 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1844 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1845 if (fMasterResubmitThreshold) {
1846 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1847 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1848 }
d2a409b2 1849 // Write a jdl with 2 input parameters: collection name and output dir name.
1850 WriteJDL(copy);
c57f56b7 1851 }
1852 // Copy jdl to grid workspace
a8739e8a 1853 if (copy) {
b5e4aaa7 1854 // Check if an output directory was defined and valid
1855 if (!fGridOutputDir.Length()) {
1856 Error("CreateJDL", "You must define AliEn output directory");
1857 return kFALSE;
1858 } else {
1859 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1860 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1861 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1862 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1863 } else {
1864 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1865 return kFALSE;
1866 }
1867 }
1868 gGrid->Cd(workdir);
1869 }
648174cf 1870 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1871 TString mergeJDLName = fExecutable;
1872 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1873 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1874 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1875 if (fProductionMode) {
648174cf 1876 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1877 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1878 }
648174cf 1879 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1880 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1881 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1882 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1883 Fatal("","Terminating");
1884// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1885 if (fMergeViaJDL) {
1886 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1887// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1888 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1889 Fatal("","Terminating");
0f389141 1890 }
648174cf 1891 }
c57f56b7 1892 if (fAdditionalLibs.Length()) {
e7c71df0 1893 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1894 TObjString *os;
1895 TIter next(arr);
1896 while ((os=(TObjString*)next())) {
c57f56b7 1897 if (os->GetString().Contains(".so")) continue;
4e5c5506 1898 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1899 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1900// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1901 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1902 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1903 Fatal("","Terminating");
c57f56b7 1904 }
1905 delete arr;
1906 }
4e5c5506 1907 if (fPackages) {
1908 TIter next(fPackages);
1909 TObject *obj;
1910 while ((obj=next())) {
fdbbc7be 1911 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1912 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1913// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1914 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1915 Form("%s/%s", workdir.Data(), obj->GetName())))
1916 Fatal("","Terminating");
4e5c5506 1917 }
1918 }
c57f56b7 1919 }
1920 return kTRUE;
1921}
1922
1923//______________________________________________________________________________
d2a409b2 1924Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1925{
1926// Writes one or more JDL's corresponding to findex. If findex is negative,
1927// all run numbers are considered in one go (jdl). For non-negative indices
1928// they correspond to the indices in the array fInputFiles.
1929 if (!fInputFiles) return kFALSE;
d3b18c4c 1930 TObject *os;
1931 TString workdir;
1932 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1933 workdir += fGridWorkingDir;
f47d5cb4 1934 TString stageName = "$2";
1935 if (fProductionMode) stageName = "$4";
1936 if (!fMergeDirName.IsNull()) {
1937 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1938 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1939 } else {
1940 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1941 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1942 }
d3b18c4c 1943 if (fProductionMode) {
a8739e8a 1944 TIter next(fInputFiles);
b3e07543 1945 while ((os=next())) {
d3b18c4c 1946 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1947 }
56754bf6 1948 if (!fOutputToRunNo)
1949 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1950 else
1951 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1952 } else {
1953 if (!fRunNumbers.Length() && !fRunRange[0]) {
1954 // One jdl with no parameters in case input data is specified by name.
1955 TIter next(fInputFiles);
1956 while ((os=next()))
1957 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1958 if (!fOutputSingle.IsNull())
1959 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1960 else {
1961 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1962 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1963 }
1964 } else {
1965 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1966 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1967 if (!fOutputSingle.IsNull()) {
1968 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1969 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1970 } else {
1971 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1972 }
1973 }
a8739e8a 1974 }
1975
a8739e8a 1976 // Generate the JDL as a string
1977 TString sjdl = fGridJDL->Generate();
0f389141 1978 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1979 // Final merge jdl
1980 if (!fMergeDirName.IsNull()) {
1981 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1982 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1983 } else {
1984 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1985 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1986 }
b3e07543 1987 TString sjdl2 = fMergingJDL->Generate();
1988 Int_t index, index1;
95e16f01 1989 sjdl.ReplaceAll("\",\"", "\",\n \"");
a8739e8a 1990 sjdl.ReplaceAll("(member", "\n (member");
1991 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1992 sjdl.ReplaceAll("{", "{\n ");
1993 sjdl.ReplaceAll("};", "\n};");
1994 sjdl.ReplaceAll("{\n \n", "{\n");
1995 sjdl.ReplaceAll("\n\n", "\n");
1996 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 1997 sjdl1.ReplaceAll("\",\"", "\",\n \"");
0f389141 1998 sjdl1.ReplaceAll("(member", "\n (member");
1999 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2000 sjdl1.ReplaceAll("{", "{\n ");
2001 sjdl1.ReplaceAll("};", "\n};");
2002 sjdl1.ReplaceAll("{\n \n", "{\n");
2003 sjdl1.ReplaceAll("\n\n", "\n");
2004 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2005 sjdl2.ReplaceAll("\",\"", "\",\n \"");
b3e07543 2006 sjdl2.ReplaceAll("(member", "\n (member");
2007 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2008 sjdl2.ReplaceAll("{", "{\n ");
2009 sjdl2.ReplaceAll("};", "\n};");
2010 sjdl2.ReplaceAll("{\n \n", "{\n");
2011 sjdl2.ReplaceAll("\n\n", "\n");
2012 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 2013 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 2014 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 2015 index = sjdl.Index("JDLVariables");
2016 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 2017 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2018 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 2019 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 2020 index = fJobTag.Index(":");
2021 if (index < 0) index = fJobTag.Length();
2022 TString jobTag = fJobTag;
f47d5cb4 2023 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 2024 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 2025 if (fProductionMode) {
2026 sjdl1.Prepend("# Generated merging jdl (production mode) \
2027 \n# $1 = full alien path to output directory to be merged \
2028 \n# $2 = train number \
2029 \n# $3 = production (like LHC10b) \
2030 \n# $4 = merging stage \
2031 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2032 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2033 sjdl2.Prepend("# Generated merging jdl \
2034 \n# $1 = full alien path to output directory to be merged \
2035 \n# $2 = train number \
2036 \n# $3 = production (like LHC10b) \
2037 \n# $4 = merging stage \
2038 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2039 } else {
2040 sjdl1.Prepend("# Generated merging jdl \
2041 \n# $1 = full alien path to output directory to be merged \
2042 \n# $2 = merging stage \
03335324 2043 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2044 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2045 sjdl2.Prepend("# Generated merging jdl \
2046 \n# $1 = full alien path to output directory to be merged \
2047 \n# $2 = merging stage \
03335324 2048 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2049 }
0f389141 2050 index = sjdl1.Index("JDLVariables");
2051 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 2052 index = sjdl2.Index("JDLVariables");
2053 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 2054 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2055 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
2056 index = sjdl2.Index("Split =");
2057 if (index>=0) {
2058 index1 = sjdl2.Index("\n", index);
2059 sjdl2.Remove(index, index1-index+1);
2060 }
2061 index = sjdl2.Index("SplitMaxInputFileNumber");
2062 if (index>=0) {
2063 index1 = sjdl2.Index("\n", index);
2064 sjdl2.Remove(index, index1-index+1);
2065 }
2066 index = sjdl2.Index("InputDataCollection");
2067 if (index>=0) {
2068 index1 = sjdl2.Index(";", index);
2069 sjdl2.Remove(index, index1-index+1);
2070 }
2071 index = sjdl2.Index("InputDataListFormat");
2072 if (index>=0) {
2073 index1 = sjdl2.Index("\n", index);
2074 sjdl2.Remove(index, index1-index+1);
2075 }
2076 index = sjdl2.Index("InputDataList");
2077 if (index>=0) {
2078 index1 = sjdl2.Index("\n", index);
2079 sjdl2.Remove(index, index1-index+1);
2080 }
f47d5cb4 2081 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2082 // Write jdl to file
a8739e8a 2083 ofstream out;
d2a409b2 2084 out.open(fJDLName.Data(), ios::out);
a8739e8a 2085 if (out.bad()) {
d3b18c4c 2086 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2087 return kFALSE;
2088 }
2089 out << sjdl << endl;
b3e07543 2090 out.close();
0f389141 2091 TString mergeJDLName = fExecutable;
2092 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2093 if (fMergeViaJDL) {
2094 ofstream out1;
2095 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2096 if (out1.bad()) {
d3b18c4c 2097 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2098 return kFALSE;
2099 }
2100 out1 << sjdl1 << endl;
b3e07543 2101 out1.close();
2102 ofstream out2;
2103 TString finalJDL = mergeJDLName;
2104 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2105 out2.open(finalJDL.Data(), ios::out);
2106 if (out2.bad()) {
2107 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2108 return kFALSE;
2109 }
2110 out2 << sjdl2 << endl;
2111 out2.close();
0f389141 2112 }
a8739e8a 2113
2114 // Copy jdl to grid workspace
2115 if (!copy) {
d3b18c4c 2116 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2117 } else {
d2a409b2 2118 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2119 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2120 TString finalJDL = mergeJDLName;
2121 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2122 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2123 if (fProductionMode) {
b5fe9cba 2124 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2125 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2126 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2127 }
d2a409b2 2128 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2129 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2130 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2131 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2132// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2133 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2134 Fatal("","Terminating");
0f389141 2135 if (fMergeViaJDL) {
b3e07543 2136 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2137// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2138// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2139 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2140 Fatal("","Terminating");
2141 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2142 Fatal("","Terminating");
0f389141 2143 }
a8739e8a 2144 }
2145 return kTRUE;
2146}
2147
2148//______________________________________________________________________________
5513444a 2149Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2150{
2151// Returns true if file exists.
5513444a 2152 if (!gGrid) return kFALSE;
2ea55496 2153 TString slfn = lfn;
2154 slfn.ReplaceAll("alien://","");
2155 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2156 if (!res) return kFALSE;
2157 TMap *map = dynamic_cast<TMap*>(res->At(0));
2158 if (!map) {
2159 delete res;
2160 return kFALSE;
2161 }
2162 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2163 if (!objs || !objs->GetString().Length()) {
2164 delete res;
2165 return kFALSE;
2166 }
2167 delete res;
2168 return kTRUE;
2169}
2170
2171//______________________________________________________________________________
923e2ca5 2172Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2173{
2174// Returns true if directory exists. Can be also a path.
2175 if (!gGrid) return kFALSE;
2176 // Check if dirname is a path
2177 TString dirstripped = dirname;
2178 dirstripped = dirstripped.Strip();
2179 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2180 TString dir = gSystem->BaseName(dirstripped);
2181 dir += "/";
2182 TString path = gSystem->DirName(dirstripped);
2183 TGridResult *res = gGrid->Ls(path, "-F");
2184 if (!res) return kFALSE;
2185 TIter next(res);
2186 TMap *map;
2187 TObject *obj;
2188 while ((map=dynamic_cast<TMap*>(next()))) {
2189 obj = map->GetValue("name");
2190 if (!obj) break;
2191 if (dir == obj->GetName()) {
2192 delete res;
2193 return kTRUE;
2194 }
2195 }
2196 delete res;
2197 return kFALSE;
2198}
2199
2200//______________________________________________________________________________
a2f5fc01 2201void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2202{
2203// Check input data type.
a2f5fc01 2204 isCollection = kFALSE;
2205 isXml = kFALSE;
2206 useTags = kFALSE;
c57f56b7 2207 if (!gGrid) {
2208 Error("CheckDataType", "No connection to grid");
2209 return;
2210 }
a2f5fc01 2211 isCollection = IsCollection(lfn);
c57f56b7 2212 TString msg = "\n##### file: ";
2213 msg += lfn;
a2f5fc01 2214 if (isCollection) {
c57f56b7 2215 msg += " type: raw_collection;";
2216 // special treatment for collections
a2f5fc01 2217 isXml = kFALSE;
c57f56b7 2218 // check for tag files in the collection
2219 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2220 if (!res) {
2221 msg += " using_tags: No (unknown)";
23329835 2222 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2223 return;
2224 }
2225 const char* typeStr = res->GetKey(0, "origLFN");
2226 if (!typeStr || !strlen(typeStr)) {
2227 msg += " using_tags: No (unknown)";
23329835 2228 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2229 return;
2230 }
2231 TString file = typeStr;
a2f5fc01 2232 useTags = file.Contains(".tag");
2233 if (useTags) msg += " using_tags: Yes";
c57f56b7 2234 else msg += " using_tags: No";
23329835 2235 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2236 return;
2237 }
2238 TString slfn(lfn);
2239 slfn.ToLower();
a2f5fc01 2240 isXml = slfn.Contains(".xml");
2241 if (isXml) {
c57f56b7 2242 // Open xml collection and check if there are tag files inside
2243 msg += " type: xml_collection;";
2244 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2245 if (!coll) {
2246 msg += " using_tags: No (unknown)";
23329835 2247 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2248 return;
2249 }
2250 TMap *map = coll->Next();
2251 if (!map) {
2252 msg += " using_tags: No (unknown)";
23329835 2253 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2254 return;
2255 }
2256 map = (TMap*)map->GetValue("");
2257 TString file;
2258 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2259 useTags = file.Contains(".tag");
c57f56b7 2260 delete coll;
a2f5fc01 2261 if (useTags) msg += " using_tags: Yes";
c57f56b7 2262 else msg += " using_tags: No";
23329835 2263 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2264 return;
2265 }
a2f5fc01 2266 useTags = slfn.Contains(".tag");
c57f56b7 2267 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2268 else msg += " type: unknown file;";
a2f5fc01 2269 if (useTags) msg += " using_tags: Yes";
c57f56b7 2270 else msg += " using_tags: No";
23329835 2271 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2272}
2273
2274//______________________________________________________________________________
4e5c5506 2275void AliAnalysisAlien::EnablePackage(const char *package)
2276{
2277// Enables a par file supposed to exist in the current directory.
2278 TString pkg(package);
2279 pkg.ReplaceAll(".par", "");
2280 pkg += ".par";
2281 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2282 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2283 return;
2284 }
2285 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2286 Info("EnablePackage", "AliEn plugin will use .par packages");
2287 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2288 if (!fPackages) {
2289 fPackages = new TObjArray();
2290 fPackages->SetOwner();
2291 }
2292 fPackages->Add(new TObjString(pkg));
2293}
2294
2295//______________________________________________________________________________
3bdcb562 2296TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2297{
2298// Make a tree from files having the location specified in fFileForTestMode.
2299// Inspired from JF's CreateESDChain.
2300 if (fFileForTestMode.IsNull()) {
2301 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2302 return NULL;
2303 }
2304 if (gSystem->AccessPathName(fFileForTestMode)) {
2305 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2306 return NULL;
2307 }
2308 // Open the file
2309 ifstream in;
2310 in.open(fFileForTestMode);
2311 Int_t count = 0;
2312 // Read the input list of files and add them to the chain
b505f1f8 2313 TString line;
d6c60197 2314 TString streeName(treeName);
2315 if (IsUseMCchain()) streeName = "TE";
2316 TChain *chain = new TChain(streeName);
b505f1f8 2317 TChain *chainFriend = 0;
d6c60197 2318 if (!fFriendChainName.IsNull()) chainFriend = new TChain(streeName);
b505f1f8 2319 while (in.good())
2320 {
3bdcb562 2321 in >> line;
ffbe06b9 2322 if (line.IsNull() || line.BeginsWith("#")) continue;
3bdcb562 2323 if (count++ == fNtestFiles) break;
2324 TString esdFile(line);
2325 TFile *file = TFile::Open(esdFile);
b505f1f8 2326 if (file && !file->IsZombie()) {
2327 chain->Add(esdFile);
3bdcb562 2328 file->Close();
b505f1f8 2329 if (!fFriendChainName.IsNull()) {
4b5d9e21 2330 if (esdFile.Index("#") > -1)
2331 esdFile.Remove(esdFile.Index("#"));
2332 esdFile = gSystem->DirName(esdFile);
2333 esdFile += "/" + fFriendChainName;
b505f1f8 2334 file = TFile::Open(esdFile);
2335 if (file && !file->IsZombie()) {
2336 file->Close();
2337 chainFriend->Add(esdFile);
2338 } else {
2339 Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data());
2340 return 0;
2341 }
4b5d9e21 2342 }
3bdcb562 2343 } else {
2344 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2345 }
b505f1f8 2346 }
2347 in.close();
2348 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2349 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2350 delete chain;
b505f1f8 2351 delete chainFriend;
3bdcb562 2352 return NULL;
b505f1f8 2353 }
3bdcb562 2354// chain->ls();
b505f1f8 2355 if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend);
2356 return chain;
3bdcb562 2357}
2358
2359//______________________________________________________________________________
d2a409b2 2360const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2361{
2362// Get job status for all jobs with jobid>jobidstart.
2363 static char mstatus[20];
2364 mstatus[0] = '\0';
2365 nrunning = 0;
2366 nwaiting = 0;
2367 nerror = 0;
2368 ndone = 0;
2369 TGridJobStatusList *list = gGrid->Ps("");
2370 if (!list) return mstatus;
2371 Int_t nentries = list->GetSize();
2372 TGridJobStatus *status;
2373 Int_t pid;
2374 for (Int_t ijob=0; ijob<nentries; ijob++) {
2375 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2376 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2377 if (pid<jobidstart) continue;
2378 if (pid == lastid) {
8ddad121 2379 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2380 }
2381 switch (status->GetStatus()) {
2382 case TGridJobStatus::kWAITING:
2383 nwaiting++; break;
2384 case TGridJobStatus::kRUNNING:
2385 nrunning++; break;
2386 case TGridJobStatus::kABORTED:
2387 case TGridJobStatus::kFAIL:
2388 case TGridJobStatus::kUNKNOWN:
2389 nerror++; break;
2390 case TGridJobStatus::kDONE:
2391 ndone++;
2392 }
2393 }
2394 list->Delete();
2395 delete list;
2396 return mstatus;
2397}
2398
2399//______________________________________________________________________________
c57f56b7 2400Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2401{
2402// Returns true if file is a collection. Functionality duplicated from
2403// TAlien::Type() because we don't want to directly depend on TAlien.
2404 if (!gGrid) {
2405 Error("IsCollection", "No connection to grid");
2406 return kFALSE;
2407 }
2408 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2409 if (!res) return kFALSE;
2410 const char* typeStr = res->GetKey(0, "type");
2411 if (!typeStr || !strlen(typeStr)) return kFALSE;
2412 if (!strcmp(typeStr, "collection")) return kTRUE;
2413 delete res;
2414 return kFALSE;
2415}
2416
2417//______________________________________________________________________________
fe2d7fc2 2418Bool_t AliAnalysisAlien::IsSingleOutput() const
2419{
2420// Check if single-ouput option is on.
2421 return (!fOutputSingle.IsNull());
2422}
2423
2424//______________________________________________________________________________
16a4353c 2425void AliAnalysisAlien::Print(Option_t *) const
2426{
2427// Print current plugin settings.
84fcd93f 2428 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2429 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2430 if (mgr && mgr->IsProofMode()) {
2431 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2432 if (TestBit(AliAnalysisGrid::kTest))
2433 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2434 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2435 if (!fProofDataSet.IsNull())
2436 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2437 if (fProofReset==1)
2438 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2439 if (fProofReset>1)
2440 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
874de7ea 2441 if (!fROOTVersion.IsNull())
2442 printf("= ROOT version requested________________________ %s\n", fROOTVersion.Data());
d3b18c4c 2443 else
2444 printf("= ROOT version requested________________________ default\n");
2445 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2446 if (!fAliRootMode.IsNull())
2447 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2448 if (fNproofWorkers)
2449 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2450 if (fNproofWorkersPerSlave)
2451 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2452 if (TestSpecialBit(kClearPackages))
2453 printf("= ClearPackages requested...\n");
2454 if (fIncludePath.Data())
2455 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2456 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2457 if (fPackages && fPackages->GetEntries()) {
2458 TIter next(fPackages);
2459 TObject *obj;
2460 TString list;
2461 while ((obj=next())) list += obj->GetName();
2462 printf("= Par files to be used: ________________________ %s\n", list.Data());
2463 }
2464 if (TestSpecialBit(kProofConnectGrid))
2465 printf("= Requested PROOF connection to grid\n");
2466 return;
2467 }
e1c22e21 2468 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2469 if (fOverwriteMode) {
2470 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2471 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2472 }
348be253 2473 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
780ec36d 2474 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO:Print");
84fcd93f 2475 printf("= Production mode:______________________________ %d\n", fProductionMode);
2476 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2477 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2478 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2479 if (fUser.Length())
84fcd93f 2480 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2481 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2482 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
780ec36d 2483 TString basedatadir = fGridDataDir;
2484 TString pattern = fDataPattern;
2485 pattern.Strip();
2486 Int_t ind = pattern.Index(" ");
2487 if (ind>=0) {
2488 basedatadir += "/%run%/";
2489 basedatadir += pattern(0, ind);
2490 pattern = pattern(ind+1, pattern.Length());
2491 }
2492 printf("= Data base directory path requested: __________ %s\n", basedatadir.Data());
2493 printf("= Data search pattern: _________________________ %s\n", pattern.Data());
84fcd93f 2494 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2495 if (fRunNumbers.Length())
84fcd93f 2496 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2497 if (fRunRange[0])
74013535 2498 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2499 if (!fRunRange[0] && !fRunNumbers.Length()) {
2500 TIter next(fInputFiles);
2501 TObject *obj;
2502 TString list;
2503 while ((obj=next())) list += obj->GetName();
84fcd93f 2504 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2505 }
2506 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2507 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2508 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2509 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2510 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2511 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2512 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2513 printf("=====================================================================\n");
2514 printf("= Job price: ___________________________________ %d\n", fPrice);
2515 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2516 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2517 if (fMaxInitFailed>0)
84fcd93f 2518 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2519 if (fMasterResubmitThreshold>0)
84fcd93f 2520 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2521 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2522 if (fNrunsPerMaster>0)
84fcd93f 2523 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2524 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2525 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2526 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2527 if (fArguments.Length())
84fcd93f 2528 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2529 if (fExecutableArgs.Length())
2530 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2531 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2532 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2533 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2534 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2535 if (fDatasetName)
84fcd93f 2536 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2537 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2538 if (fIncludePath.Data())
84fcd93f 2539 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2540 if (fCloseSE.Length())
84fcd93f 2541 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2542 if (fFriendChainName.Length())
84fcd93f 2543 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2544 if (fPackages && fPackages->GetEntries()) {
16a4353c 2545 TIter next(fPackages);
2546 TObject *obj;
2547 TString list;
2548 while ((obj=next())) list += obj->GetName();
84fcd93f 2549 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2550 }
2551}
2552
2553//______________________________________________________________________________
c57f56b7 2554void AliAnalysisAlien::SetDefaults()
2555{
2556// Set default values for everything. What cannot be filled will be left empty.
2557 if (fGridJDL) delete fGridJDL;
2558 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2559 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2560 fPrice = 1;
2561 fTTL = 30000;
2562 fSplitMaxInputFileNumber = 100;
2563 fMaxInitFailed = 0;
2564 fMasterResubmitThreshold = 0;
bb885a9e 2565 fNtestFiles = 10;
149d288c 2566 fNreplicas = 2;
a8739e8a 2567 fRunRange[0] = 0;
2568 fRunRange[1] = 0;
8e3bd06a 2569 fRunPrefix = "%d";
319593fb 2570 fNrunsPerMaster = 1;
16a4353c 2571 fMaxMergeFiles = 100;
c57f56b7 2572 fRunNumbers = "";
2573 fExecutable = "analysis.sh";
5aa94dc3 2574 fExecutableCommand = "root -b -q -x";
c57f56b7 2575 fArguments = "";
631c0b05 2576 fExecutableArgs = "";
c57f56b7 2577 fAnalysisMacro = "myAnalysis.C";
2578 fAnalysisSource = "";
2579 fAdditionalLibs = "";
2580 fSplitMode = "se";
2581 fAPIVersion = "";
2582 fROOTVersion = "";
2583 fAliROOTVersion = "";
2584 fUser = ""; // Your alien user name
2585 fGridWorkingDir = "";
2586 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2587 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2588 fFriendChainName = "";
c57f56b7 2589 fGridOutputDir = "output";
b83f84f0 2590 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2591 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2592 fInputFormat = "xml-single";
2593 fJDLName = "analysis.jdl";
c6cb3634 2594 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2595 fMergeExcludes = "";
0f389141 2596 fMergeViaJDL = 0;
348be253 2597 SetUseCopy(kTRUE);
2598 SetCheckCopy(kTRUE);
149d288c 2599 SetDefaultOutputs(kTRUE);
e1c22e21 2600 fOverwriteMode = 1;
c57f56b7 2601}
2602
2603//______________________________________________________________________________
a41b2882 2604void AliAnalysisAlien::SetFriendChainName(const char *name, const char *libnames)
2605{
2606 // Set file name for the chain of friends and optionally additional libs to be loaded.
2607 // Libs should be separated by blancs.
2608 fFriendChainName = name;
2609 fFriendLibs = libnames;
1a3f255c 2610 if (fFriendLibs.Length()) {
2611 if(!fFriendLibs.Contains(".so"))
2612 Fatal("SetFriendChainName()", "You should provide explicit library names (with extension)");
2613 fFriendLibs.ReplaceAll(",", " ");
2614 fFriendLibs.Strip();
2615 fFriendLibs.ReplaceAll(" ", " ");
a41b2882 2616 }
2617}
2618
2619//______________________________________________________________________________
874de7ea 2620void AliAnalysisAlien::SetRootVersionForProof(const char *version)
2621{
2622// Obsolete method. Use SetROOTVersion instead
2623 Warning("SetRootVersionForProof", "Obsolete. Use SetROOTVersion instead");
95e16f01 2624 if (fROOTVersion.IsNull()) SetROOTVersion(version);
2625 else Error("SetRootVersionForProof", "ROOT version already set to %s", fROOTVersion.Data());
874de7ea 2626}
2627
2628//______________________________________________________________________________
b3e07543 2629Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2630{
b3e07543 2631// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2632 // First check if the result is already in the output directory.
2633 if (FileExists(Form("%s/%s",aliendir,filename))) {
2634 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2635 return kFALSE;
2636 }
b3e07543 2637 // Now check the last stage done.
2638 Int_t stage = 0;
2639 while (1) {
2640 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2641 stage++;
2642 }
2643 // Next stage of merging
2644 stage++;
2645 TString pattern = "*root_archive.zip";
2646 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2647 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2648 if (res) delete res;
2649 // Write standard output to file
2650 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2651 // Count the number of files inside
2652 ifstream ifile;
2653 ifile.open(Form("Stage_%d.xml",stage));
2654 if (!ifile.good()) {
2655 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2656 return kFALSE;
b3e07543 2657 }
2658 TString line;
2659 Int_t nfiles = 0;
2660 while (!ifile.eof()) {
2661 ifile >> line;
2662 if (line.Contains("/event")) nfiles++;
7c2cd90a 2663 }
b3e07543 2664 ifile.close();
2665 if (!nfiles) {
2666 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2667 return kFALSE;
2668 } else {
2669 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2670 }
b3e07543 2671 // Copy the file in the output directory
2672 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2673// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2674 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2675 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2676 // Check if this is the last stage to be done.
2677 Bool_t laststage = (nfiles<nperchunk);
2678 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
0ad3ea1f 2679 Int_t jobId = 0;
b3e07543 2680 if (laststage) {
2681 printf("### Submiting final merging stage %d\n", stage);
2682 TString finalJDL = jdl;
2683 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2684 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
0ad3ea1f 2685 jobId = SubmitSingleJob(query);
b3e07543 2686 } else {
2687 printf("### Submiting merging stage %d\n", stage);
03335324 2688 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
0ad3ea1f 2689 jobId = SubmitSingleJob(query);
7c2cd90a 2690 }
0ad3ea1f 2691 if (!jobId) return kFALSE;
2692
2693 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
2694 fGridJobIDs.Append(Form("%d", jobId));
2695 if (!fGridStages.IsNull()) fGridStages.Append(" ");
2696 fGridStages.Append(Form("%s_merge_stage%d",
2697 laststage ? "final" : "partial", stage));
2698
b3e07543 2699 return kTRUE;
2700}
7c2cd90a 2701
2702//______________________________________________________________________________
105631b5 2703AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2704{
2705// Loat the analysis manager from a file.
2706 TFile *file = TFile::Open(fname);
2707 if (!file) {
2708 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2709 return 0;
2710 }
2711 TIter nextkey(file->GetListOfKeys());
2712 AliAnalysisManager *mgr = 0;
2713 TKey *key;
2714 while ((key=(TKey*)nextkey())) {
2715 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2716 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2717 }
2718 if (!mgr)
2719 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2720 return mgr;
2721}
2722
2723//______________________________________________________________________________
7c2cd90a 2724Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2725{
2726// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2727 if (!gGrid) return 0;
2728 printf("=> %s ------> ",query);
2729 TGridResult *res = gGrid->Command(query);
2730 if (!res) return 0;
2731 TString jobId = res->GetKey(0,"jobId");
2732 delete res;
2733 if (jobId.IsNull()) {
2734 printf("submission failed. Reason:\n");
2735 gGrid->Stdout();
2736 gGrid->Stderr();
2737 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2738 return 0;
2739 }
0ad3ea1f 2740 Int_t ijobId = jobId.Atoi();
2741 printf(" Job id: '%s' (%d)\n", jobId.Data(), ijobId);
2742 return ijobId;
7c2cd90a 2743}
2744
2745//______________________________________________________________________________
ec5acdce 2746Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2747{
2748// Merges a collection of output files using concatenation.
d66aa2a9 2749 TString scoll(collection);
2750 if (!scoll.Contains(".xml")) return kFALSE;
ec5acdce 2751 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2752 if (!coll) {
2753 ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2754 return kFALSE;
2755 }
2756 // Iterate grid collection
2757 TString outtmp;
2758 Bool_t merged = kFALSE;
2759 Int_t ifile = 0;
2760 while (coll->Next()) {
2761 TString fname = gSystem->DirName(coll->GetTURL());
2762 fname += "/";
2763 fname += output;
2764 outtmp = Form("%d_%s", ifile, output);
2765 if (!TFile::Cp(fname, outtmp)) {
2766 ::Error("MergeInfo", "Could not copy %s", fname.Data());
2767 continue;
2768 }
2769 ifile++;
2770 if (ifile<2) {
2771 gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2772 continue;
2773 }
2774 gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2775 gSystem->Exec("cp tempmerged lastmerged");
2776 }
2777 if (ifile) {
2778 gSystem->Exec(Form("cp lastmerged %s", output));
2779 gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2780 merged = kTRUE;
2781 }
2782 return merged;
2783}
2784
2785//______________________________________________________________________________
b3e07543 2786Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2787{
b3e07543 2788// Merge given output files from basedir. Basedir can be an alien output directory
2789// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2790// files in a group (ignored for xml input). Merging can be done in stages:
2791// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2792// stage=1 : works with an xml of all root_archive.zip in the output directory
2793// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2794 TString outputFile = output;
0f389141 2795 TString command;
a2f5fc01 2796 TString outputChunk;
2797 TString previousChunk = "";
b3e07543 2798 TObjArray *listoffiles = new TObjArray();
2799// listoffiles->SetOwner();
a2f5fc01 2800 Int_t countChunk = 0;
2801 Int_t countZero = nmaxmerge;
0f389141 2802 Bool_t merged = kTRUE;
d66aa2a9 2803 Bool_t isGrid = kTRUE;
a2f5fc01 2804 Int_t index = outputFile.Index("@");
2805 if (index > 0) outputFile.Remove(index);
7c2cd90a 2806 TString inputFile = outputFile;
b3e07543 2807 TString sbasedir = basedir;
2808 if (sbasedir.Contains(".xml")) {
2809 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2810 nmaxmerge = 9999999;
2811 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2812 if (!coll) {
2813 ::Error("MergeOutput", "Input XML collection empty.");
2814 return kFALSE;
2815 }
2816 // Iterate grid collection
2817 while (coll->Next()) {
2818 TString fname = gSystem->DirName(coll->GetTURL());
2819 fname += "/";
2820 fname += inputFile;
2821 listoffiles->Add(new TNamed(fname.Data(),""));
2822 }
ffbe06b9 2823 } else if (sbasedir.Contains(".txt")) {
d66aa2a9 2824 // The file having the .txt extension is expected to contain a list of
2825 // folders where the output files will be looked. For alien folders,
2826 // the full folder LFN is expected (starting with alien://)
ffbe06b9 2827 // Assume lfn's on each line
2828 TString line;
2829 ifstream in;
2830 in.open(sbasedir);
d66aa2a9 2831 if (in.fail()) {
2832 ::Error("MergeOutput", "File %s cannot be opened. Merging stopped." ,sbasedir.Data());
2833 return kTRUE;
2834 }
ffbe06b9 2835 Int_t nfiles = 0;
2836 while (in.good()) {
2837 in >> line;
2838 if (line.IsNull() || line.BeginsWith("#")) continue;
d66aa2a9 2839 line.Strip();
2840 if (!line.Contains("alien:")) isGrid = kFALSE;
2841 line += "/";
2842 line += outputFile;
ffbe06b9 2843 nfiles++;
2844 listoffiles->Add(new TNamed(line.Data(),""));
2845 }
d66aa2a9 2846 in.close();
ffbe06b9 2847 if (!nfiles) {
2848 ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data());
2849 delete listoffiles;
2850 return kFALSE;
2851 }
b3e07543 2852 } else {
2853 command = Form("find %s/ *%s", basedir, inputFile.Data());
2854 printf("command: %s\n", command.Data());
2855 TGridResult *res = gGrid->Command(command);
2856 if (!res) {
2857 ::Error("MergeOutput","No result for the find command\n");
2858 delete listoffiles;
2859 return kFALSE;
2860 }
2861 TIter nextmap(res);
2862 TMap *map = 0;
2863 while ((map=(TMap*)nextmap())) {
2864 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2865 if (!objs || !objs->GetString().Length()) {
2866 // Nothing found - skip this output
2867 delete res;
2868 delete listoffiles;
2869 return kFALSE;
2870 }
2871 listoffiles->Add(new TNamed(objs->GetName(),""));
2872 }
2873 delete res;
2874 }
2875 if (!listoffiles->GetEntries()) {
7c2cd90a 2876 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2877 delete listoffiles;
0f389141 2878 return kFALSE;
2879 }
2880
2881 TFileMerger *fm = 0;
b3e07543 2882 TIter next0(listoffiles);
2883 TObjArray *listoffilestmp = new TObjArray();
2884 listoffilestmp->SetOwner();
2885 TObject *nextfile;
2886 TString snextfile;
2887 // Keep only the files at upper level
2888 Int_t countChar = 0;
2889 while ((nextfile=next0())) {
2890 snextfile = nextfile->GetName();
2891 Int_t crtCount = snextfile.CountChar('/');
2892 if (nextfile == listoffiles->First()) countChar = crtCount;
2893 if (crtCount < countChar) countChar = crtCount;
2894 }
2895 next0.Reset();
2896 while ((nextfile=next0())) {
2897 snextfile = nextfile->GetName();
2898 Int_t crtCount = snextfile.CountChar('/');
2899 if (crtCount > countChar) {
2900 delete nextfile;
2901 continue;
2902 }
2903 listoffilestmp->Add(nextfile);
2904 }
2905 delete listoffiles;
2906 listoffiles = listoffilestmp; // Now contains 'good' files
2907 listoffiles->Print();
2908 TIter next(listoffiles);
7c2cd90a 2909 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2910 outputChunk = outputFile;
2911 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2912 // Check for existent temporary merge files
e1c22e21 2913 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2914 // Preserve old merging functionality for stage 0.
2915 if (stage==0) {
2916 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2917 while (1) {
2918 // Skip as many input files as in a chunk
2ea55496 2919 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2920 nextfile = next();
2921 if (!nextfile) {
2ea55496 2922 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2923 delete listoffiles;
2ea55496 2924 return kFALSE;
2925 }
b3e07543 2926 snextfile = nextfile->GetName();
7c2cd90a 2927 }
2928 outputChunk = outputFile;
2929 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2930 countChunk++;
2931 if (gSystem->AccessPathName(outputChunk)) continue;
2932 // Merged file with chunks up to <countChunk> found
2933 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2934 previousChunk = outputChunk;
2935 break;
2936 }
2937 }
2938 countZero = nmaxmerge;
2939
b3e07543 2940 while ((nextfile=next())) {
2941 snextfile = nextfile->GetName();
2ea55496 2942 // Loop 'find' results and get next LFN
7c2cd90a 2943 if (countZero == nmaxmerge) {
2944 // First file in chunk - create file merger and add previous chunk if any.
d66aa2a9 2945 fm = new TFileMerger(isGrid);
7c2cd90a 2946 fm->SetFastMethod(kTRUE);
2947 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2948 outputChunk = outputFile;
2949 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2950 }
2951 // If last file found, put merged results in the output file
b3e07543 2952 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2953 // Add file to be merged and decrement chunk counter.
b3e07543 2954 fm->AddFile(snextfile);
7c2cd90a 2955 countZero--;
b3e07543 2956 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2957 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2958 // Nothing found - skip this output
2959 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2960 merged = kFALSE;
2961 break;
7c2cd90a 2962 }
2963 fm->OutputFile(outputChunk);
2964 // Merge the outputs, then go to next chunk
2965 if (!fm->Merge()) {
2966 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2967 merged = kFALSE;
2968 break;
7c2cd90a 2969 } else {
2970 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2971 gSystem->Unlink(previousChunk);
2972 }
b3e07543 2973 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2974 countChunk++;
2