Fix bug in building local list of valid files.
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
715da79c 32#include "TInterpreter.h"
c57f56b7 33#include "TFile.h"
830acc4c 34#include "TFileCollection.h"
3bdcb562 35#include "TChain.h"
c57f56b7 36#include "TObjString.h"
37#include "TObjArray.h"
77f34eae 38#include "TMacro.h"
c57f56b7 39#include "TGrid.h"
40#include "TGridResult.h"
41#include "TGridCollection.h"
42#include "TGridJDL.h"
d2a409b2 43#include "TGridJobStatusList.h"
44#include "TGridJobStatus.h"
c57f56b7 45#include "TFileMerger.h"
46#include "AliAnalysisManager.h"
77f34eae 47#include "AliAnalysisTaskCfg.h"
bb885a9e 48#include "AliVEventHandler.h"
49#include "AliAnalysisDataContainer.h"
65eb22f8 50#include "AliMultiInputEventHandler.h"
c57f56b7 51
c82bb898 52using std::ofstream;
53using std::ifstream;
54using std::ios;
55using std::endl;
c57f56b7 56ClassImp(AliAnalysisAlien)
6c395669 57#if 0
58;
59#endif
c57f56b7 60
6c395669 61namespace {
62 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
63 {
64 TString sl(Form("file:%s", loc));
65 TString sr(Form("alien://%s", rem));
66 Bool_t ret = TFile::Cp(sl, sr);
67 if (!ret) {
68 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
69 }
70 return ret;
71 }
72}
73
c57f56b7 74//______________________________________________________________________________
75AliAnalysisAlien::AliAnalysisAlien()
76 :AliAnalysisGrid(),
77 fGridJDL(NULL),
0f389141 78 fMergingJDL(NULL),
c57f56b7 79 fPrice(0),
80 fTTL(0),
81 fSplitMaxInputFileNumber(0),
82 fMaxInitFailed(0),
83 fMasterResubmitThreshold(0),
bb885a9e 84 fNtestFiles(0),
319593fb 85 fNrunsPerMaster(0),
16a4353c 86 fMaxMergeFiles(0),
b3e07543 87 fMaxMergeStages(0),
d2a409b2 88 fNsubmitted(0),
a3e84053 89 fProductionMode(0),
cd11251e 90 fOutputToRunNo(0),
0f389141 91 fMergeViaJDL(0),
92 fFastReadOption(0),
e1c22e21 93 fOverwriteMode(1),
149d288c 94 fNreplicas(2),
3bdcb562 95 fNproofWorkers(0),
27734f0e 96 fNproofWorkersPerSlave(0),
3bdcb562 97 fProofReset(0),
715da79c 98 fNMCevents(0),
99 fNMCjobs(0),
c57f56b7 100 fRunNumbers(),
101 fExecutable(),
0a1c1f7f 102 fExecutableCommand(),
c57f56b7 103 fArguments(),
631c0b05 104 fExecutableArgs(),
c57f56b7 105 fAnalysisMacro(),
106 fAnalysisSource(),
d3b18c4c 107 fValidationScript(),
d5c6455a 108 fAdditionalRootLibs(),
c57f56b7 109 fAdditionalLibs(),
715da79c 110 fGeneratorLibs(),
c57f56b7 111 fSplitMode(),
112 fAPIVersion(),
113 fROOTVersion(),
114 fAliROOTVersion(),
648174cf 115 fExternalPackages(),
c57f56b7 116 fUser(),
117 fGridWorkingDir(),
118 fGridDataDir(),
119 fDataPattern(),
120 fGridOutputDir(),
121 fOutputArchive(),
122 fOutputFiles(),
123 fInputFormat(),
e7c71df0 124 fDatasetName(),
c57f56b7 125 fJDLName(),
d3b18c4c 126 fTerminateFiles(),
bb885a9e 127 fMergeExcludes(),
37d21c01 128 fRegisterExcludes(),
f965131e 129 fIncludePath(),
bb885a9e 130 fCloseSE(),
0df6ccf2 131 fFriendChainName(),
c6cb3634 132 fJobTag(),
648174cf 133 fOutputSingle(),
5fce53f4 134 fRunPrefix(),
3bdcb562 135 fProofCluster(),
136 fProofDataSet(),
137 fFileForTestMode(),
3bdcb562 138 fAliRootMode(),
d88e4159 139 fProofProcessOpt(),
f47d5cb4 140 fMergeDirName(),
4e5c5506 141 fInputFiles(0),
28ce6c9c 142 fPackages(0),
77f34eae 143 fModules(0),
d88e4159 144 fProofParam(),
a41b2882 145 fDropToShell(true),
715da79c 146 fMCLoop(false),
a41b2882 147 fGridJobIDs(""),
148 fGridStages(""),
780ec36d 149 fFriendLibs(""),
150 fTreeName()
c57f56b7 151{
152// Dummy ctor.
153 SetDefaults();
154}
155
156//______________________________________________________________________________
157AliAnalysisAlien::AliAnalysisAlien(const char *name)
158 :AliAnalysisGrid(name),
159 fGridJDL(NULL),
0f389141 160 fMergingJDL(NULL),
c57f56b7 161 fPrice(0),
162 fTTL(0),
163 fSplitMaxInputFileNumber(0),
164 fMaxInitFailed(0),
165 fMasterResubmitThreshold(0),
bb885a9e 166 fNtestFiles(0),
319593fb 167 fNrunsPerMaster(0),
16a4353c 168 fMaxMergeFiles(0),
b3e07543 169 fMaxMergeStages(0),
d2a409b2 170 fNsubmitted(0),
a3e84053 171 fProductionMode(0),
cd11251e 172 fOutputToRunNo(0),
0f389141 173 fMergeViaJDL(0),
174 fFastReadOption(0),
e1c22e21 175 fOverwriteMode(1),
149d288c 176 fNreplicas(2),
3bdcb562 177 fNproofWorkers(0),
27734f0e 178 fNproofWorkersPerSlave(0),
3bdcb562 179 fProofReset(0),
715da79c 180 fNMCevents(0),
181 fNMCjobs(0),
c57f56b7 182 fRunNumbers(),
183 fExecutable(),
0a1c1f7f 184 fExecutableCommand(),
c57f56b7 185 fArguments(),
631c0b05 186 fExecutableArgs(),
c57f56b7 187 fAnalysisMacro(),
188 fAnalysisSource(),
d3b18c4c 189 fValidationScript(),
d5c6455a 190 fAdditionalRootLibs(),
c57f56b7 191 fAdditionalLibs(),
715da79c 192 fGeneratorLibs(),
c57f56b7 193 fSplitMode(),
194 fAPIVersion(),
195 fROOTVersion(),
196 fAliROOTVersion(),
648174cf 197 fExternalPackages(),
c57f56b7 198 fUser(),
199 fGridWorkingDir(),
200 fGridDataDir(),
201 fDataPattern(),
202 fGridOutputDir(),
203 fOutputArchive(),
204 fOutputFiles(),
205 fInputFormat(),
e7c71df0 206 fDatasetName(),
c57f56b7 207 fJDLName(),
d3b18c4c 208 fTerminateFiles(),
bb885a9e 209 fMergeExcludes(),
37d21c01 210 fRegisterExcludes(),
f965131e 211 fIncludePath(),
bb885a9e 212 fCloseSE(),
0df6ccf2 213 fFriendChainName(),
c6cb3634 214 fJobTag(),
648174cf 215 fOutputSingle(),
5fce53f4 216 fRunPrefix(),
3bdcb562 217 fProofCluster(),
218 fProofDataSet(),
219 fFileForTestMode(),
3bdcb562 220 fAliRootMode(),
d88e4159 221 fProofProcessOpt(),
f47d5cb4 222 fMergeDirName(),
4e5c5506 223 fInputFiles(0),
28ce6c9c 224 fPackages(0),
77f34eae 225 fModules(0),
d88e4159 226 fProofParam(),
0ad3ea1f 227 fDropToShell(true),
715da79c 228 fMCLoop(false),
0ad3ea1f 229 fGridJobIDs(""),
a41b2882 230 fGridStages(""),
780ec36d 231 fFriendLibs(""),
232 fTreeName()
c57f56b7 233{
234// Default ctor.
235 SetDefaults();
236}
237
238//______________________________________________________________________________
239AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
240 :AliAnalysisGrid(other),
241 fGridJDL(NULL),
0f389141 242 fMergingJDL(NULL),
c57f56b7 243 fPrice(other.fPrice),
244 fTTL(other.fTTL),
245 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
246 fMaxInitFailed(other.fMaxInitFailed),
247 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 248 fNtestFiles(other.fNtestFiles),
319593fb 249 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 250 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 251 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 252 fNsubmitted(other.fNsubmitted),
a3e84053 253 fProductionMode(other.fProductionMode),
cd11251e 254 fOutputToRunNo(other.fOutputToRunNo),
0f389141 255 fMergeViaJDL(other.fMergeViaJDL),
256 fFastReadOption(other.fFastReadOption),
257 fOverwriteMode(other.fOverwriteMode),
149d288c 258 fNreplicas(other.fNreplicas),
3bdcb562 259 fNproofWorkers(other.fNproofWorkers),
27734f0e 260 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 261 fProofReset(other.fProofReset),
715da79c 262 fNMCevents(other.fNMCevents),
263 fNMCjobs(other.fNMCjobs),
c57f56b7 264 fRunNumbers(other.fRunNumbers),
265 fExecutable(other.fExecutable),
0a1c1f7f 266 fExecutableCommand(other.fExecutableCommand),
c57f56b7 267 fArguments(other.fArguments),
631c0b05 268 fExecutableArgs(other.fExecutableArgs),
c57f56b7 269 fAnalysisMacro(other.fAnalysisMacro),
270 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 271 fValidationScript(other.fValidationScript),
d5c6455a 272 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 273 fAdditionalLibs(other.fAdditionalLibs),
715da79c 274 fGeneratorLibs(other.fGeneratorLibs),
c57f56b7 275 fSplitMode(other.fSplitMode),
276 fAPIVersion(other.fAPIVersion),
277 fROOTVersion(other.fROOTVersion),
278 fAliROOTVersion(other.fAliROOTVersion),
648174cf 279 fExternalPackages(other.fExternalPackages),
c57f56b7 280 fUser(other.fUser),
281 fGridWorkingDir(other.fGridWorkingDir),
282 fGridDataDir(other.fGridDataDir),
283 fDataPattern(other.fDataPattern),
284 fGridOutputDir(other.fGridOutputDir),
285 fOutputArchive(other.fOutputArchive),
286 fOutputFiles(other.fOutputFiles),
287 fInputFormat(other.fInputFormat),
e7c71df0 288 fDatasetName(other.fDatasetName),
c57f56b7 289 fJDLName(other.fJDLName),
d3b18c4c 290 fTerminateFiles(other.fTerminateFiles),
bb885a9e 291 fMergeExcludes(other.fMergeExcludes),
37d21c01 292 fRegisterExcludes(other.fRegisterExcludes),
f965131e 293 fIncludePath(other.fIncludePath),
bb885a9e 294 fCloseSE(other.fCloseSE),
0df6ccf2 295 fFriendChainName(other.fFriendChainName),
c6cb3634 296 fJobTag(other.fJobTag),
648174cf 297 fOutputSingle(other.fOutputSingle),
5fce53f4 298 fRunPrefix(other.fRunPrefix),
3bdcb562 299 fProofCluster(other.fProofCluster),
300 fProofDataSet(other.fProofDataSet),
301 fFileForTestMode(other.fFileForTestMode),
3bdcb562 302 fAliRootMode(other.fAliRootMode),
d88e4159 303 fProofProcessOpt(other.fProofProcessOpt),
f47d5cb4 304 fMergeDirName(other.fMergeDirName),
4e5c5506 305 fInputFiles(0),
28ce6c9c 306 fPackages(0),
77f34eae 307 fModules(0),
d88e4159 308 fProofParam(),
0ad3ea1f 309 fDropToShell(other.fDropToShell),
715da79c 310 fMCLoop(other.fMCLoop),
0ad3ea1f 311 fGridJobIDs(other.fGridJobIDs),
a41b2882 312 fGridStages(other.fGridStages),
780ec36d 313 fFriendLibs(other.fFriendLibs),
314 fTreeName(other.fTreeName)
c57f56b7 315{
316// Copy ctor.
317 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 318 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 319 fRunRange[0] = other.fRunRange[0];
320 fRunRange[1] = other.fRunRange[1];
c57f56b7 321 if (other.fInputFiles) {
322 fInputFiles = new TObjArray();
323 TIter next(other.fInputFiles);
324 TObject *obj;
325 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
326 fInputFiles->SetOwner();
327 }
4e5c5506 328 if (other.fPackages) {
329 fPackages = new TObjArray();
330 TIter next(other.fPackages);
331 TObject *obj;
332 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
333 fPackages->SetOwner();
334 }
77f34eae 335 if (other.fModules) {
336 fModules = new TObjArray();
337 fModules->SetOwner();
338 TIter next(other.fModules);
339 AliAnalysisTaskCfg *mod, *crt;
340 while ((crt=(AliAnalysisTaskCfg*)next())) {
341 mod = new AliAnalysisTaskCfg(*crt);
342 fModules->Add(mod);
343 }
344 }
c57f56b7 345}
346
347//______________________________________________________________________________
348AliAnalysisAlien::~AliAnalysisAlien()
349{
350// Destructor.
77f34eae 351 delete fGridJDL;
352 delete fMergingJDL;
353 delete fInputFiles;
354 delete fPackages;
355 delete fModules;
28ce6c9c 356 fProofParam.DeleteAll();
c57f56b7 357}
358
359//______________________________________________________________________________
360AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
361{
362// Assignment.
363 if (this != &other) {
364 AliAnalysisGrid::operator=(other);
365 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 366 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 367 fPrice = other.fPrice;
368 fTTL = other.fTTL;
369 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
370 fMaxInitFailed = other.fMaxInitFailed;
371 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 372 fNtestFiles = other.fNtestFiles;
a3e84053 373 fNrunsPerMaster = other.fNrunsPerMaster;
374 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 375 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 376 fNsubmitted = other.fNsubmitted;
377 fProductionMode = other.fProductionMode;
cd11251e 378 fOutputToRunNo = other.fOutputToRunNo;
0f389141 379 fMergeViaJDL = other.fMergeViaJDL;
380 fFastReadOption = other.fFastReadOption;
381 fOverwriteMode = other.fOverwriteMode;
149d288c 382 fNreplicas = other.fNreplicas;
3bdcb562 383 fNproofWorkers = other.fNproofWorkers;
27734f0e 384 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 385 fProofReset = other.fProofReset;
715da79c 386 fNMCevents = other.fNMCevents;
387 fNMCjobs = other.fNMCjobs;
c57f56b7 388 fRunNumbers = other.fRunNumbers;
389 fExecutable = other.fExecutable;
0a1c1f7f 390 fExecutableCommand = other.fExecutableCommand;
c57f56b7 391 fArguments = other.fArguments;
631c0b05 392 fExecutableArgs = other.fExecutableArgs;
c57f56b7 393 fAnalysisMacro = other.fAnalysisMacro;
394 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 395 fValidationScript = other.fValidationScript;
d5c6455a 396 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 397 fAdditionalLibs = other.fAdditionalLibs;
715da79c 398 fGeneratorLibs = other.fGeneratorLibs;
c57f56b7 399 fSplitMode = other.fSplitMode;
400 fAPIVersion = other.fAPIVersion;
401 fROOTVersion = other.fROOTVersion;
402 fAliROOTVersion = other.fAliROOTVersion;
648174cf 403 fExternalPackages = other.fExternalPackages;
c57f56b7 404 fUser = other.fUser;
405 fGridWorkingDir = other.fGridWorkingDir;
406 fGridDataDir = other.fGridDataDir;
407 fDataPattern = other.fDataPattern;
408 fGridOutputDir = other.fGridOutputDir;
409 fOutputArchive = other.fOutputArchive;
410 fOutputFiles = other.fOutputFiles;
411 fInputFormat = other.fInputFormat;
e7c71df0 412 fDatasetName = other.fDatasetName;
c57f56b7 413 fJDLName = other.fJDLName;
d3b18c4c 414 fTerminateFiles = other.fTerminateFiles;
bb885a9e 415 fMergeExcludes = other.fMergeExcludes;
37d21c01 416 fRegisterExcludes = other.fRegisterExcludes;
f965131e 417 fIncludePath = other.fIncludePath;
bb885a9e 418 fCloseSE = other.fCloseSE;
0df6ccf2 419 fFriendChainName = other.fFriendChainName;
c6cb3634 420 fJobTag = other.fJobTag;
648174cf 421 fOutputSingle = other.fOutputSingle;
5fce53f4 422 fRunPrefix = other.fRunPrefix;
3bdcb562 423 fProofCluster = other.fProofCluster;
424 fProofDataSet = other.fProofDataSet;
425 fFileForTestMode = other.fFileForTestMode;
3bdcb562 426 fAliRootMode = other.fAliRootMode;
d88e4159 427 fProofProcessOpt = other.fProofProcessOpt;
f47d5cb4 428 fMergeDirName = other.fMergeDirName;
d88e4159 429 fDropToShell = other.fDropToShell;
715da79c 430 fMCLoop = other.fMCLoop;
0ad3ea1f 431 fGridJobIDs = other.fGridJobIDs;
432 fGridStages = other.fGridStages;
a41b2882 433 fFriendLibs = other.fFriendLibs;
780ec36d 434 fTreeName = other.fTreeName;
c57f56b7 435 if (other.fInputFiles) {
436 fInputFiles = new TObjArray();
437 TIter next(other.fInputFiles);
438 TObject *obj;
439 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
440 fInputFiles->SetOwner();
441 }
4e5c5506 442 if (other.fPackages) {
443 fPackages = new TObjArray();
444 TIter next(other.fPackages);
445 TObject *obj;
446 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
447 fPackages->SetOwner();
448 }
77f34eae 449 if (other.fModules) {
450 fModules = new TObjArray();
451 fModules->SetOwner();
452 TIter next(other.fModules);
453 AliAnalysisTaskCfg *mod, *crt;
454 while ((crt=(AliAnalysisTaskCfg*)next())) {
455 mod = new AliAnalysisTaskCfg(*crt);
456 fModules->Add(mod);
457 }
458 }
c57f56b7 459 }
460 return *this;
461}
462
463//______________________________________________________________________________
37d21c01 464void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
465{
466// Add a single additional library to be loaded. Extension must be present.
467 TString lib(name);
468 if (!lib.Contains(".")) {
469 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
470 return;
471 }
472 if (fAdditionalLibs.Contains(name)) {
473 Warning("AddAdditionalLibrary", "Library %s already added.", name);
474 return;
475 }
476 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
477 fAdditionalLibs += lib;
478}
479
480//______________________________________________________________________________
77f34eae 481void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
482{
483// Adding a module. Checks if already existing. Becomes owned by this.
484 if (!module) return;
485 if (GetModule(module->GetName())) {
486 Error("AddModule", "A module having the same name %s already added", module->GetName());
487 return;
488 }
489 if (!fModules) {
490 fModules = new TObjArray();
491 fModules->SetOwner();
492 }
493 fModules->Add(module);
494}
495
496//______________________________________________________________________________
497void AliAnalysisAlien::AddModules(TObjArray *list)
498{
499// Adding a list of modules. Checks if already existing. Becomes owned by this.
500 TIter next(list);
501 AliAnalysisTaskCfg *module;
502 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
503}
504
505//______________________________________________________________________________
506Bool_t AliAnalysisAlien::CheckDependencies()
507{
508// Check if all dependencies are satisfied. Reorder modules if needed.
509 Int_t nmodules = GetNmodules();
510 if (!nmodules) {
511 Warning("CheckDependencies", "No modules added yet to check their dependencies");
512 return kTRUE;
513 }
514 AliAnalysisTaskCfg *mod = 0;
515 AliAnalysisTaskCfg *dep = 0;
516 TString depname;
517 Int_t i, j, k;
518 for (i=0; i<nmodules; i++) {
519 mod = (AliAnalysisTaskCfg*) fModules->At(i);
520 Int_t ndeps = mod->GetNdeps();
521 Int_t istart = i;
522 for (j=0; j<ndeps; j++) {
523 depname = mod->GetDependency(j);
524 dep = GetModule(depname);
525 if (!dep) {
526 Error("CheckDependencies","Dependency %s not added for module %s",
527 depname.Data(), mod->GetName());
528 return kFALSE;
529 }
530 if (dep->NeedsDependency(mod->GetName())) {
531 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
532 mod->GetName(), dep->GetName());
533 return kFALSE;
534 }
535 Int_t idep = fModules->IndexOf(dep);
536 // The dependency task must come first
537 if (idep>i) {
538 // Remove at idep and move all objects below up one slot
539 // down to index i included.
540 fModules->RemoveAt(idep);
3e40fd4c 541 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 542 fModules->AddAt(dep, i++);
543 }
544 //Redo from istart if dependencies were inserted
545 if (i>istart) i=istart-1;
546 }
547 }
548 return kTRUE;
549}
550
551//______________________________________________________________________________
3e40fd4c 552AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
553{
554// Create the analysis manager and optionally execute the macro in filename.
555 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
715da79c 556 if (mgr) {
557 mgr->SetMCLoop(fMCLoop);
558 return mgr;
559 }
3e40fd4c 560 mgr = new AliAnalysisManager(name);
561 mgr->SetGridHandler((AliAnalysisGrid*)this);
715da79c 562 mgr->SetMCLoop(fMCLoop);
3e40fd4c 563 if (strlen(filename)) {
564 TString line = gSystem->ExpandPathName(filename);
565 line.Prepend(".x ");
566 gROOT->ProcessLine(line.Data());
567 }
568 return mgr;
569}
570
571//______________________________________________________________________________
77f34eae 572Int_t AliAnalysisAlien::GetNmodules() const
573{
574// Get number of modules.
575 if (!fModules) return 0;
576 return fModules->GetEntries();
577}
578
579//______________________________________________________________________________
580AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
581{
582// Get a module by name.
583 if (!fModules) return 0;
584 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
585}
586
587//______________________________________________________________________________
588Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
589{
590// Load a given module.
591 if (mod->IsLoaded()) return kTRUE;
192ddca2 592 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
593 if (!mgr) {
594 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
595 return kFALSE;
596 }
77f34eae 597 Int_t ndeps = mod->GetNdeps();
598 TString depname;
599 for (Int_t j=0; j<ndeps; j++) {
600 depname = mod->GetDependency(j);
601 AliAnalysisTaskCfg *dep = GetModule(depname);
602 if (!dep) {
603 Error("LoadModule","Dependency %s not existing for module %s",
604 depname.Data(), mod->GetName());
605 return kFALSE;
606 }
607 if (!LoadModule(dep)) {
608 Error("LoadModule","Dependency %s for module %s could not be loaded",
609 depname.Data(), mod->GetName());
610 return kFALSE;
611 }
612 }
613 // Load libraries for the module
614 if (!mod->CheckLoadLibraries()) {
615 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
616 return kFALSE;
617 }
192ddca2 618 // Check if a custom file name was requested
619 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
620
621 // Check if a custom terminate file name was requested
622 if (strlen(mod->GetTerminateFileName())) {
623 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
624 fTerminateFiles += mod->GetTerminateFileName();
625 }
626
77f34eae 627 // Execute the macro
628 if (mod->ExecuteMacro()<0) {
629 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
630 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
631 return kFALSE;
632 }
633 // Configure dependencies
634 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
635 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
636 mod->GetConfigMacro()->GetTitle(), mod->GetName());
637 return kFALSE;
638 }
5e8c7c22 639 // Adjust extra libraries
640 Int_t nlibs = mod->GetNlibs();
641 TString lib;
642 for (Int_t i=0; i<nlibs; i++) {
643 lib = mod->GetLibrary(i);
5e8c7c22 644 lib = Form("lib%s.so", lib.Data());
c9d6bd12 645 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 646 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
647 fAdditionalLibs += lib;
648 }
77f34eae 649 return kTRUE;
650}
651
652//______________________________________________________________________________
139fe6ba 653Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
654{
655// Generate the full train.
656 fAdditionalLibs = "";
657 if (!LoadModules()) return kFALSE;
658 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
659 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 660 mgr->RunLocalInit();
139fe6ba 661 mgr->PrintStatus();
662 Int_t productionMode = fProductionMode;
663 SetProductionMode();
664 TString macro = fAnalysisMacro;
665 TString executable = fExecutable;
666 TString validation = fValidationScript;
667 TString execCommand = fExecutableCommand;
668 SetAnalysisMacro(Form("%s.C", name));
669 SetExecutable(Form("%s.sh", name));
a8e31fb0 670// SetExecutableCommand("aliroot -b -q ");
139fe6ba 671 SetValidationScript(Form("%s_validation.sh", name));
672 StartAnalysis();
673 SetProductionMode(productionMode);
674 fAnalysisMacro = macro;
675 fExecutable = executable;
676 fExecutableCommand = execCommand;
677 fValidationScript = validation;
678 return kTRUE;
679}
680
681//______________________________________________________________________________
21ca8e59 682Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 683{
684// Generate test macros for a single module or for the full train.
5e8c7c22 685 fAdditionalLibs = "";
77f34eae 686 if (strlen(modname)) {
21ca8e59 687 if (!CheckDependencies()) return kFALSE;
688 AliAnalysisTaskCfg *mod = GetModule(modname);
689 if (!mod) {
690 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
691 return kFALSE;
692 }
693 if (!LoadModule(mod)) return kFALSE;
a41b2882 694 if (!LoadFriendLibs()) return kFALSE;
21ca8e59 695 } else if (!LoadModules()) return kFALSE;
696 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
697 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 698 mgr->RunLocalInit();
21ca8e59 699 mgr->PrintStatus();
700 SetLocalTest(kTRUE);
701 Int_t productionMode = fProductionMode;
702 SetProductionMode();
703 TString macro = fAnalysisMacro;
704 TString executable = fExecutable;
705 TString validation = fValidationScript;
706 TString execCommand = fExecutableCommand;
707 SetAnalysisMacro(Form("%s.C", name));
708 SetExecutable(Form("%s.sh", name));
d66aa2a9 709 fOutputFiles = GetListOfFiles("outaod");
710 // Add extra files registered to the analysis manager
711 TString extra = GetListOfFiles("ext");
712 if (!extra.IsNull()) {
713 extra.ReplaceAll(".root", "*.root");
714 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
715 fOutputFiles += extra;
716 }
a8e31fb0 717// SetExecutableCommand("aliroot -b -q ");
21ca8e59 718 SetValidationScript(Form("%s_validation.sh", name));
719 WriteAnalysisFile();
720 WriteAnalysisMacro();
721 WriteExecutable();
722 WriteValidationScript();
ffbe06b9 723 WriteMergingMacro();
724 WriteMergeExecutable();
725 WriteValidationScript(kTRUE);
21ca8e59 726 SetLocalTest(kFALSE);
727 SetProductionMode(productionMode);
728 fAnalysisMacro = macro;
729 fExecutable = executable;
730 fExecutableCommand = execCommand;
731 fValidationScript = validation;
77f34eae 732 return kTRUE;
733}
734
735//______________________________________________________________________________
736Bool_t AliAnalysisAlien::LoadModules()
737{
738// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 739 fAdditionalLibs = "";
21ca8e59 740 Int_t nmodules = GetNmodules();
741 if (!nmodules) {
742 Warning("LoadModules", "No module to be loaded");
743 return kTRUE;
744 }
3e40fd4c 745 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
746 if (!mgr) {
747 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
748 return kFALSE;
749 }
77f34eae 750 if (!CheckDependencies()) return kFALSE;
21ca8e59 751 nmodules = GetNmodules();
77f34eae 752 AliAnalysisTaskCfg *mod;
21ca8e59 753 for (Int_t imod=0; imod<nmodules; imod++) {
754 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 755 if (!LoadModule(mod)) return kFALSE;
756 }
a41b2882 757 // Load additional friend libraries
758 return LoadFriendLibs();
77f34eae 759}
760
761//______________________________________________________________________________
a41b2882 762Bool_t AliAnalysisAlien::LoadFriendLibs() const
763{
764// Load libraries required for reading friends.
765 if (fFriendLibs.Length()) {
766 TObjArray *list = 0;
767 TString lib;
768 if (fFriendLibs.Contains(",")) list = fFriendLibs.Tokenize(",");
769 else list = fFriendLibs.Tokenize(" ");
770 for (Int_t ilib=0; ilib<list->GetEntriesFast(); ilib++) {
771 lib = list->At(ilib)->GetName();
772 lib.ReplaceAll(".so","");
2dffffd6 773 lib.ReplaceAll(".dylib","");
a41b2882 774 lib.ReplaceAll(" ","");
775 if (lib.BeginsWith("lib")) lib.Remove(0, 3);
776 lib.Prepend("lib");
777 Int_t loaded = strlen(gSystem->GetLibraries(lib,"",kFALSE));
778 if (!loaded) loaded = gSystem->Load(lib);
779 if (loaded < 0) {
780 Error("LoadModules", "Cannot load library for friends %s", lib.Data());
781 return kFALSE;
782 }
783 }
784 delete list;
785 }
786 return kTRUE;
787}
788
789//______________________________________________________________________________
74013535 790void AliAnalysisAlien::SetRunPrefix(const char *prefix)
791{
792// Set the run number format. Can be a prefix or a format like "%09d"
793 fRunPrefix = prefix;
794 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
795}
796
797//______________________________________________________________________________
f965131e 798void AliAnalysisAlien::AddIncludePath(const char *path)
799{
800// Add include path in the remote analysis macro.
801 TString p(path);
802 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
803 else fIncludePath += Form("-I%s ", path);
804}
805
806//______________________________________________________________________________
c57f56b7 807void AliAnalysisAlien::AddRunNumber(Int_t run)
808{
809// Add a run number to the list of runs to be processed.
810 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 811 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 812}
813
814//______________________________________________________________________________
98690895 815void AliAnalysisAlien::AddRunList(const char* runList)
816{
817// Add several runs into the list of runs; they are expected to be separated by a blank character.
818 TString sList = runList;
819 TObjArray *list = sList.Tokenize(" ");
820 Int_t n = list->GetEntries();
821 for (Int_t i = 0; i < n; i++) {
822 TObjString *os = (TObjString*)list->At(i);
823 AddRunNumber(os->GetString().Atoi());
824 }
825 delete list;
826}
827
828//______________________________________________________________________________
ee75cfc3 829void AliAnalysisAlien::AddRunNumber(const char* run)
830{
831// Add a run number to the list of runs to be processed.
7aa5d539 832 TString runs = run;
833 TObjString *os;
834 TObjArray *arr = runs.Tokenize(" ");
835 TIter next(arr);
836 TString prefix;
837 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
838 while ((os=(TObjString*)next())){
839 if (fRunNumbers.Length()) fRunNumbers += " ";
840 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
841 }
842 delete arr;
ee75cfc3 843}
844
845//______________________________________________________________________________
c57f56b7 846void AliAnalysisAlien::AddDataFile(const char *lfn)
847{
848// Adds a data file to the input to be analysed. The file should be a valid LFN
849// or point to an existing file in the alien workdir.
850 if (!fInputFiles) fInputFiles = new TObjArray();
851 fInputFiles->Add(new TObjString(lfn));
852}
648174cf 853
854//______________________________________________________________________________
855void AliAnalysisAlien::AddExternalPackage(const char *package)
856{
857// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
858 if (fExternalPackages) fExternalPackages += " ";
859 fExternalPackages += package;
860}
861
c57f56b7 862//______________________________________________________________________________
863Bool_t AliAnalysisAlien::Connect()
864{
865// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
866 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 867 if (fProductionMode) return kTRUE;
c57f56b7 868 if (!gGrid) {
869 Info("Connect", "Trying to connect to AliEn ...");
870 TGrid::Connect("alien://");
871 }
872 if (!gGrid || !gGrid->IsConnected()) {
873 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
874 return kFALSE;
875 }
876 fUser = gGrid->GetUser();
877 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
878 return kTRUE;
879}
880
881//______________________________________________________________________________
882void AliAnalysisAlien::CdWork()
883{
884// Check validity of alien workspace. Create directory if possible.
885 if (!Connect()) {
886 Error("CdWork", "Alien connection required");
887 return;
888 }
889 TString homedir = gGrid->GetHomeDirectory();
890 TString workdir = homedir + fGridWorkingDir;
923e2ca5 891 if (DirectoryExists(workdir)) {
892 gGrid->Cd(workdir);
893 return;
894 }
895 // Work directory not existing - create it
896 gGrid->Cd(homedir);
b93f8109 897 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 898 gGrid->Cd(fGridWorkingDir);
d3b18c4c 899 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 900 } else {
d3b18c4c 901 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 902 workdir.Data(), homedir.Data());
903 fGridWorkingDir = "";
904 }
c57f56b7 905}
906
907//______________________________________________________________________________
348be253 908Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
909{
910// Check if file copying is possible.
d3b18c4c 911 if (fProductionMode) return kTRUE;
b5ae4fe0 912 TString salienpath(alienpath);
913 if (salienpath.Contains(" ")) {
914 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
915 return kFALSE;
916 }
348be253 917 if (!Connect()) {
918 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
919 return kFALSE;
920 }
d3339be3 921 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
922 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 923 // Check if alien_CLOSE_SE is defined
924 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
925 if (!closeSE.IsNull()) {
926 Info("CheckFileCopy", "Your current close storage is pointing to: \
927 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
928 } else {
929 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
930 }
931 // Check if grid directory exists.
932 if (!DirectoryExists(alienpath)) {
933 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
934 return kFALSE;
935 }
b5ae4fe0 936 TString stest = "plugin_test_copy";
937 TFile f(stest, "RECREATE");
348be253 938 // User may not have write permissions to current directory
939 if (f.IsZombie()) {
940 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
941 gSystem->WorkingDirectory());
942 return kFALSE;
943 }
944 f.Close();
b5ae4fe0 945 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
946 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 947 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 948 \n# 1. Make sure you have write permissions there. If this is the case: \
949 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
950 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
951 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
952 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 953 gSystem->Unlink(stest.Data());
348be253 954 return kFALSE;
955 }
b5ae4fe0 956 gSystem->Unlink(stest.Data());
957 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 958 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 959 return kTRUE;
960}
961
962//______________________________________________________________________________
c57f56b7 963Bool_t AliAnalysisAlien::CheckInputData()
964{
965// Check validity of input data. If necessary, create xml files.
d3b18c4c 966 if (fProductionMode) return kTRUE;
a8739e8a 967 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 968 if (!fGridDataDir.Length()) {
969 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
970 return kFALSE;
971 }
9f020952 972 if (fMergeViaJDL) {
973 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
974 return kFALSE;
975 }
d2a409b2 976 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 977 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
978 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 979 return kTRUE;
c57f56b7 980 }
981 // Process declared files
a2f5fc01 982 Bool_t isCollection = kFALSE;
983 Bool_t isXml = kFALSE;
984 Bool_t useTags = kFALSE;
c57f56b7 985 Bool_t checked = kFALSE;
d3b18c4c 986 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 987 TString file;
988 TString workdir = gGrid->GetHomeDirectory();
989 workdir += fGridWorkingDir;
990 if (fInputFiles) {
991 TObjString *objstr;
992 TIter next(fInputFiles);
993 while ((objstr=(TObjString*)next())) {
994 file = workdir;
995 file += "/";
996 file += objstr->GetString();
997 // Store full lfn path
998 if (FileExists(file)) objstr->SetString(file);
999 else {
1000 file = objstr->GetName();
1001 if (!FileExists(objstr->GetName())) {
1002 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
1003 objstr->GetName(), workdir.Data());
1004 return kFALSE;
1005 }
1006 }
1007 Bool_t iscoll, isxml, usetags;
1008 CheckDataType(file, iscoll, isxml, usetags);
1009 if (!checked) {
1010 checked = kTRUE;
a2f5fc01 1011 isCollection = iscoll;
1012 isXml = isxml;
1013 useTags = usetags;
1014 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1015 } else {
a2f5fc01 1016 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 1017 Error("CheckInputData", "Some conflict was found in the types of inputs");
1018 return kFALSE;
1019 }
1020 }
1021 }
1022 }
1023 // Process requested run numbers
a8739e8a 1024 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 1025 // Check validity of alien data directory
1026 if (!fGridDataDir.Length()) {
1027 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
1028 return kFALSE;
1029 }
923e2ca5 1030 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 1031 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
1032 return kFALSE;
1033 }
a2f5fc01 1034 if (isCollection) {
c57f56b7 1035 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
1036 return kFALSE;
1037 }
1038
a2f5fc01 1039 if (checked && !isXml) {
c57f56b7 1040 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
1041 return kFALSE;
1042 }
1043 // Check validity of run number(s)
1044 TObjArray *arr;
1045 TObjString *os;
74013535 1046 TString format;
319593fb 1047 Int_t nruns = 0;
904f9f5f 1048 TString schunk, schunk2;
c57f56b7 1049 TString path;
1050 if (!checked) {
1051 checked = kTRUE;
a2f5fc01 1052 useTags = fDataPattern.Contains("tag");
1053 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1054 }
a2f5fc01 1055 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 1056 Error("CheckInputData", "Cannot mix input files using/not using tags");
1057 return kFALSE;
1058 }
1059 if (fRunNumbers.Length()) {
a8739e8a 1060 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 1061 arr = fRunNumbers.Tokenize(" ");
1062 TIter next(arr);
1063 while ((os=(TObjString*)next())) {
1064 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1065 if (!DirectoryExists(path)) {
1066 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 1067 continue;
c57f56b7 1068 }
1069 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
1070 TString msg = "\n##### file: ";
1071 msg += path;
1072 msg += " type: xml_collection;";
a2f5fc01 1073 if (useTags) msg += " using_tags: Yes";
c57f56b7 1074 else msg += " using_tags: No";
23329835 1075 Info("CheckDataType", "%s", msg.Data());
319593fb 1076 if (fNrunsPerMaster<2) {
d2a409b2 1077 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 1078 } else {
1079 nruns++;
1080 if (((nruns-1)%fNrunsPerMaster) == 0) {
1081 schunk = os->GetString();
1082 }
1083 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1084 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1085 AddDataFile(schunk);
319593fb 1086 }
c57f56b7 1087 }
1088 delete arr;
a8739e8a 1089 } else {
1090 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1091 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1092 format = Form("%%s/%s ", fRunPrefix.Data());
1093 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1094 if (!DirectoryExists(path)) {
a8739e8a 1095 continue;
1096 }
74013535 1097 format = Form("%%s/%s.xml", fRunPrefix.Data());
1098 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1099 TString msg = "\n##### file: ";
1100 msg += path;
1101 msg += " type: xml_collection;";
a2f5fc01 1102 if (useTags) msg += " using_tags: Yes";
a8739e8a 1103 else msg += " using_tags: No";
23329835 1104 Info("CheckDataType", "%s", msg.Data());
319593fb 1105 if (fNrunsPerMaster<2) {
74013535 1106 format = Form("%s.xml", fRunPrefix.Data());
1107 AddDataFile(Form(format.Data(),irun));
319593fb 1108 } else {
1109 nruns++;
1110 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1111 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1112 }
74013535 1113 format = Form("_%s.xml", fRunPrefix.Data());
1114 schunk2 = Form(format.Data(), irun);
319593fb 1115 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1116 schunk += schunk2;
d2a409b2 1117 AddDataFile(schunk);
319593fb 1118 }
a8739e8a 1119 }
904f9f5f 1120 if (!fInputFiles) {
1121 schunk += schunk2;
1122 AddDataFile(schunk);
1123 }
c57f56b7 1124 }
1125 return kTRUE;
1126}
1127
1128//______________________________________________________________________________
4a8b667c 1129Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1130{
1131// Copy data from the given grid directory according a pattern and make a local
1132// dataset.
2ba4c287 1133// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1134 if (!Connect()) {
1135 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
4a8b667c 1136 return 0;
21ca8e59 1137 }
1138 if (!DirectoryExists(griddir)) {
1139 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
4a8b667c 1140 return 0;
21ca8e59 1141 }
1142 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1143 printf("Running command: %s\n", command.Data());
1144 TGridResult *res = gGrid->Command(command);
1145 Int_t nfound = res->GetEntries();
1146 if (!nfound) {
1147 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
4a8b667c 1148 return 0;
21ca8e59 1149 }
1150 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1151
1152 // archives
1153 TObjArray* additionalArchives = 0;
1154 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1155 additionalArchives = TString(archivefile).Tokenize(";");
1156 archivefile = additionalArchives->At(0)->GetName();
1157 additionalArchives->RemoveAt(0);
1158 additionalArchives->Compress();
1159 }
1160
21ca8e59 1161 // Copy files locally
1162 ofstream out;
1163 out.open(output, ios::out);
1164 TMap *map;
1165 TString turl, dirname, filename, temp;
1166 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1167 gSystem->MakeDirectory(outputdir);
1168 gSystem->ChangeDirectory(outputdir);
4a8b667c 1169 Int_t ncopied = 0;
21ca8e59 1170 for (Int_t i=0; i<nfound; i++) {
1171 map = (TMap*)res->At(i);
1172 turl = map->GetValue("turl")->GetName();
1173 filename = gSystem->BaseName(turl.Data());
1174 dirname = gSystem->DirName(turl.Data());
1175 dirname = gSystem->BaseName(dirname.Data());
1176 gSystem->MakeDirectory(dirname);
43b807f6 1177
1178 TString source(turl);
1179 TString targetFileName(filename);
1180
1181 if (strlen(archivefile) > 0) {
4a8b667c 1182// TODO here the archive in which the file resides should be determined
1183// however whereis returns only a guid, and guid2lfn does not work
1184// Therefore we use the one provided as argument for now
1185 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1186 targetFileName = archivefile;
43b807f6 1187 }
1188 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
4a8b667c 1189 Bool_t success = kTRUE;
1190 if (additionalArchives) {
1191 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1192 TString target;
1193 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1194 gSystem->MakeDirectory(gSystem->DirName(target));
1195 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1196 }
1197 }
2ba4c287 1198
4a8b667c 1199 if (success) {
1200 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1201 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1202 ncopied++;
1203 }
21ca8e59 1204 }
1205 }
1206 gSystem->ChangeDirectory(cdir);
1207 delete res;
4a8b667c 1208 delete additionalArchives;
1209 return ncopied;
21ca8e59 1210}
1211
1212//______________________________________________________________________________
c57f56b7 1213Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1214{
1215// Create dataset for the grid data directory + run number.
3af861ff 1216 const Int_t gMaxEntries = 15000;
d3b18c4c 1217 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1218 if (!Connect()) {
1219 Error("CreateDataset", "Cannot create dataset with no grid connection");
1220 return kFALSE;
1221 }
1222
1223 // Cd workspace
d3b18c4c 1224 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1225 TString workdir = gGrid->GetHomeDirectory();
1226 workdir += fGridWorkingDir;
1227
1228 // Compose the 'find' command arguments
74013535 1229 TString format;
c57f56b7 1230 TString command;
780ec36d 1231 TString delimiter = pattern;
1232 delimiter.Strip();
1233 if (delimiter.Contains(" ")) delimiter = "";
1234 else delimiter = " ";
c57f56b7 1235 TString options = "-x collection ";
bb885a9e 1236 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1237 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1238 TString conditions = "";
3af861ff 1239 Int_t nstart = 0;
1240 Int_t ncount = 0;
1241 Int_t stage = 0;
c57f56b7 1242 TString file;
1243 TString path;
319593fb 1244 Int_t nruns = 0;
904f9f5f 1245 TString schunk, schunk2;
ab254fd1 1246 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1247 if (!fRunNumbers.Length() && !fRunRange[0]) {
1248 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1249 // Make a single data collection from data directory.
1250 path = fGridDataDir;
923e2ca5 1251 if (!DirectoryExists(path)) {
d2a409b2 1252 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1253 return kFALSE;
780ec36d 1254 }
923e2ca5 1255// CdWork();
d2a409b2 1256 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1257 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1258 while (1) {
1259 ncount = 0;
1260 stage++;
1261 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1262 command = "find ";
1263 command += Form("%s -o %d ",options.Data(), nstart);
1264 command += path;
780ec36d 1265 command += delimiter;
3af861ff 1266 command += pattern;
1267 command += conditions;
1268 printf("command: %s\n", command.Data());
1269 TGridResult *res = gGrid->Command(command);
1270 if (res) delete res;
1271 // Write standard output to file
1272 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1273 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1274 Bool_t nullFile = kFALSE;
1275 if (!hasGrep) {
1276 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1277 } else {
1278 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1279 if (nullFile) {
1280 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1281 gSystem->Exec("rm -f __tmp*");
1282 return kFALSE;
1283 }
1284 TString line;
1285 ifstream in;
1286 in.open("__tmp__");
1287 in >> line;
1288 in.close();
1289 gSystem->Exec("rm -f __tmp__");
1290 ncount = line.Atoi();
1291 }
1292 }
1293 if (ncount == gMaxEntries) {
1294 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1295 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1296 if (!cbase) cbase = cadd;
1297 else {
1298 cbase->Add(cadd);
1299 delete cadd;
defd7a3a 1300 }
3af861ff 1301 nstart += ncount;
1302 } else {
1303 if (cbase) {
1304 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1305 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1306 cbase->Add(cadd);
1307 delete cadd;
1308 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1309 delete cbase; cbase = 0;
1310 } else {
1311 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1312 }
1313 gSystem->Exec("rm -f __tmp*");
1314 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1315 break;
1316 }
0f389141 1317 }
1318 Bool_t fileExists = FileExists(file);
1319 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1320 // Copy xml file to alien space
0f389141 1321 if (fileExists) gGrid->Rm(file);
d2a409b2 1322 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1323 if (!FileExists(file)) {
1324 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1325 return kFALSE;
1326 }
1327 // Update list of files to be processed.
1328 }
1329 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1330 return kTRUE;
1331 }
c57f56b7 1332 // Several runs
a2f5fc01 1333 Bool_t nullResult = kTRUE;
a8739e8a 1334 if (fRunNumbers.Length()) {
1335 TObjArray *arr = fRunNumbers.Tokenize(" ");
1336 TObjString *os;
1337 TIter next(arr);
1338 while ((os=(TObjString*)next())) {
3af861ff 1339 nstart = 0;
1340 stage = 0;
0dab9d6d 1341 path = Form("%s/%s", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1342 if (!DirectoryExists(path)) continue;
1343// CdWork();
a8739e8a 1344 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1345 else file = Form("%s.xml", os->GetString().Data());
319593fb 1346 // If local collection file does not exist, create it via 'find' command.
3af861ff 1347 while (1) {
1348 ncount = 0;
1349 stage++;
1350 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1351 command = "find ";
1352 command += Form("%s -o %d ",options.Data(), nstart);
1353 command += path;
780ec36d 1354 command += delimiter;
3af861ff 1355 command += pattern;
1356 command += conditions;
1357 TGridResult *res = gGrid->Command(command);
1358 if (res) delete res;
1359 // Write standard output to file
1360 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1361 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1362 Bool_t nullFile = kFALSE;
1363 if (!hasGrep) {
1364 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1365 } else {
1366 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1367 if (nullFile) {
1368 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1369 gSystem->Exec("rm -f __tmp*");
1370 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1371 break;
3af861ff 1372 }
1373 TString line;
1374 ifstream in;
1375 in.open("__tmp__");
1376 in >> line;
1377 in.close();
1378 gSystem->Exec("rm -f __tmp__");
1379 ncount = line.Atoi();
1380 }
1381 nullResult = kFALSE;
1382 }
1383 if (ncount == gMaxEntries) {
1384 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1385 if (fNrunsPerMaster > 1) {
1386 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1387 file.Data(),gMaxEntries);
1388 return kFALSE;
1389 }
1390 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1391 if (!cbase) cbase = cadd;
1392 else {
1393 cbase->Add(cadd);
1394 delete cadd;
defd7a3a 1395 }
3af861ff 1396 nstart += ncount;
1397 } else {
c89a84bc 1398 if (cbase && fNrunsPerMaster<2) {
3af861ff 1399 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1400 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1401 cbase->Add(cadd);
1402 delete cadd;
1403 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1404 delete cbase; cbase = 0;
1405 } else {
1406 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1407 }
1408 gSystem->Exec("rm -f __tmp*");
1409 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1410 break;
d3339be3 1411 }
3af861ff 1412 }
a8739e8a 1413 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1414 // Check if there is one run per master job.
1415 if (fNrunsPerMaster<2) {
1416 if (FileExists(file)) {
0f389141 1417 if (fOverwriteMode) gGrid->Rm(file);
1418 else {
1419 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1420 continue;
1421 }
319593fb 1422 }
1423 // Copy xml file to alien space
1424 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1425 if (!FileExists(file)) {
1426 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1427 delete arr;
1428 return kFALSE;
1429 }
1430 } else {
1431 nruns++;
1432 if (((nruns-1)%fNrunsPerMaster) == 0) {
1433 schunk = os->GetString();
1434 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1435 } else {
1436 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1437 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1438 cbase->Add(cadd);
1439 delete cadd;
1440 }
1441 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1442 continue;
1443 }
1444 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1445 if (FileExists(schunk)) {
1446 if (fOverwriteMode) gGrid->Rm(file);
1447 else {
1448 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1449 continue;
1450 }
319593fb 1451 }
84fcd93f 1452 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1453 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1454 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1455 if (!FileExists(schunk)) {
1456 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1457 delete arr;
1458 return kFALSE;
1459 }
d3339be3 1460 }
a8739e8a 1461 }
1462 delete arr;
a2f5fc01 1463 if (nullResult) {
d3339be3 1464 Error("CreateDataset", "No valid dataset corresponding to the query!");
1465 return kFALSE;
1466 }
a8739e8a 1467 } else {
1468 // Process a full run range.
1469 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
0dab9d6d 1470 format = Form("%%s/%s", fRunPrefix.Data());
3af861ff 1471 nstart = 0;
1472 stage = 0;
74013535 1473 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1474 if (!DirectoryExists(path)) continue;
1475// CdWork();
74013535 1476 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1477 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1478 else file = Form(format.Data(), irun);
0f389141 1479 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1480 if (fOverwriteMode) gGrid->Rm(file);
1481 else {
1482 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1483 continue;
1484 }
a8739e8a 1485 }
319593fb 1486 // If local collection file does not exist, create it via 'find' command.
3af861ff 1487 while (1) {
1488 ncount = 0;
1489 stage++;
1490 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1491 command = "find ";
1492 command += Form("%s -o %d ",options.Data(), nstart);
1493 command += path;
780ec36d 1494 command += delimiter;
3af861ff 1495 command += pattern;
1496 command += conditions;
1497 TGridResult *res = gGrid->Command(command);
1498 if (res) delete res;
1499 // Write standard output to file
1500 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1501 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1502 Bool_t nullFile = kFALSE;
1503 if (!hasGrep) {
1504 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1505 } else {
1506 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1507 if (nullFile) {
1508 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1509 gSystem->Exec("rm -f __tmp*");
1604c9ed 1510 break;
3af861ff 1511 }
1512 TString line;
1513 ifstream in;
1514 in.open("__tmp__");
1515 in >> line;
1516 in.close();
1517 gSystem->Exec("rm -f __tmp__");
1518 ncount = line.Atoi();
1519 }
1520 nullResult = kFALSE;
1521 }
1522 if (ncount == gMaxEntries) {
1523 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1524 if (fNrunsPerMaster > 1) {
1525 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1526 file.Data(),gMaxEntries);
1527 return kFALSE;
1528 }
1529 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1530 if (!cbase) cbase = cadd;
1531 else {
1532 cbase->Add(cadd);
1533 delete cadd;
defd7a3a 1534 }
3af861ff 1535 nstart += ncount;
1536 } else {
c89a84bc 1537 if (cbase && fNrunsPerMaster<2) {
3af861ff 1538 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1539 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1540 cbase->Add(cadd);
1541 delete cadd;
1542 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1543 delete cbase; cbase = 0;
1544 } else {
1545 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1546 }
1547 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1548 break;
d3339be3 1549 }
319593fb 1550 }
a8739e8a 1551 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1552 // Check if there is one run per master job.
1553 if (fNrunsPerMaster<2) {
1554 if (FileExists(file)) {
0f389141 1555 if (fOverwriteMode) gGrid->Rm(file);
1556 else {
1557 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1558 continue;
1559 }
319593fb 1560 }
1561 // Copy xml file to alien space
1562 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1563 if (!FileExists(file)) {
1564 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1565 return kFALSE;
1566 }
1567 } else {
1568 nruns++;
95e5b448 1569 // Check if the collection for the chunk exist locally.
1570 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1571 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1572 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1573 else continue;
1574 }
84fcd93f 1575 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1576 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1577 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1578 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1579 } else {
1580 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1581 cbase->Add(cadd);
1582 delete cadd;
1583 }
74013535 1584 format = Form("%%s_%s.xml", fRunPrefix.Data());
1585 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1586 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1587 continue;
1588 }
904f9f5f 1589 schunk = schunk2;
319593fb 1590 if (FileExists(schunk)) {
0f389141 1591 if (fOverwriteMode) gGrid->Rm(schunk);
1592 else {
1593 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1594 continue;
1595 }
319593fb 1596 }
84fcd93f 1597 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1598 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1599 if (FileExists(schunk)) {
0f389141 1600 if (fOverwriteMode) gGrid->Rm(schunk);
1601 else {
1602 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1603 continue;
1604 }
95e5b448 1605 }
319593fb 1606 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1607 if (!FileExists(schunk)) {
1608 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1609 return kFALSE;
1610 }
1611 }
c57f56b7 1612 }
a2f5fc01 1613 if (nullResult) {
d3339be3 1614 Error("CreateDataset", "No valid dataset corresponding to the query!");
1615 return kFALSE;
1616 }
a8739e8a 1617 }
c57f56b7 1618 return kTRUE;
1619}
1620
1621//______________________________________________________________________________
1622Bool_t AliAnalysisAlien::CreateJDL()
1623{
1624// Generate a JDL file according to current settings. The name of the file is
1625// specified by fJDLName.
ec5acdce 1626 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
c57f56b7 1627 Bool_t error = kFALSE;
1628 TObjArray *arr = 0;
1629 Bool_t copy = kTRUE;
d3b18c4c 1630 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1631 Bool_t generate = kTRUE;
1632 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1633 if (!Connect()) {
1634 Error("CreateJDL", "Alien connection required");
1635 return kFALSE;
1636 }
1637 // Check validity of alien workspace
d3b18c4c 1638 TString workdir;
1639 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1640 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1641 workdir += fGridWorkingDir;
1642 if (generate) {
1643 TObjString *os;
1e466628 1644 if (!fInputFiles && !fMCLoop) {
c57f56b7 1645 Error("CreateJDL()", "Define some input files for your analysis.");
1646 error = kTRUE;
1647 }
1648 // Compose list of input files
1649 // Check if output files were defined
1650 if (!fOutputFiles.Length()) {
1651 Error("CreateJDL", "You must define at least one output file");
1652 error = kTRUE;
1653 }
1654 // Check if an output directory was defined and valid
1655 if (!fGridOutputDir.Length()) {
1656 Error("CreateJDL", "You must define AliEn output directory");
1657 error = kTRUE;
1658 } else {
d3b18c4c 1659 if (!fProductionMode) {
1660 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1661 if (!DirectoryExists(fGridOutputDir)) {
1662 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1663 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1664 } else {
1665 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1666 // error = kTRUE;
1667 }
67a3b254 1668 } else {
1669 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1670 }
1671 gGrid->Cd(workdir);
1672 }
c57f56b7 1673 }
1674 // Exit if any error up to now
1675 if (error) return kFALSE;
1676 // Set JDL fields
0f389141 1677 if (!fUser.IsNull()) {
1678 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1679 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
6da4839e
AG
1680 }
1681 TString executable = fExecutable;
1682 if (!executable.BeginsWith("/"))
1683 executable.Prepend(Form("%s/", workdir.Data()));
1684 fGridJDL->SetExecutable(executable, "This is the startup script");
1685 TString mergeExec = executable;
0f389141 1686 mergeExec.ReplaceAll(".sh", "_merge.sh");
1687 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1688 mergeExec.ReplaceAll(".sh", ".C");
9c5cde18 1689 fMergingJDL->AddToInputSandbox(Form("LF:%s", mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1690 if (!fArguments.IsNull())
1691 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1692 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1693 else {
1694 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1695 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1696 }
b3e07543 1697
1f0d1ca2 1698 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1699 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1700 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1701 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1702
0f389141 1703 if (fMaxInitFailed > 0) {
c57f56b7 1704 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1705 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1706 }
6da4839e 1707 if (fSplitMaxInputFileNumber > 0 && !fMCLoop) {
c57f56b7 1708 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1709 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1710 }
d7eb1f39 1711 if (!IsOneStageMerging()) {
f47d5cb4 1712 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1713 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1714 }
1715 if (fSplitMode.Length()) {
c57f56b7 1716 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1717 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1718 }
1719 fMergingJDL->SetValue("Split", "\"se\"");
1720 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1721 if (!fAliROOTVersion.IsNull()) {
1722 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1723 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1724 }
1725 if (!fROOTVersion.IsNull()) {
c57f56b7 1726 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1727 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1728 }
1729 if (!fAPIVersion.IsNull()) {
c57f56b7 1730 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1731 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1732 }
648174cf 1733 if (!fExternalPackages.IsNull()) {
1734 arr = fExternalPackages.Tokenize(" ");
1735 TIter next(arr);
1736 while ((os=(TObjString*)next())) {
1737 TString pkgname = os->GetString();
1738 Int_t index = pkgname.Index("::");
1739 TString pkgversion = pkgname(index+2, pkgname.Length());
1740 pkgname.Remove(index);
1741 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1742 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1743 }
1744 delete arr;
1745 }
6da4839e
AG
1746 if (!fMCLoop) {
1747 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1748 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
6da4839e 1749 }
d7eb1f39
AG
1750 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1751 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1752 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1753 TString analysisFile = fExecutable;
1754 analysisFile.ReplaceAll(".sh", ".root");
1755 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1756 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1757 if (fAdditionalLibs.Length()) {
1758 arr = fAdditionalLibs.Tokenize(" ");
1759 TIter next(arr);
1760 while ((os=(TObjString*)next())) {
2dffffd6 1761 if (os->GetString().Contains(".so") ||
1762 os->GetString().Contains(".dylib")) continue;
c57f56b7 1763 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1764 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1765 }
1766 delete arr;
1767 }
4e5c5506 1768 if (fPackages) {
1769 TIter next(fPackages);
1770 TObject *obj;
0f389141 1771 while ((obj=next())) {
4e5c5506 1772 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1773 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1774 }
4e5c5506 1775 }
95e16f01 1776 const char *comment = "List of output files and archives";
c57f56b7 1777 if (fOutputArchive.Length()) {
e4d9a069 1778 TString outputArchive = fOutputArchive;
1779 if (!fRegisterExcludes.IsNull()) {
1780 arr = fRegisterExcludes.Tokenize(" ");
1781 TIter next1(arr);
1782 while ((os=(TObjString*)next1())) {
1783 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1784 outputArchive.ReplaceAll(os->GetString(),"");
1785 }
1786 delete arr;
1787 }
1788 arr = outputArchive.Tokenize(" ");
c57f56b7 1789 TIter next(arr);
0f389141 1790 Bool_t first = kTRUE;
0f389141 1791 while ((os=(TObjString*)next())) {
0f389141 1792 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1793 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
0f389141 1794 else
95e16f01 1795 fGridJDL->AddToSet("Output", os->GetString());
1796 if (first) fGridJDL->AddToSetDescription("Output", comment);
0f389141 1797 first = kFALSE;
1798 }
c57f56b7 1799 delete arr;
f790bc1b 1800 // Output archive for the merging jdl
f790bc1b 1801 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1802 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1803 // Add normal output files, extra files + terminate files
0c0825e4 1804 TString files;
1805 if (IsMergeAOD()) files = GetListOfFiles("outaodextter");
1806 else files = GetListOfFiles("outextter");
37d21c01 1807 // Do not register files in fRegisterExcludes
1808 if (!fRegisterExcludes.IsNull()) {
1809 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1810 TIter next1(arr);
1811 while ((os=(TObjString*)next1())) {
1812 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1813 files.ReplaceAll(os->GetString(),"");
1814 }
66d71516 1815 delete arr;
0f389141 1816 }
f790bc1b 1817 files.ReplaceAll(".root", "*.root");
ec5acdce 1818
1819 if (mgr->IsCollectThroughput())
95e16f01 1820 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",files.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 1821 else
1822 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1823 } else {
95f84779 1824 TString files = fOutputArchive;
1825 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1826 outputArchive = files;
f790bc1b 1827 }
0f389141 1828 arr = outputArchive.Tokenize(" ");
705adb3e 1829 TIter next2(arr);
0f389141 1830 first = kTRUE;
705adb3e 1831 while ((os=(TObjString*)next2())) {
0cdf65a8 1832 TString currentfile = os->GetString();
0cdf65a8 1833 if (!currentfile.Contains("@") && fCloseSE.Length())
95e16f01 1834 fMergingJDL->AddToSet("Output", Form("%s@%s",currentfile.Data(), fCloseSE.Data()));
0f389141 1835 else
95e16f01 1836 fMergingJDL->AddToSet("Output", currentfile);
1837 if (first) fMergingJDL->AddToSetDescription("Output", comment);
0f389141 1838 first = kFALSE;
1839 }
1840 delete arr;
c57f56b7 1841 }
149d288c 1842 arr = fOutputFiles.Tokenize(",");
c57f56b7 1843 TIter next(arr);
0f389141 1844 Bool_t first = kTRUE;
43da816a 1845 while ((os=(TObjString*)next())) {
e1eaf596 1846 // Ignore ouputs in jdl that are also in outputarchive
1847 TString sout = os->GetString();
f790bc1b 1848 sout.ReplaceAll("*", "");
1849 sout.ReplaceAll(".root", "");
e1eaf596 1850 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1851 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1852 // Ignore fRegisterExcludes
1853 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1854 if (!first) comment = NULL;
43da816a 1855 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1856 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
43da816a 1857 else
95e16f01 1858 fGridJDL->AddToSet("Output", os->GetString());
1859 if (first) fGridJDL->AddToSetDescription("Output", comment);
f790bc1b 1860 if (fMergeExcludes.Contains(sout)) continue;
1861 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1862 fMergingJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
f790bc1b 1863 else
95e16f01 1864 fMergingJDL->AddToSet("Output", os->GetString());
1865 if (first) fMergingJDL->AddToSetDescription("Output", comment);
1866 first = kFALSE;
43da816a 1867 }
c57f56b7 1868 delete arr;
0f389141 1869 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1870 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1871 TString validationScript = fValidationScript;
0f389141 1872 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1873 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1874 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1875 if (fMasterResubmitThreshold) {
1876 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1877 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1878 }
d2a409b2 1879 // Write a jdl with 2 input parameters: collection name and output dir name.
1880 WriteJDL(copy);
c57f56b7 1881 }
1882 // Copy jdl to grid workspace
a8739e8a 1883 if (copy) {
b5e4aaa7 1884 // Check if an output directory was defined and valid
1885 if (!fGridOutputDir.Length()) {
1886 Error("CreateJDL", "You must define AliEn output directory");
1887 return kFALSE;
1888 } else {
1889 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1890 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1891 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1892 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1893 } else {
1894 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1895 return kFALSE;
1896 }
1897 }
1898 gGrid->Cd(workdir);
1899 }
648174cf 1900 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1901 TString mergeJDLName = fExecutable;
1902 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1903 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1904 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1905 if (fProductionMode) {
648174cf 1906 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1907 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1908 }
648174cf 1909 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1910 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1911 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1912 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1913 Fatal("","Terminating");
1914// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1915 if (fMergeViaJDL) {
1916 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1917// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1918 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1919 Fatal("","Terminating");
0f389141 1920 }
648174cf 1921 }
c57f56b7 1922 if (fAdditionalLibs.Length()) {
e7c71df0 1923 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1924 TObjString *os;
1925 TIter next(arr);
1926 while ((os=(TObjString*)next())) {
2dffffd6 1927 if (os->GetString().Contains(".so") ||
1928 os->GetString().Contains(".dylib")) continue;
4e5c5506 1929 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1930 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1931// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1932 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1933 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1934 Fatal("","Terminating");
c57f56b7 1935 }
1936 delete arr;
1937 }
4e5c5506 1938 if (fPackages) {
1939 TIter next(fPackages);
1940 TObject *obj;
1941 while ((obj=next())) {
fdbbc7be 1942 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1943 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1944// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1945 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1946 Form("%s/%s", workdir.Data(), obj->GetName())))
1947 Fatal("","Terminating");
4e5c5506 1948 }
1949 }
c57f56b7 1950 }
1951 return kTRUE;
1952}
1953
1954//______________________________________________________________________________
d2a409b2 1955Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1956{
1957// Writes one or more JDL's corresponding to findex. If findex is negative,
1958// all run numbers are considered in one go (jdl). For non-negative indices
1959// they correspond to the indices in the array fInputFiles.
dec33edc 1960 if (!fInputFiles && !fMCLoop) return kFALSE;
d3b18c4c 1961 TObject *os;
1962 TString workdir;
1963 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1964 workdir += fGridWorkingDir;
f47d5cb4 1965 TString stageName = "$2";
1966 if (fProductionMode) stageName = "$4";
1967 if (!fMergeDirName.IsNull()) {
1968 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1969 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1970 } else {
1971 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1972 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1973 }
d3b18c4c 1974 if (fProductionMode) {
a8739e8a 1975 TIter next(fInputFiles);
b3e07543 1976 while ((os=next())) {
d3b18c4c 1977 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1978 }
56754bf6 1979 if (!fOutputToRunNo)
1980 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1981 else
1982 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1983 } else {
1984 if (!fRunNumbers.Length() && !fRunRange[0]) {
1985 // One jdl with no parameters in case input data is specified by name.
1986 TIter next(fInputFiles);
1987 while ((os=next()))
1988 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1989 if (!fOutputSingle.IsNull())
1990 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1991 else {
1992 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
b1dade98 1993// fMergingJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1994 }
1995 } else {
1996 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1997 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1998 if (!fOutputSingle.IsNull()) {
1999 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
2000 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
2001 } else {
2002 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 2003 }
2004 }
a8739e8a 2005 }
2006
a8739e8a 2007 // Generate the JDL as a string
2008 TString sjdl = fGridJDL->Generate();
0f389141 2009 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 2010 // Final merge jdl
2011 if (!fMergeDirName.IsNull()) {
2012 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
2013 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
2014 } else {
2015 fMergingJDL->SetOutputDirectory("$1", "Output directory");
2016 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
2017 }
b3e07543 2018 TString sjdl2 = fMergingJDL->Generate();
2019 Int_t index, index1;
95e16f01 2020 sjdl.ReplaceAll("\",\"", "\",\n \"");
a8739e8a 2021 sjdl.ReplaceAll("(member", "\n (member");
2022 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2023 sjdl.ReplaceAll("{", "{\n ");
2024 sjdl.ReplaceAll("};", "\n};");
2025 sjdl.ReplaceAll("{\n \n", "{\n");
2026 sjdl.ReplaceAll("\n\n", "\n");
2027 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2028 sjdl1.ReplaceAll("\",\"", "\",\n \"");
0f389141 2029 sjdl1.ReplaceAll("(member", "\n (member");
2030 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2031 sjdl1.ReplaceAll("{", "{\n ");
2032 sjdl1.ReplaceAll("};", "\n};");
2033 sjdl1.ReplaceAll("{\n \n", "{\n");
2034 sjdl1.ReplaceAll("\n\n", "\n");
2035 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 2036 sjdl2.ReplaceAll("\",\"", "\",\n \"");
b3e07543 2037 sjdl2.ReplaceAll("(member", "\n (member");
2038 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2039 sjdl2.ReplaceAll("{", "{\n ");
2040 sjdl2.ReplaceAll("};", "\n};");
2041 sjdl2.ReplaceAll("{\n \n", "{\n");
2042 sjdl2.ReplaceAll("\n\n", "\n");
2043 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 2044 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 2045 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 2046 index = sjdl.Index("JDLVariables");
2047 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 2048 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2049 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 2050 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 2051 index = fJobTag.Index(":");
2052 if (index < 0) index = fJobTag.Length();
2053 TString jobTag = fJobTag;
f47d5cb4 2054 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 2055 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 2056 if (fProductionMode) {
2057 sjdl1.Prepend("# Generated merging jdl (production mode) \
2058 \n# $1 = full alien path to output directory to be merged \
2059 \n# $2 = train number \
2060 \n# $3 = production (like LHC10b) \
2061 \n# $4 = merging stage \
2062 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2063 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2064 sjdl2.Prepend("# Generated merging jdl \
2065 \n# $1 = full alien path to output directory to be merged \
2066 \n# $2 = train number \
2067 \n# $3 = production (like LHC10b) \
2068 \n# $4 = merging stage \
2069 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2070 } else {
2071 sjdl1.Prepend("# Generated merging jdl \
2072 \n# $1 = full alien path to output directory to be merged \
2073 \n# $2 = merging stage \
03335324 2074 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2075 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2076 sjdl2.Prepend("# Generated merging jdl \
2077 \n# $1 = full alien path to output directory to be merged \
2078 \n# $2 = merging stage \
03335324 2079 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2080 }
0f389141 2081 index = sjdl1.Index("JDLVariables");
2082 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 2083 index = sjdl2.Index("JDLVariables");
2084 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 2085 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2086 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
2087 index = sjdl2.Index("Split =");
2088 if (index>=0) {
2089 index1 = sjdl2.Index("\n", index);
2090 sjdl2.Remove(index, index1-index+1);
2091 }
2092 index = sjdl2.Index("SplitMaxInputFileNumber");
2093 if (index>=0) {
2094 index1 = sjdl2.Index("\n", index);
2095 sjdl2.Remove(index, index1-index+1);
2096 }
2097 index = sjdl2.Index("InputDataCollection");
2098 if (index>=0) {
2099 index1 = sjdl2.Index(";", index);
2100 sjdl2.Remove(index, index1-index+1);
2101 }
2102 index = sjdl2.Index("InputDataListFormat");
2103 if (index>=0) {
2104 index1 = sjdl2.Index("\n", index);
2105 sjdl2.Remove(index, index1-index+1);
2106 }
2107 index = sjdl2.Index("InputDataList");
2108 if (index>=0) {
2109 index1 = sjdl2.Index("\n", index);
2110 sjdl2.Remove(index, index1-index+1);
2111 }
f47d5cb4 2112 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2113 // Write jdl to file
a8739e8a 2114 ofstream out;
d2a409b2 2115 out.open(fJDLName.Data(), ios::out);
a8739e8a 2116 if (out.bad()) {
d3b18c4c 2117 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2118 return kFALSE;
2119 }
2120 out << sjdl << endl;
b3e07543 2121 out.close();
0f389141 2122 TString mergeJDLName = fExecutable;
2123 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2124 if (fMergeViaJDL) {
2125 ofstream out1;
2126 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2127 if (out1.bad()) {
d3b18c4c 2128 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2129 return kFALSE;
2130 }
2131 out1 << sjdl1 << endl;
b3e07543 2132 out1.close();
2133 ofstream out2;
2134 TString finalJDL = mergeJDLName;
2135 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2136 out2.open(finalJDL.Data(), ios::out);
2137 if (out2.bad()) {
2138 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2139 return kFALSE;
2140 }
2141 out2 << sjdl2 << endl;
2142 out2.close();
0f389141 2143 }
a8739e8a 2144
2145 // Copy jdl to grid workspace
2146 if (!copy) {
d3b18c4c 2147 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2148 } else {
d2a409b2 2149 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2150 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2151 TString finalJDL = mergeJDLName;
2152 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2153 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2154 if (fProductionMode) {
b5fe9cba 2155 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2156 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2157 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2158 }
d2a409b2 2159 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2160 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2161 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2162 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2163// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2164 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2165 Fatal("","Terminating");
0f389141 2166 if (fMergeViaJDL) {
b3e07543 2167 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2168// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2169// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2170 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2171 Fatal("","Terminating");
2172 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2173 Fatal("","Terminating");
0f389141 2174 }
a8739e8a 2175 }
2176 return kTRUE;
2177}
2178
2179//______________________________________________________________________________
5513444a 2180Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2181{
2182// Returns true if file exists.
5513444a 2183 if (!gGrid) return kFALSE;
2ea55496 2184 TString slfn = lfn;
2185 slfn.ReplaceAll("alien://","");
2186 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2187 if (!res) return kFALSE;
2188 TMap *map = dynamic_cast<TMap*>(res->At(0));
2189 if (!map) {
2190 delete res;
2191 return kFALSE;
2192 }
2193 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2194 if (!objs || !objs->GetString().Length()) {
2195 delete res;
2196 return kFALSE;
2197 }
2198 delete res;
2199 return kTRUE;
2200}
2201
2202//______________________________________________________________________________
923e2ca5 2203Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2204{
2205// Returns true if directory exists. Can be also a path.
d7aac489 2206// Since there is not API in TAlien, we use the Cd trick:
923e2ca5 2207 if (!gGrid) return kFALSE;
d7aac489 2208 // Backup current path
2209 TString cpath = gGrid->Pwd();
2210 TString command = "cd ";
2211 TString sdir(dirname);
2212 sdir.ReplaceAll("alien://", "");
2213 command += sdir;
2214 TGridResult *res = gGrid->Command(command);
2215 if (!res) {
2216 gGrid->Cd(cpath);
2217 return kFALSE;
2218 }
2219 TMap *map = (TMap*)res->At(0);
2220 if (!map) {
2221 gGrid->Cd(cpath);
2222 delete res;
2223 return kFALSE;
923e2ca5 2224 }
d7aac489 2225 TString sval = map->GetValue("__result__")->GetName();
2226 Bool_t retval = (Bool_t)sval.Atoi();
2227 gGrid->Cd(cpath);
923e2ca5 2228 delete res;
d7aac489 2229 return retval;
2230}
923e2ca5 2231
2232//______________________________________________________________________________
a2f5fc01 2233void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2234{
2235// Check input data type.
a2f5fc01 2236 isCollection = kFALSE;
2237 isXml = kFALSE;
2238 useTags = kFALSE;
c57f56b7 2239 if (!gGrid) {
2240 Error("CheckDataType", "No connection to grid");
2241 return;
2242 }
a2f5fc01 2243 isCollection = IsCollection(lfn);
c57f56b7 2244 TString msg = "\n##### file: ";
2245 msg += lfn;
a2f5fc01 2246 if (isCollection) {
c57f56b7 2247 msg += " type: raw_collection;";
2248 // special treatment for collections
a2f5fc01 2249 isXml = kFALSE;
c57f56b7 2250 // check for tag files in the collection
2251 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2252 if (!res) {
2253 msg += " using_tags: No (unknown)";
23329835 2254 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2255 return;
2256 }
2257 const char* typeStr = res->GetKey(0, "origLFN");
2258 if (!typeStr || !strlen(typeStr)) {
2259 msg += " using_tags: No (unknown)";
23329835 2260 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2261 return;
2262 }
2263 TString file = typeStr;
a2f5fc01 2264 useTags = file.Contains(".tag");
2265 if (useTags) msg += " using_tags: Yes";
c57f56b7 2266 else msg += " using_tags: No";
23329835 2267 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2268 return;
2269 }
2270 TString slfn(lfn);
2271 slfn.ToLower();
a2f5fc01 2272 isXml = slfn.Contains(".xml");
2273 if (isXml) {
c57f56b7 2274 // Open xml collection and check if there are tag files inside
2275 msg += " type: xml_collection;";
2276 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2277 if (!coll) {
2278 msg += " using_tags: No (unknown)";
23329835 2279 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2280 return;
2281 }
2282 TMap *map = coll->Next();
2283 if (!map) {
2284 msg += " using_tags: No (unknown)";
23329835 2285 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2286 return;
2287 }
2288 map = (TMap*)map->GetValue("");
2289 TString file;
2290 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2291 useTags = file.Contains(".tag");
c57f56b7 2292 delete coll;
a2f5fc01 2293 if (useTags) msg += " using_tags: Yes";
c57f56b7 2294 else msg += " using_tags: No";
23329835 2295 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2296 return;
2297 }
a2f5fc01 2298 useTags = slfn.Contains(".tag");
c57f56b7 2299 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2300 else msg += " type: unknown file;";
a2f5fc01 2301 if (useTags) msg += " using_tags: Yes";
c57f56b7 2302 else msg += " using_tags: No";
23329835 2303 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2304}
2305
2306//______________________________________________________________________________
4e5c5506 2307void AliAnalysisAlien::EnablePackage(const char *package)
2308{
2309// Enables a par file supposed to exist in the current directory.
2310 TString pkg(package);
2311 pkg.ReplaceAll(".par", "");
2312 pkg += ".par";
2313 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2314 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2315 return;
2316 }
2317 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2318 Info("EnablePackage", "AliEn plugin will use .par packages");
2319 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2320 if (!fPackages) {
2321 fPackages = new TObjArray();
2322 fPackages->SetOwner();
2323 }
2324 fPackages->Add(new TObjString(pkg));
2325}
2326
2327//______________________________________________________________________________
3bdcb562 2328TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2329{
2330// Make a tree from files having the location specified in fFileForTestMode.
2331// Inspired from JF's CreateESDChain.
2332 if (fFileForTestMode.IsNull()) {
2333 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2334 return NULL;
2335 }
2336 if (gSystem->AccessPathName(fFileForTestMode)) {
2337 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2338 return NULL;
2339 }
2340 // Open the file
2341 ifstream in;
2342 in.open(fFileForTestMode);
2343 Int_t count = 0;
2344 // Read the input list of files and add them to the chain
b505f1f8 2345 TString line;
d6c60197 2346 TString streeName(treeName);
2347 if (IsUseMCchain()) streeName = "TE";
2348 TChain *chain = new TChain(streeName);
06f67114 2349 TList *friends = new TList();
2350 TChain *cfriend = 0;
2351 if (!fFriendChainName.IsNull()) {
2352 TObjArray *list = fFriendChainName.Tokenize(" ");
2353 TIter next(list);
2354 TObjString *str;
2355 while((str=(TObjString*)next())) {
2356 cfriend = new TChain(streeName, str->GetName());
2357 friends->Add(cfriend);
2358 chain->AddFriend(cfriend);
2359 }
2360 delete list;
2361 }
2362 TString bpath;
2363 TIter nextfriend(friends);
b505f1f8 2364 while (in.good())
2365 {
3bdcb562 2366 in >> line;
ffbe06b9 2367 if (line.IsNull() || line.BeginsWith("#")) continue;
3bdcb562 2368 if (count++ == fNtestFiles) break;
2369 TString esdFile(line);
2370 TFile *file = TFile::Open(esdFile);
b505f1f8 2371 if (file && !file->IsZombie()) {
2372 chain->Add(esdFile);
3bdcb562 2373 file->Close();
b505f1f8 2374 if (!fFriendChainName.IsNull()) {
4b5d9e21 2375 if (esdFile.Index("#") > -1)
2376 esdFile.Remove(esdFile.Index("#"));
06f67114 2377 bpath = gSystem->DirName(esdFile);
2378 bpath += "/";
2379 TString fileFriend;
2380 nextfriend.Reset();
2381 while ((cfriend=(TChain*)nextfriend())) {
2382 fileFriend = bpath;
2383 fileFriend += cfriend->GetTitle();
2384 file = TFile::Open(fileFriend);
2385 if (file && !file->IsZombie()) {
2386 file->Close();
2387 cfriend->Add(fileFriend);
2388 } else {
2389 Fatal("GetChainForTestMode", "Cannot open friend file: %s", fileFriend.Data());
2390 return 0;
2391 }
2392 }
4b5d9e21 2393 }
3bdcb562 2394 } else {
2395 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2396 }
b505f1f8 2397 }
2398 in.close();
2399 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2400 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2401 delete chain;
06f67114 2402 friends->Delete();
2403 delete friends;
3bdcb562 2404 return NULL;
b505f1f8 2405 }
b505f1f8 2406 return chain;
3bdcb562 2407}
2408
2409//______________________________________________________________________________
d2a409b2 2410const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2411{
2412// Get job status for all jobs with jobid>jobidstart.
2413 static char mstatus[20];
2414 mstatus[0] = '\0';
2415 nrunning = 0;
2416 nwaiting = 0;
2417 nerror = 0;
2418 ndone = 0;
2419 TGridJobStatusList *list = gGrid->Ps("");
2420 if (!list) return mstatus;
2421 Int_t nentries = list->GetSize();
2422 TGridJobStatus *status;
2423 Int_t pid;
2424 for (Int_t ijob=0; ijob<nentries; ijob++) {
2425 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2426 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2427 if (pid<jobidstart) continue;
2428 if (pid == lastid) {
8ddad121 2429 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2430 }
2431 switch (status->GetStatus()) {
2432 case TGridJobStatus::kWAITING:
2433 nwaiting++; break;
2434 case TGridJobStatus::kRUNNING:
2435 nrunning++; break;
2436 case TGridJobStatus::kABORTED:
2437 case TGridJobStatus::kFAIL:
2438 case TGridJobStatus::kUNKNOWN:
2439 nerror++; break;
2440 case TGridJobStatus::kDONE:
2441 ndone++;
2442 }
2443 }
2444 list->Delete();
2445 delete list;
2446 return mstatus;
2447}
2448
2449//______________________________________________________________________________
c57f56b7 2450Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2451{
2452// Returns true if file is a collection. Functionality duplicated from
2453// TAlien::Type() because we don't want to directly depend on TAlien.
2454 if (!gGrid) {
2455 Error("IsCollection", "No connection to grid");
2456 return kFALSE;
2457 }
2458 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2459 if (!res) return kFALSE;
2460 const char* typeStr = res->GetKey(0, "type");
2461 if (!typeStr || !strlen(typeStr)) return kFALSE;
2462 if (!strcmp(typeStr, "collection")) return kTRUE;
2463 delete res;
2464 return kFALSE;
2465}
2466
2467//______________________________________________________________________________
fe2d7fc2 2468Bool_t AliAnalysisAlien::IsSingleOutput() const
2469{
2470// Check if single-ouput option is on.
2471 return (!fOutputSingle.IsNull());
2472}
715da79c 2473
2474//______________________________________________________________________________
2475Long64_t AliAnalysisAlien::RunMacroAndExtractLibs(const char* macro, const char *args, TString &libs)
2476{
2477// Tries to run the specified macro and return the libraries that it loads.
2478 TString expname;
2479 if (strlen(macro)) expname = gSystem->ExpandPathName(macro);
2480 if (expname.IsNull() || gSystem->AccessPathName(expname)) {
2481 ::Error("RunMacroAndExtractLibs","Cannot find macro %s in current directory", macro);
2482 return -1;
2483 }
2484 TString oldlibs = gSystem->GetLibraries();
2485 TMacro m(expname);
2486 Int_t error = 0;
2487 Long64_t retval = m.Exec(args, &error);
2488 if (error != TInterpreter::kNoError)
2489 {
2490 ::Error("RunMacroAndExtractLibs", "Macro interpretation %s failed", macro);
2491 return -1;
2492 }
2493 libs = gSystem->GetLibraries();
2494 libs.ReplaceAll(oldlibs, "");
2495 libs.Strip(TString::kLeading);
2496 TObjArray *libTokens = libs.Tokenize(" ");
2497 libs = "";
2498 for (Int_t i=0; i<libTokens->GetEntries(); i++) {
2499 if (!libs.IsNull()) libs += " ";
2500 libs += gSystem->BaseName(libTokens->At(i)->GetName());
2501 }
2502 delete libTokens;
2503 return retval;
2504}
2505
fe2d7fc2 2506//______________________________________________________________________________
16a4353c 2507void AliAnalysisAlien::Print(Option_t *) const
2508{
2509// Print current plugin settings.
84fcd93f 2510 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2511 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2512 if (mgr && mgr->IsProofMode()) {
2513 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2514 if (TestBit(AliAnalysisGrid::kTest))
2515 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2516 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2517 if (!fProofDataSet.IsNull())
2518 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2519 if (fProofReset==1)
2520 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2521 if (fProofReset>1)
2522 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
874de7ea 2523 if (!fROOTVersion.IsNull())
2524 printf("= ROOT version requested________________________ %s\n", fROOTVersion.Data());
d3b18c4c 2525 else
2526 printf("= ROOT version requested________________________ default\n");
2527 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2528 if (!fAliRootMode.IsNull())
2529 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2530 if (fNproofWorkers)
2531 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2532 if (fNproofWorkersPerSlave)
2533 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2534 if (TestSpecialBit(kClearPackages))
2535 printf("= ClearPackages requested...\n");
2536 if (fIncludePath.Data())
2537 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2538 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2539 if (fPackages && fPackages->GetEntries()) {
2540 TIter next(fPackages);
2541 TObject *obj;
2542 TString list;
2543 while ((obj=next())) list += obj->GetName();
2544 printf("= Par files to be used: ________________________ %s\n", list.Data());
2545 }
2546 if (TestSpecialBit(kProofConnectGrid))
2547 printf("= Requested PROOF connection to grid\n");
2548 return;
2549 }
e1c22e21 2550 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2551 if (fOverwriteMode) {
2552 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2553 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2554 }
348be253 2555 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
780ec36d 2556 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO:Print");
84fcd93f 2557 printf("= Production mode:______________________________ %d\n", fProductionMode);
2558 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2559 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2560 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2561 if (fUser.Length())
84fcd93f 2562 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2563 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2564 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
780ec36d 2565 TString basedatadir = fGridDataDir;
2566 TString pattern = fDataPattern;
2567 pattern.Strip();
2568 Int_t ind = pattern.Index(" ");
2569 if (ind>=0) {
2570 basedatadir += "/%run%/";
2571 basedatadir += pattern(0, ind);
2572 pattern = pattern(ind+1, pattern.Length());
2573 }
2574 printf("= Data base directory path requested: __________ %s\n", basedatadir.Data());
2575 printf("= Data search pattern: _________________________ %s\n", pattern.Data());
84fcd93f 2576 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2577 if (fRunNumbers.Length())
84fcd93f 2578 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2579 if (fRunRange[0])
74013535 2580 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2581 if (!fRunRange[0] && !fRunNumbers.Length()) {
2582 TIter next(fInputFiles);
2583 TObject *obj;
2584 TString list;
2585 while ((obj=next())) list += obj->GetName();
84fcd93f 2586 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2587 }
2588 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2589 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2590 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2591 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2592 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2593 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2594 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2595 printf("=====================================================================\n");
2596 printf("= Job price: ___________________________________ %d\n", fPrice);
2597 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2598 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2599 if (fMaxInitFailed>0)
84fcd93f 2600 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2601 if (fMasterResubmitThreshold>0)
84fcd93f 2602 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2603 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2604 if (fNrunsPerMaster>0)
84fcd93f 2605 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2606 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2607 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2608 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2609 if (fArguments.Length())
84fcd93f 2610 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2611 if (fExecutableArgs.Length())
2612 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2613 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2614 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2615 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2616 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2617 if (fDatasetName)
84fcd93f 2618 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2619 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2620 if (fIncludePath.Data())
84fcd93f 2621 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2622 if (fCloseSE.Length())
84fcd93f 2623 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2624 if (fFriendChainName.Length())
84fcd93f 2625 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2626 if (fPackages && fPackages->GetEntries()) {
16a4353c 2627 TIter next(fPackages);
2628 TObject *obj;
2629 TString list;
2630 while ((obj=next())) list += obj->GetName();
84fcd93f 2631 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2632 }
2633}
2634
2635//______________________________________________________________________________
c57f56b7 2636void AliAnalysisAlien::SetDefaults()
2637{
2638// Set default values for everything. What cannot be filled will be left empty.
2639 if (fGridJDL) delete fGridJDL;
2640 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2641 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2642 fPrice = 1;
2643 fTTL = 30000;
2644 fSplitMaxInputFileNumber = 100;
2645 fMaxInitFailed = 0;
2646 fMasterResubmitThreshold = 0;
bb885a9e 2647 fNtestFiles = 10;
149d288c 2648 fNreplicas = 2;
a8739e8a 2649 fRunRange[0] = 0;
2650 fRunRange[1] = 0;
8e3bd06a 2651 fRunPrefix = "%d";
319593fb 2652 fNrunsPerMaster = 1;
16a4353c 2653 fMaxMergeFiles = 100;
c57f56b7 2654 fRunNumbers = "";
2655 fExecutable = "analysis.sh";
5aa94dc3 2656 fExecutableCommand = "root -b -q -x";
c57f56b7 2657 fArguments = "";
631c0b05 2658 fExecutableArgs = "";
c57f56b7 2659 fAnalysisMacro = "myAnalysis.C";
2660 fAnalysisSource = "";
2661 fAdditionalLibs = "";
2662 fSplitMode = "se";
2663 fAPIVersion = "";
2664 fROOTVersion = "";
2665 fAliROOTVersion = "";
2666 fUser = ""; // Your alien user name
2667 fGridWorkingDir = "";
2668 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2669 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2670 fFriendChainName = "";
c57f56b7 2671 fGridOutputDir = "output";
b83f84f0 2672 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2673 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2674 fInputFormat = "xml-single";
2675 fJDLName = "analysis.jdl";
c6cb3634 2676 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2677 fMergeExcludes = "";
0f389141 2678 fMergeViaJDL = 0;
348be253 2679 SetUseCopy(kTRUE);
2680 SetCheckCopy(kTRUE);
149d288c 2681 SetDefaultOutputs(kTRUE);
e1c22e21 2682 fOverwriteMode = 1;
c57f56b7 2683}
2684
2685//______________________________________________________________________________
a41b2882 2686void AliAnalysisAlien::SetFriendChainName(const char *name, const char *libnames)
2687{
2688 // Set file name for the chain of friends and optionally additional libs to be loaded.
2689 // Libs should be separated by blancs.
2690 fFriendChainName = name;
06f67114 2691 fFriendChainName.ReplaceAll(",", " ");
2692 fFriendChainName.Strip();
2693 fFriendChainName.ReplaceAll(" ", " ");
2694
a41b2882 2695 fFriendLibs = libnames;
1a3f255c 2696 if (fFriendLibs.Length()) {
2dffffd6 2697 if(!fFriendLibs.Contains(".so") &&
2698 !fFriendLibs.Contains(".dylib"))
1a3f255c 2699 Fatal("SetFriendChainName()", "You should provide explicit library names (with extension)");
2700 fFriendLibs.ReplaceAll(",", " ");
2701 fFriendLibs.Strip();
2702 fFriendLibs.ReplaceAll(" ", " ");
a41b2882 2703 }
2704}
2705
2706//______________________________________________________________________________
874de7ea 2707void AliAnalysisAlien::SetRootVersionForProof(const char *version)
2708{
2709// Obsolete method. Use SetROOTVersion instead
2710 Warning("SetRootVersionForProof", "Obsolete. Use SetROOTVersion instead");
95e16f01 2711 if (fROOTVersion.IsNull()) SetROOTVersion(version);
2712 else Error("SetRootVersionForProof", "ROOT version already set to %s", fROOTVersion.Data());
874de7ea 2713}
2714
2715//______________________________________________________________________________
b3e07543 2716Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2717{
b3e07543 2718// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2719 // First check if the result is already in the output directory.
2720 if (FileExists(Form("%s/%s",aliendir,filename))) {
2721 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2722 return kFALSE;
2723 }
b3e07543 2724 // Now check the last stage done.
2725 Int_t stage = 0;
2726 while (1) {
2727 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2728 stage++;
2729 }
2730 // Next stage of merging
2731 stage++;
2732 TString pattern = "*root_archive.zip";
2733 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2734 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2735 if (res) delete res;
2736 // Write standard output to file
2737 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2738 // Count the number of files inside
2739 ifstream ifile;
2740 ifile.open(Form("Stage_%d.xml",stage));
2741 if (!ifile.good()) {
2742 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2743 return kFALSE;
b3e07543 2744 }
2745 TString line;
2746 Int_t nfiles = 0;
2747 while (!ifile.eof()) {
2748 ifile >> line;
2749 if (line.Contains("/event")) nfiles++;
7c2cd90a 2750 }
b3e07543 2751 ifile.close();
2752 if (!nfiles) {
2753 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2754 return kFALSE;
2755 } else {
2756 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2757 }
b3e07543 2758 // Copy the file in the output directory
2759 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2760// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2761 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2762 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2763 // Check if this is the last stage to be done.
2764 Bool_t laststage = (nfiles<nperchunk);
2765 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
0ad3ea1f 2766 Int_t jobId = 0;
b3e07543 2767 if (laststage) {
2768 printf("### Submiting final merging stage %d\n", stage);
2769 TString finalJDL = jdl;
2770 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2771 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
0ad3ea1f 2772 jobId = SubmitSingleJob(query);
b3e07543 2773 } else {
2774 printf("### Submiting merging stage %d\n", stage);
03335324 2775 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
0ad3ea1f 2776 jobId = SubmitSingleJob(query);
7c2cd90a 2777 }
0ad3ea1f 2778 if (!jobId) return kFALSE;
2779
2780 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
2781 fGridJobIDs.Append(Form("%d", jobId));
2782 if (!fGridStages.IsNull()) fGridStages.Append(" ");
2783 fGridStages.Append(Form("%s_merge_stage%d",
2784 laststage ? "final" : "partial", stage));
2785
b3e07543 2786 return kTRUE;
2787}
7c2cd90a 2788
2789//______________________________________________________________________________
105631b5 2790AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2791{
2792// Loat the analysis manager from a file.
2793 TFile *file = TFile::Open(fname);
2794 if (!file) {
2795 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2796 return 0;
2797 }
2798 TIter nextkey(file->GetListOfKeys());
2799 AliAnalysisManager *mgr = 0;
2800 TKey *key;
2801 while ((key=(TKey*)nextkey())) {
2802 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2803 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2804 }
2805 if (!mgr)
2806 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2807 return mgr;
2808}
2809
2810//______________________________________________________________________________
7c2cd90a 2811Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2812{
2813// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2814 if (!gGrid) return 0;
2815 printf("=> %s ------> ",query);
2816 TGridResult *res = gGrid->Command(query);
2817 if (!res) return 0;
2818 TString jobId = res->GetKey(0,"jobId");
2819 delete res;
2820 if (jobId.IsNull()) {
2821 printf("submission failed. Reason:\n");
2822 gGrid->Stdout();
2823 gGrid->Stderr();
2824 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2825 return 0;
2826 }
0ad3ea1f 2827 Int_t ijobId = jobId.Atoi();
2828 printf(" Job id: '%s' (%d)\n", jobId.Data(), ijobId);
2829 return ijobId;
7c2cd90a 2830}
2831
2832//______________________________________________________________________________
ec5acdce 2833Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2834{
2835// Merges a collection of output files using concatenation.
d66aa2a9 2836 TString scoll(collection);
2837 if (!scoll.Contains(".xml")) return kFALSE;
ec5acdce 2838 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2839 if (!coll) {
2840 ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2841 return kFALSE;
2842 }
2843 // Iterate grid collection
2844 TString outtmp;
2845 Bool_t merged = kFALSE;
2846 Int_t ifile = 0;
2847 while (coll->Next()) {
2848 TString fname = gSystem->DirName(coll->GetTURL());
2849 fname += "/";
2850 fname += output;
2851 outtmp = Form("%d_%s", ifile, output);
2852 if (!TFile::Cp(fname, outtmp)) {
2853 ::Error("MergeInfo", "Could not copy %s", fname.Data());
2854 continue;
2855 }
2856 ifile++;
2857 if (ifile<2) {
2858 gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2859 continue;
2860 }
2861 gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2862 gSystem->Exec("cp tempmerged lastmerged");
2863 }
2864 if (ifile) {
2865 gSystem->Exec(Form("cp lastmerged %s", output));
2866 gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2867 merged = kTRUE;
2868 }
2869 return merged;
2870}
2871
2872//______________________________________________________________________________
b3e07543 2873Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2874{
b3e07543 2875// Merge given output files from basedir. Basedir can be an alien output directory
2876// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2877// files in a group (ignored for xml input). Merging can be done in stages:
2878// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2879// stage=1 : works with an xml of all root_archive.zip in the output directory
2880// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2881 TString outputFile = output;
0f389141 2882 TString command;
a2f5fc01 2883 TString outputChunk;
2884 TString previousChunk = "";
b3e07543 2885 TObjArray *listoffiles = new TObjArray();
2886// listoffiles->SetOwner();
a2f5fc01 2887 Int_t countChunk = 0;
2888 Int_t countZero = nmaxmerge;
0f389141 2889 Bool_t merged = kTRUE;
d66aa2a9 2890 Bool_t isGrid = kTRUE;
a2f5fc01 2891 Int_t index = outputFile.Index("@");
2892 if (index > 0) outputFile.Remove(index);
7c2cd90a 2893 TString inputFile = outputFile;
b3e07543 2894 TString sbasedir = basedir;
2895 if (sbasedir.Contains(".xml")) {
2896 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2897 nmaxmerge = 9999999;
2898 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2899 if (!coll) {
2900 ::Error("MergeOutput", "Input XML collection empty.");
2901 return kFALSE;
2902 }
2903 // Iterate grid collection
2904 while (coll->Next()) {
2905 TString fname = gSystem->DirName(coll->GetTURL());
2906 fname += "/";
2907 fname += inputFile;
2908 listoffiles->Add(new TNamed(fname.Data(),""));
2909 }
ffbe06b9 2910 } else if (sbasedir.Contains(".txt")) {
d66aa2a9 2911 // The file having the .txt extension is expected to contain a list of
2912 // folders where the output files will be looked. For alien folders,
2913 // the full folder LFN is expected (starting with alien://)
ffbe06b9 2914 // Assume lfn's on each line
2915 TString line;
2916 ifstream in;
2917 in.open(sbasedir);
d66aa2a9 2918 if (in.fail()) {
2919 ::Error("MergeOutput", "File %s cannot be opened. Merging stopped." ,sbasedir.Data());
2920 return kTRUE;
2921 }
ffbe06b9 2922 Int_t nfiles = 0;
2923 while (in.good()) {
2924 in >> line;
2925 if (line.IsNull() || line.BeginsWith("#")) continue;
d66aa2a9 2926 line.Strip();
2927 if (!line.Contains("alien:")) isGrid = kFALSE;
2928 line += "/";
2929 line += outputFile;
ffbe06b9 2930 nfiles++;
2931 listoffiles->Add(new TNamed(line.Data(),""));
2932 }
d66aa2a9 2933 in.close();
ffbe06b9 2934 if (!nfiles) {
2935 ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data());
2936 delete listoffiles;
2937 return kFALSE;
2938 }
b3e07543 2939 } else {
2940 command = Form("find %s/ *%s", basedir, inputFile.Data());
2941 printf("command: %s\n", command.Data());
2942 TGridResult *res = gGrid->Command(command);
2943 if (!res) {
2944 ::Error("MergeOutput","No result for the find command\n");
2945 delete listoffiles;
2946 return kFALSE;
2947 }
2948 TIter nextmap(res);
2949 TMap *map = 0;
2950 while ((map=(TMap*)nextmap())) {
2951 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2952 if (!objs || !objs->GetString().Length()) {
2953 // Nothing found - skip this output
2954 delete res;
2955 delete listoffiles;
2956 return kFALSE;
2957 }
2958 listoffiles->Add(new TNamed(objs->GetName(),""));
2959 }
2960 delete res;
2961 }
2962 if (!listoffiles