fixed bug that could ignore libSTEER if libSTEERbase was loaded in LoadModule (JFGO...
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
51ClassImp(AliAnalysisAlien)
6c395669 52#if 0
53;
54#endif
c57f56b7 55
6c395669 56namespace {
57 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
58 {
59 TString sl(Form("file:%s", loc));
60 TString sr(Form("alien://%s", rem));
61 Bool_t ret = TFile::Cp(sl, sr);
62 if (!ret) {
63 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
64 }
65 return ret;
66 }
67}
68
c57f56b7 69//______________________________________________________________________________
70AliAnalysisAlien::AliAnalysisAlien()
71 :AliAnalysisGrid(),
72 fGridJDL(NULL),
0f389141 73 fMergingJDL(NULL),
c57f56b7 74 fPrice(0),
75 fTTL(0),
76 fSplitMaxInputFileNumber(0),
77 fMaxInitFailed(0),
78 fMasterResubmitThreshold(0),
bb885a9e 79 fNtestFiles(0),
319593fb 80 fNrunsPerMaster(0),
16a4353c 81 fMaxMergeFiles(0),
b3e07543 82 fMaxMergeStages(0),
d2a409b2 83 fNsubmitted(0),
a3e84053 84 fProductionMode(0),
cd11251e 85 fOutputToRunNo(0),
0f389141 86 fMergeViaJDL(0),
87 fFastReadOption(0),
e1c22e21 88 fOverwriteMode(1),
149d288c 89 fNreplicas(2),
3bdcb562 90 fNproofWorkers(0),
27734f0e 91 fNproofWorkersPerSlave(0),
3bdcb562 92 fProofReset(0),
c57f56b7 93 fRunNumbers(),
94 fExecutable(),
0a1c1f7f 95 fExecutableCommand(),
c57f56b7 96 fArguments(),
631c0b05 97 fExecutableArgs(),
c57f56b7 98 fAnalysisMacro(),
99 fAnalysisSource(),
d3b18c4c 100 fValidationScript(),
d5c6455a 101 fAdditionalRootLibs(),
c57f56b7 102 fAdditionalLibs(),
103 fSplitMode(),
104 fAPIVersion(),
105 fROOTVersion(),
106 fAliROOTVersion(),
648174cf 107 fExternalPackages(),
c57f56b7 108 fUser(),
109 fGridWorkingDir(),
110 fGridDataDir(),
111 fDataPattern(),
112 fGridOutputDir(),
113 fOutputArchive(),
114 fOutputFiles(),
115 fInputFormat(),
e7c71df0 116 fDatasetName(),
c57f56b7 117 fJDLName(),
d3b18c4c 118 fTerminateFiles(),
bb885a9e 119 fMergeExcludes(),
37d21c01 120 fRegisterExcludes(),
f965131e 121 fIncludePath(),
bb885a9e 122 fCloseSE(),
0df6ccf2 123 fFriendChainName(),
c6cb3634 124 fJobTag(),
648174cf 125 fOutputSingle(),
5fce53f4 126 fRunPrefix(),
3bdcb562 127 fProofCluster(),
128 fProofDataSet(),
129 fFileForTestMode(),
130 fRootVersionForProof(),
131 fAliRootMode(),
f47d5cb4 132 fMergeDirName(),
4e5c5506 133 fInputFiles(0),
28ce6c9c 134 fPackages(0),
77f34eae 135 fModules(0),
28ce6c9c 136 fProofParam()
c57f56b7 137{
138// Dummy ctor.
139 SetDefaults();
140}
141
142//______________________________________________________________________________
143AliAnalysisAlien::AliAnalysisAlien(const char *name)
144 :AliAnalysisGrid(name),
145 fGridJDL(NULL),
0f389141 146 fMergingJDL(NULL),
c57f56b7 147 fPrice(0),
148 fTTL(0),
149 fSplitMaxInputFileNumber(0),
150 fMaxInitFailed(0),
151 fMasterResubmitThreshold(0),
bb885a9e 152 fNtestFiles(0),
319593fb 153 fNrunsPerMaster(0),
16a4353c 154 fMaxMergeFiles(0),
b3e07543 155 fMaxMergeStages(0),
d2a409b2 156 fNsubmitted(0),
a3e84053 157 fProductionMode(0),
cd11251e 158 fOutputToRunNo(0),
0f389141 159 fMergeViaJDL(0),
160 fFastReadOption(0),
e1c22e21 161 fOverwriteMode(1),
149d288c 162 fNreplicas(2),
3bdcb562 163 fNproofWorkers(0),
27734f0e 164 fNproofWorkersPerSlave(0),
3bdcb562 165 fProofReset(0),
c57f56b7 166 fRunNumbers(),
167 fExecutable(),
0a1c1f7f 168 fExecutableCommand(),
c57f56b7 169 fArguments(),
631c0b05 170 fExecutableArgs(),
c57f56b7 171 fAnalysisMacro(),
172 fAnalysisSource(),
d3b18c4c 173 fValidationScript(),
d5c6455a 174 fAdditionalRootLibs(),
c57f56b7 175 fAdditionalLibs(),
176 fSplitMode(),
177 fAPIVersion(),
178 fROOTVersion(),
179 fAliROOTVersion(),
648174cf 180 fExternalPackages(),
c57f56b7 181 fUser(),
182 fGridWorkingDir(),
183 fGridDataDir(),
184 fDataPattern(),
185 fGridOutputDir(),
186 fOutputArchive(),
187 fOutputFiles(),
188 fInputFormat(),
e7c71df0 189 fDatasetName(),
c57f56b7 190 fJDLName(),
d3b18c4c 191 fTerminateFiles(),
bb885a9e 192 fMergeExcludes(),
37d21c01 193 fRegisterExcludes(),
f965131e 194 fIncludePath(),
bb885a9e 195 fCloseSE(),
0df6ccf2 196 fFriendChainName(),
c6cb3634 197 fJobTag(),
648174cf 198 fOutputSingle(),
5fce53f4 199 fRunPrefix(),
3bdcb562 200 fProofCluster(),
201 fProofDataSet(),
202 fFileForTestMode(),
203 fRootVersionForProof(),
204 fAliRootMode(),
f47d5cb4 205 fMergeDirName(),
4e5c5506 206 fInputFiles(0),
28ce6c9c 207 fPackages(0),
77f34eae 208 fModules(0),
28ce6c9c 209 fProofParam()
c57f56b7 210{
211// Default ctor.
212 SetDefaults();
213}
214
215//______________________________________________________________________________
216AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
217 :AliAnalysisGrid(other),
218 fGridJDL(NULL),
0f389141 219 fMergingJDL(NULL),
c57f56b7 220 fPrice(other.fPrice),
221 fTTL(other.fTTL),
222 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
223 fMaxInitFailed(other.fMaxInitFailed),
224 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 225 fNtestFiles(other.fNtestFiles),
319593fb 226 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 227 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 228 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 229 fNsubmitted(other.fNsubmitted),
a3e84053 230 fProductionMode(other.fProductionMode),
cd11251e 231 fOutputToRunNo(other.fOutputToRunNo),
0f389141 232 fMergeViaJDL(other.fMergeViaJDL),
233 fFastReadOption(other.fFastReadOption),
234 fOverwriteMode(other.fOverwriteMode),
149d288c 235 fNreplicas(other.fNreplicas),
3bdcb562 236 fNproofWorkers(other.fNproofWorkers),
27734f0e 237 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 238 fProofReset(other.fProofReset),
c57f56b7 239 fRunNumbers(other.fRunNumbers),
240 fExecutable(other.fExecutable),
0a1c1f7f 241 fExecutableCommand(other.fExecutableCommand),
c57f56b7 242 fArguments(other.fArguments),
631c0b05 243 fExecutableArgs(other.fExecutableArgs),
c57f56b7 244 fAnalysisMacro(other.fAnalysisMacro),
245 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 246 fValidationScript(other.fValidationScript),
d5c6455a 247 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 248 fAdditionalLibs(other.fAdditionalLibs),
249 fSplitMode(other.fSplitMode),
250 fAPIVersion(other.fAPIVersion),
251 fROOTVersion(other.fROOTVersion),
252 fAliROOTVersion(other.fAliROOTVersion),
648174cf 253 fExternalPackages(other.fExternalPackages),
c57f56b7 254 fUser(other.fUser),
255 fGridWorkingDir(other.fGridWorkingDir),
256 fGridDataDir(other.fGridDataDir),
257 fDataPattern(other.fDataPattern),
258 fGridOutputDir(other.fGridOutputDir),
259 fOutputArchive(other.fOutputArchive),
260 fOutputFiles(other.fOutputFiles),
261 fInputFormat(other.fInputFormat),
e7c71df0 262 fDatasetName(other.fDatasetName),
c57f56b7 263 fJDLName(other.fJDLName),
d3b18c4c 264 fTerminateFiles(other.fTerminateFiles),
bb885a9e 265 fMergeExcludes(other.fMergeExcludes),
37d21c01 266 fRegisterExcludes(other.fRegisterExcludes),
f965131e 267 fIncludePath(other.fIncludePath),
bb885a9e 268 fCloseSE(other.fCloseSE),
0df6ccf2 269 fFriendChainName(other.fFriendChainName),
c6cb3634 270 fJobTag(other.fJobTag),
648174cf 271 fOutputSingle(other.fOutputSingle),
5fce53f4 272 fRunPrefix(other.fRunPrefix),
3bdcb562 273 fProofCluster(other.fProofCluster),
274 fProofDataSet(other.fProofDataSet),
275 fFileForTestMode(other.fFileForTestMode),
276 fRootVersionForProof(other.fRootVersionForProof),
277 fAliRootMode(other.fAliRootMode),
f47d5cb4 278 fMergeDirName(other.fMergeDirName),
4e5c5506 279 fInputFiles(0),
28ce6c9c 280 fPackages(0),
77f34eae 281 fModules(0),
28ce6c9c 282 fProofParam()
c57f56b7 283{
284// Copy ctor.
285 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 286 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 287 fRunRange[0] = other.fRunRange[0];
288 fRunRange[1] = other.fRunRange[1];
c57f56b7 289 if (other.fInputFiles) {
290 fInputFiles = new TObjArray();
291 TIter next(other.fInputFiles);
292 TObject *obj;
293 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
294 fInputFiles->SetOwner();
295 }
4e5c5506 296 if (other.fPackages) {
297 fPackages = new TObjArray();
298 TIter next(other.fPackages);
299 TObject *obj;
300 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
301 fPackages->SetOwner();
302 }
77f34eae 303 if (other.fModules) {
304 fModules = new TObjArray();
305 fModules->SetOwner();
306 TIter next(other.fModules);
307 AliAnalysisTaskCfg *mod, *crt;
308 while ((crt=(AliAnalysisTaskCfg*)next())) {
309 mod = new AliAnalysisTaskCfg(*crt);
310 fModules->Add(mod);
311 }
312 }
c57f56b7 313}
314
315//______________________________________________________________________________
316AliAnalysisAlien::~AliAnalysisAlien()
317{
318// Destructor.
77f34eae 319 delete fGridJDL;
320 delete fMergingJDL;
321 delete fInputFiles;
322 delete fPackages;
323 delete fModules;
28ce6c9c 324 fProofParam.DeleteAll();
c57f56b7 325}
326
327//______________________________________________________________________________
328AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
329{
330// Assignment.
331 if (this != &other) {
332 AliAnalysisGrid::operator=(other);
333 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 334 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 335 fPrice = other.fPrice;
336 fTTL = other.fTTL;
337 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
338 fMaxInitFailed = other.fMaxInitFailed;
339 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 340 fNtestFiles = other.fNtestFiles;
a3e84053 341 fNrunsPerMaster = other.fNrunsPerMaster;
342 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 343 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 344 fNsubmitted = other.fNsubmitted;
345 fProductionMode = other.fProductionMode;
cd11251e 346 fOutputToRunNo = other.fOutputToRunNo;
0f389141 347 fMergeViaJDL = other.fMergeViaJDL;
348 fFastReadOption = other.fFastReadOption;
349 fOverwriteMode = other.fOverwriteMode;
149d288c 350 fNreplicas = other.fNreplicas;
3bdcb562 351 fNproofWorkers = other.fNproofWorkers;
27734f0e 352 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 353 fProofReset = other.fProofReset;
c57f56b7 354 fRunNumbers = other.fRunNumbers;
355 fExecutable = other.fExecutable;
0a1c1f7f 356 fExecutableCommand = other.fExecutableCommand;
c57f56b7 357 fArguments = other.fArguments;
631c0b05 358 fExecutableArgs = other.fExecutableArgs;
c57f56b7 359 fAnalysisMacro = other.fAnalysisMacro;
360 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 361 fValidationScript = other.fValidationScript;
d5c6455a 362 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 363 fAdditionalLibs = other.fAdditionalLibs;
364 fSplitMode = other.fSplitMode;
365 fAPIVersion = other.fAPIVersion;
366 fROOTVersion = other.fROOTVersion;
367 fAliROOTVersion = other.fAliROOTVersion;
648174cf 368 fExternalPackages = other.fExternalPackages;
c57f56b7 369 fUser = other.fUser;
370 fGridWorkingDir = other.fGridWorkingDir;
371 fGridDataDir = other.fGridDataDir;
372 fDataPattern = other.fDataPattern;
373 fGridOutputDir = other.fGridOutputDir;
374 fOutputArchive = other.fOutputArchive;
375 fOutputFiles = other.fOutputFiles;
376 fInputFormat = other.fInputFormat;
e7c71df0 377 fDatasetName = other.fDatasetName;
c57f56b7 378 fJDLName = other.fJDLName;
d3b18c4c 379 fTerminateFiles = other.fTerminateFiles;
bb885a9e 380 fMergeExcludes = other.fMergeExcludes;
37d21c01 381 fRegisterExcludes = other.fRegisterExcludes;
f965131e 382 fIncludePath = other.fIncludePath;
bb885a9e 383 fCloseSE = other.fCloseSE;
0df6ccf2 384 fFriendChainName = other.fFriendChainName;
c6cb3634 385 fJobTag = other.fJobTag;
648174cf 386 fOutputSingle = other.fOutputSingle;
5fce53f4 387 fRunPrefix = other.fRunPrefix;
3bdcb562 388 fProofCluster = other.fProofCluster;
389 fProofDataSet = other.fProofDataSet;
390 fFileForTestMode = other.fFileForTestMode;
391 fRootVersionForProof = other.fRootVersionForProof;
392 fAliRootMode = other.fAliRootMode;
f47d5cb4 393 fMergeDirName = other.fMergeDirName;
c57f56b7 394 if (other.fInputFiles) {
395 fInputFiles = new TObjArray();
396 TIter next(other.fInputFiles);
397 TObject *obj;
398 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
399 fInputFiles->SetOwner();
400 }
4e5c5506 401 if (other.fPackages) {
402 fPackages = new TObjArray();
403 TIter next(other.fPackages);
404 TObject *obj;
405 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
406 fPackages->SetOwner();
407 }
77f34eae 408 if (other.fModules) {
409 fModules = new TObjArray();
410 fModules->SetOwner();
411 TIter next(other.fModules);
412 AliAnalysisTaskCfg *mod, *crt;
413 while ((crt=(AliAnalysisTaskCfg*)next())) {
414 mod = new AliAnalysisTaskCfg(*crt);
415 fModules->Add(mod);
416 }
417 }
c57f56b7 418 }
419 return *this;
420}
421
422//______________________________________________________________________________
37d21c01 423void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
424{
425// Add a single additional library to be loaded. Extension must be present.
426 TString lib(name);
427 if (!lib.Contains(".")) {
428 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
429 return;
430 }
431 if (fAdditionalLibs.Contains(name)) {
432 Warning("AddAdditionalLibrary", "Library %s already added.", name);
433 return;
434 }
435 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
436 fAdditionalLibs += lib;
437}
438
439//______________________________________________________________________________
77f34eae 440void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
441{
442// Adding a module. Checks if already existing. Becomes owned by this.
443 if (!module) return;
444 if (GetModule(module->GetName())) {
445 Error("AddModule", "A module having the same name %s already added", module->GetName());
446 return;
447 }
448 if (!fModules) {
449 fModules = new TObjArray();
450 fModules->SetOwner();
451 }
452 fModules->Add(module);
453}
454
455//______________________________________________________________________________
456void AliAnalysisAlien::AddModules(TObjArray *list)
457{
458// Adding a list of modules. Checks if already existing. Becomes owned by this.
459 TIter next(list);
460 AliAnalysisTaskCfg *module;
461 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
462}
463
464//______________________________________________________________________________
465Bool_t AliAnalysisAlien::CheckDependencies()
466{
467// Check if all dependencies are satisfied. Reorder modules if needed.
468 Int_t nmodules = GetNmodules();
469 if (!nmodules) {
470 Warning("CheckDependencies", "No modules added yet to check their dependencies");
471 return kTRUE;
472 }
473 AliAnalysisTaskCfg *mod = 0;
474 AliAnalysisTaskCfg *dep = 0;
475 TString depname;
476 Int_t i, j, k;
477 for (i=0; i<nmodules; i++) {
478 mod = (AliAnalysisTaskCfg*) fModules->At(i);
479 Int_t ndeps = mod->GetNdeps();
480 Int_t istart = i;
481 for (j=0; j<ndeps; j++) {
482 depname = mod->GetDependency(j);
483 dep = GetModule(depname);
484 if (!dep) {
485 Error("CheckDependencies","Dependency %s not added for module %s",
486 depname.Data(), mod->GetName());
487 return kFALSE;
488 }
489 if (dep->NeedsDependency(mod->GetName())) {
490 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
491 mod->GetName(), dep->GetName());
492 return kFALSE;
493 }
494 Int_t idep = fModules->IndexOf(dep);
495 // The dependency task must come first
496 if (idep>i) {
497 // Remove at idep and move all objects below up one slot
498 // down to index i included.
499 fModules->RemoveAt(idep);
3e40fd4c 500 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 501 fModules->AddAt(dep, i++);
502 }
503 //Redo from istart if dependencies were inserted
504 if (i>istart) i=istart-1;
505 }
506 }
507 return kTRUE;
508}
509
510//______________________________________________________________________________
3e40fd4c 511AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
512{
513// Create the analysis manager and optionally execute the macro in filename.
514 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
515 if (mgr) return mgr;
516 mgr = new AliAnalysisManager(name);
517 mgr->SetGridHandler((AliAnalysisGrid*)this);
518 if (strlen(filename)) {
519 TString line = gSystem->ExpandPathName(filename);
520 line.Prepend(".x ");
521 gROOT->ProcessLine(line.Data());
522 }
523 return mgr;
524}
525
526//______________________________________________________________________________
77f34eae 527Int_t AliAnalysisAlien::GetNmodules() const
528{
529// Get number of modules.
530 if (!fModules) return 0;
531 return fModules->GetEntries();
532}
533
534//______________________________________________________________________________
535AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
536{
537// Get a module by name.
538 if (!fModules) return 0;
539 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
540}
541
542//______________________________________________________________________________
543Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
544{
545// Load a given module.
546 if (mod->IsLoaded()) return kTRUE;
192ddca2 547 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
548 if (!mgr) {
549 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
550 return kFALSE;
551 }
77f34eae 552 Int_t ndeps = mod->GetNdeps();
553 TString depname;
554 for (Int_t j=0; j<ndeps; j++) {
555 depname = mod->GetDependency(j);
556 AliAnalysisTaskCfg *dep = GetModule(depname);
557 if (!dep) {
558 Error("LoadModule","Dependency %s not existing for module %s",
559 depname.Data(), mod->GetName());
560 return kFALSE;
561 }
562 if (!LoadModule(dep)) {
563 Error("LoadModule","Dependency %s for module %s could not be loaded",
564 depname.Data(), mod->GetName());
565 return kFALSE;
566 }
567 }
568 // Load libraries for the module
569 if (!mod->CheckLoadLibraries()) {
570 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
571 return kFALSE;
572 }
192ddca2 573 // Check if a custom file name was requested
574 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
575
576 // Check if a custom terminate file name was requested
577 if (strlen(mod->GetTerminateFileName())) {
578 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
579 fTerminateFiles += mod->GetTerminateFileName();
580 }
581
77f34eae 582 // Execute the macro
583 if (mod->ExecuteMacro()<0) {
584 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
585 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
586 return kFALSE;
587 }
588 // Configure dependencies
589 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
590 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
591 mod->GetConfigMacro()->GetTitle(), mod->GetName());
592 return kFALSE;
593 }
5e8c7c22 594 // Adjust extra libraries
595 Int_t nlibs = mod->GetNlibs();
596 TString lib;
597 for (Int_t i=0; i<nlibs; i++) {
598 lib = mod->GetLibrary(i);
5e8c7c22 599 lib = Form("lib%s.so", lib.Data());
c9d6bd12 600 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 601 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
602 fAdditionalLibs += lib;
603 }
77f34eae 604 return kTRUE;
605}
606
607//______________________________________________________________________________
139fe6ba 608Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
609{
610// Generate the full train.
611 fAdditionalLibs = "";
612 if (!LoadModules()) return kFALSE;
613 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
614 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 615 mgr->RunLocalInit();
139fe6ba 616 mgr->PrintStatus();
617 Int_t productionMode = fProductionMode;
618 SetProductionMode();
619 TString macro = fAnalysisMacro;
620 TString executable = fExecutable;
621 TString validation = fValidationScript;
622 TString execCommand = fExecutableCommand;
623 SetAnalysisMacro(Form("%s.C", name));
624 SetExecutable(Form("%s.sh", name));
a8e31fb0 625// SetExecutableCommand("aliroot -b -q ");
139fe6ba 626 SetValidationScript(Form("%s_validation.sh", name));
627 StartAnalysis();
628 SetProductionMode(productionMode);
629 fAnalysisMacro = macro;
630 fExecutable = executable;
631 fExecutableCommand = execCommand;
632 fValidationScript = validation;
633 return kTRUE;
634}
635
636//______________________________________________________________________________
21ca8e59 637Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 638{
639// Generate test macros for a single module or for the full train.
5e8c7c22 640 fAdditionalLibs = "";
77f34eae 641 if (strlen(modname)) {
21ca8e59 642 if (!CheckDependencies()) return kFALSE;
643 AliAnalysisTaskCfg *mod = GetModule(modname);
644 if (!mod) {
645 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
646 return kFALSE;
647 }
648 if (!LoadModule(mod)) return kFALSE;
649 } else if (!LoadModules()) return kFALSE;
650 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
651 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 652 mgr->RunLocalInit();
21ca8e59 653 mgr->PrintStatus();
654 SetLocalTest(kTRUE);
655 Int_t productionMode = fProductionMode;
656 SetProductionMode();
657 TString macro = fAnalysisMacro;
658 TString executable = fExecutable;
659 TString validation = fValidationScript;
660 TString execCommand = fExecutableCommand;
661 SetAnalysisMacro(Form("%s.C", name));
662 SetExecutable(Form("%s.sh", name));
a8e31fb0 663// SetExecutableCommand("aliroot -b -q ");
21ca8e59 664 SetValidationScript(Form("%s_validation.sh", name));
665 WriteAnalysisFile();
666 WriteAnalysisMacro();
667 WriteExecutable();
668 WriteValidationScript();
669 SetLocalTest(kFALSE);
670 SetProductionMode(productionMode);
671 fAnalysisMacro = macro;
672 fExecutable = executable;
673 fExecutableCommand = execCommand;
674 fValidationScript = validation;
77f34eae 675 return kTRUE;
676}
677
678//______________________________________________________________________________
679Bool_t AliAnalysisAlien::LoadModules()
680{
681// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 682 fAdditionalLibs = "";
21ca8e59 683 Int_t nmodules = GetNmodules();
684 if (!nmodules) {
685 Warning("LoadModules", "No module to be loaded");
686 return kTRUE;
687 }
3e40fd4c 688 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
689 if (!mgr) {
690 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
691 return kFALSE;
692 }
77f34eae 693 if (!CheckDependencies()) return kFALSE;
21ca8e59 694 nmodules = GetNmodules();
77f34eae 695 AliAnalysisTaskCfg *mod;
21ca8e59 696 for (Int_t imod=0; imod<nmodules; imod++) {
697 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 698 if (!LoadModule(mod)) return kFALSE;
699 }
77f34eae 700 return kTRUE;
701}
702
703//______________________________________________________________________________
74013535 704void AliAnalysisAlien::SetRunPrefix(const char *prefix)
705{
706// Set the run number format. Can be a prefix or a format like "%09d"
707 fRunPrefix = prefix;
708 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
709}
710
711//______________________________________________________________________________
f965131e 712void AliAnalysisAlien::AddIncludePath(const char *path)
713{
714// Add include path in the remote analysis macro.
715 TString p(path);
716 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
717 else fIncludePath += Form("-I%s ", path);
718}
719
720//______________________________________________________________________________
c57f56b7 721void AliAnalysisAlien::AddRunNumber(Int_t run)
722{
723// Add a run number to the list of runs to be processed.
724 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 725 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 726}
727
728//______________________________________________________________________________
98690895 729void AliAnalysisAlien::AddRunList(const char* runList)
730{
731// Add several runs into the list of runs; they are expected to be separated by a blank character.
732 TString sList = runList;
733 TObjArray *list = sList.Tokenize(" ");
734 Int_t n = list->GetEntries();
735 for (Int_t i = 0; i < n; i++) {
736 TObjString *os = (TObjString*)list->At(i);
737 AddRunNumber(os->GetString().Atoi());
738 }
739 delete list;
740}
741
742//______________________________________________________________________________
ee75cfc3 743void AliAnalysisAlien::AddRunNumber(const char* run)
744{
745// Add a run number to the list of runs to be processed.
7aa5d539 746 TString runs = run;
747 TObjString *os;
748 TObjArray *arr = runs.Tokenize(" ");
749 TIter next(arr);
750 TString prefix;
751 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
752 while ((os=(TObjString*)next())){
753 if (fRunNumbers.Length()) fRunNumbers += " ";
754 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
755 }
756 delete arr;
ee75cfc3 757}
758
759//______________________________________________________________________________
c57f56b7 760void AliAnalysisAlien::AddDataFile(const char *lfn)
761{
762// Adds a data file to the input to be analysed. The file should be a valid LFN
763// or point to an existing file in the alien workdir.
764 if (!fInputFiles) fInputFiles = new TObjArray();
765 fInputFiles->Add(new TObjString(lfn));
766}
648174cf 767
768//______________________________________________________________________________
769void AliAnalysisAlien::AddExternalPackage(const char *package)
770{
771// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
772 if (fExternalPackages) fExternalPackages += " ";
773 fExternalPackages += package;
774}
775
c57f56b7 776//______________________________________________________________________________
777Bool_t AliAnalysisAlien::Connect()
778{
779// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
780 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 781 if (fProductionMode) return kTRUE;
c57f56b7 782 if (!gGrid) {
783 Info("Connect", "Trying to connect to AliEn ...");
784 TGrid::Connect("alien://");
785 }
786 if (!gGrid || !gGrid->IsConnected()) {
787 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
788 return kFALSE;
789 }
790 fUser = gGrid->GetUser();
791 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
792 return kTRUE;
793}
794
795//______________________________________________________________________________
796void AliAnalysisAlien::CdWork()
797{
798// Check validity of alien workspace. Create directory if possible.
799 if (!Connect()) {
800 Error("CdWork", "Alien connection required");
801 return;
802 }
803 TString homedir = gGrid->GetHomeDirectory();
804 TString workdir = homedir + fGridWorkingDir;
923e2ca5 805 if (DirectoryExists(workdir)) {
806 gGrid->Cd(workdir);
807 return;
808 }
809 // Work directory not existing - create it
810 gGrid->Cd(homedir);
b93f8109 811 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 812 gGrid->Cd(fGridWorkingDir);
d3b18c4c 813 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 814 } else {
d3b18c4c 815 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 816 workdir.Data(), homedir.Data());
817 fGridWorkingDir = "";
818 }
c57f56b7 819}
820
821//______________________________________________________________________________
348be253 822Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
823{
824// Check if file copying is possible.
d3b18c4c 825 if (fProductionMode) return kTRUE;
b5ae4fe0 826 TString salienpath(alienpath);
827 if (salienpath.Contains(" ")) {
828 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
829 return kFALSE;
830 }
348be253 831 if (!Connect()) {
832 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
833 return kFALSE;
834 }
d3339be3 835 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
836 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 837 // Check if alien_CLOSE_SE is defined
838 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
839 if (!closeSE.IsNull()) {
840 Info("CheckFileCopy", "Your current close storage is pointing to: \
841 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
842 } else {
843 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
844 }
845 // Check if grid directory exists.
846 if (!DirectoryExists(alienpath)) {
847 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
848 return kFALSE;
849 }
b5ae4fe0 850 TString stest = "plugin_test_copy";
851 TFile f(stest, "RECREATE");
348be253 852 // User may not have write permissions to current directory
853 if (f.IsZombie()) {
854 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
855 gSystem->WorkingDirectory());
856 return kFALSE;
857 }
858 f.Close();
b5ae4fe0 859 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
860 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 861 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 862 \n# 1. Make sure you have write permissions there. If this is the case: \
863 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
864 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
865 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
866 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 867 gSystem->Unlink(stest.Data());
348be253 868 return kFALSE;
869 }
b5ae4fe0 870 gSystem->Unlink(stest.Data());
871 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 872 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 873 return kTRUE;
874}
875
876//______________________________________________________________________________
c57f56b7 877Bool_t AliAnalysisAlien::CheckInputData()
878{
879// Check validity of input data. If necessary, create xml files.
d3b18c4c 880 if (fProductionMode) return kTRUE;
a8739e8a 881 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 882 if (!fGridDataDir.Length()) {
883 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
884 return kFALSE;
885 }
9f020952 886 if (fMergeViaJDL) {
887 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
888 return kFALSE;
889 }
d2a409b2 890 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 891 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
892 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 893 return kTRUE;
c57f56b7 894 }
895 // Process declared files
a2f5fc01 896 Bool_t isCollection = kFALSE;
897 Bool_t isXml = kFALSE;
898 Bool_t useTags = kFALSE;
c57f56b7 899 Bool_t checked = kFALSE;
d3b18c4c 900 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 901 TString file;
902 TString workdir = gGrid->GetHomeDirectory();
903 workdir += fGridWorkingDir;
904 if (fInputFiles) {
905 TObjString *objstr;
906 TIter next(fInputFiles);
907 while ((objstr=(TObjString*)next())) {
908 file = workdir;
909 file += "/";
910 file += objstr->GetString();
911 // Store full lfn path
912 if (FileExists(file)) objstr->SetString(file);
913 else {
914 file = objstr->GetName();
915 if (!FileExists(objstr->GetName())) {
916 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
917 objstr->GetName(), workdir.Data());
918 return kFALSE;
919 }
920 }
921 Bool_t iscoll, isxml, usetags;
922 CheckDataType(file, iscoll, isxml, usetags);
923 if (!checked) {
924 checked = kTRUE;
a2f5fc01 925 isCollection = iscoll;
926 isXml = isxml;
927 useTags = usetags;
928 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 929 } else {
a2f5fc01 930 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 931 Error("CheckInputData", "Some conflict was found in the types of inputs");
932 return kFALSE;
933 }
934 }
935 }
936 }
937 // Process requested run numbers
a8739e8a 938 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 939 // Check validity of alien data directory
940 if (!fGridDataDir.Length()) {
941 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
942 return kFALSE;
943 }
923e2ca5 944 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 945 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
946 return kFALSE;
947 }
a2f5fc01 948 if (isCollection) {
c57f56b7 949 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
950 return kFALSE;
951 }
952
a2f5fc01 953 if (checked && !isXml) {
c57f56b7 954 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
955 return kFALSE;
956 }
957 // Check validity of run number(s)
958 TObjArray *arr;
959 TObjString *os;
74013535 960 TString format;
319593fb 961 Int_t nruns = 0;
904f9f5f 962 TString schunk, schunk2;
c57f56b7 963 TString path;
964 if (!checked) {
965 checked = kTRUE;
a2f5fc01 966 useTags = fDataPattern.Contains("tag");
967 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 968 }
a2f5fc01 969 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 970 Error("CheckInputData", "Cannot mix input files using/not using tags");
971 return kFALSE;
972 }
973 if (fRunNumbers.Length()) {
a8739e8a 974 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 975 arr = fRunNumbers.Tokenize(" ");
976 TIter next(arr);
977 while ((os=(TObjString*)next())) {
978 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 979 if (!DirectoryExists(path)) {
980 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 981 continue;
c57f56b7 982 }
983 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
984 TString msg = "\n##### file: ";
985 msg += path;
986 msg += " type: xml_collection;";
a2f5fc01 987 if (useTags) msg += " using_tags: Yes";
c57f56b7 988 else msg += " using_tags: No";
23329835 989 Info("CheckDataType", "%s", msg.Data());
319593fb 990 if (fNrunsPerMaster<2) {
d2a409b2 991 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 992 } else {
993 nruns++;
994 if (((nruns-1)%fNrunsPerMaster) == 0) {
995 schunk = os->GetString();
996 }
997 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
998 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 999 AddDataFile(schunk);
319593fb 1000 }
c57f56b7 1001 }
1002 delete arr;
a8739e8a 1003 } else {
1004 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1005 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1006 format = Form("%%s/%s ", fRunPrefix.Data());
1007 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1008 if (!DirectoryExists(path)) {
a8739e8a 1009 continue;
1010 }
74013535 1011 format = Form("%%s/%s.xml", fRunPrefix.Data());
1012 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1013 TString msg = "\n##### file: ";
1014 msg += path;
1015 msg += " type: xml_collection;";
a2f5fc01 1016 if (useTags) msg += " using_tags: Yes";
a8739e8a 1017 else msg += " using_tags: No";
23329835 1018 Info("CheckDataType", "%s", msg.Data());
319593fb 1019 if (fNrunsPerMaster<2) {
74013535 1020 format = Form("%s.xml", fRunPrefix.Data());
1021 AddDataFile(Form(format.Data(),irun));
319593fb 1022 } else {
1023 nruns++;
1024 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1025 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1026 }
74013535 1027 format = Form("_%s.xml", fRunPrefix.Data());
1028 schunk2 = Form(format.Data(), irun);
319593fb 1029 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1030 schunk += schunk2;
d2a409b2 1031 AddDataFile(schunk);
319593fb 1032 }
a8739e8a 1033 }
904f9f5f 1034 if (!fInputFiles) {
1035 schunk += schunk2;
1036 AddDataFile(schunk);
1037 }
c57f56b7 1038 }
1039 return kTRUE;
1040}
1041
1042//______________________________________________________________________________
43b807f6 1043Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1044{
1045// Copy data from the given grid directory according a pattern and make a local
1046// dataset.
2ba4c287 1047// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1048 if (!Connect()) {
1049 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
1050 return kFALSE;
1051 }
1052 if (!DirectoryExists(griddir)) {
1053 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
1054 return kFALSE;
1055 }
1056 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1057 printf("Running command: %s\n", command.Data());
1058 TGridResult *res = gGrid->Command(command);
1059 Int_t nfound = res->GetEntries();
1060 if (!nfound) {
1061 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
1062 return kFALSE;
1063 }
1064 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1065
1066 // archives
1067 TObjArray* additionalArchives = 0;
1068 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1069 additionalArchives = TString(archivefile).Tokenize(";");
1070 archivefile = additionalArchives->At(0)->GetName();
1071 additionalArchives->RemoveAt(0);
1072 additionalArchives->Compress();
1073 }
1074
21ca8e59 1075 // Copy files locally
1076 ofstream out;
1077 out.open(output, ios::out);
1078 TMap *map;
1079 TString turl, dirname, filename, temp;
1080 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1081 gSystem->MakeDirectory(outputdir);
1082 gSystem->ChangeDirectory(outputdir);
21ca8e59 1083 for (Int_t i=0; i<nfound; i++) {
1084 map = (TMap*)res->At(i);
1085 turl = map->GetValue("turl")->GetName();
1086 filename = gSystem->BaseName(turl.Data());
1087 dirname = gSystem->DirName(turl.Data());
1088 dirname = gSystem->BaseName(dirname.Data());
1089 gSystem->MakeDirectory(dirname);
43b807f6 1090
1091 TString source(turl);
1092 TString targetFileName(filename);
1093
1094 if (strlen(archivefile) > 0) {
1095 // TODO here the archive in which the file resides should be determined
1096 // however whereis returns only a guid, and guid2lfn does not work
1097 // Therefore we use the one provided as argument for now
1098 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1099 targetFileName = archivefile;
1100 }
1101 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
2ba4c287 1102 Bool_t success = kTRUE;
1103 if (additionalArchives)
1104 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++)
1105 {
1106 TString target;
1107 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1108 gSystem->MakeDirectory(gSystem->DirName(target));
1109 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1110 }
1111
1112 if (success) {
1113 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1114 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1115 }
21ca8e59 1116 }
1117 }
1118 gSystem->ChangeDirectory(cdir);
1119 delete res;
2ba4c287 1120 if (additionalArchives)
1121 delete additionalArchives;
21ca8e59 1122 return kTRUE;
1123}
1124
1125//______________________________________________________________________________
c57f56b7 1126Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1127{
1128// Create dataset for the grid data directory + run number.
3af861ff 1129 const Int_t gMaxEntries = 15000;
d3b18c4c 1130 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1131 if (!Connect()) {
1132 Error("CreateDataset", "Cannot create dataset with no grid connection");
1133 return kFALSE;
1134 }
1135
1136 // Cd workspace
d3b18c4c 1137 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1138 TString workdir = gGrid->GetHomeDirectory();
1139 workdir += fGridWorkingDir;
1140
1141 // Compose the 'find' command arguments
74013535 1142 TString format;
c57f56b7 1143 TString command;
1144 TString options = "-x collection ";
bb885a9e 1145 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1146 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1147 TString conditions = "";
3af861ff 1148 Int_t nstart = 0;
1149 Int_t ncount = 0;
1150 Int_t stage = 0;
c57f56b7 1151 TString file;
1152 TString path;
319593fb 1153 Int_t nruns = 0;
904f9f5f 1154 TString schunk, schunk2;
ab254fd1 1155 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1156 if (!fRunNumbers.Length() && !fRunRange[0]) {
1157 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1158 // Make a single data collection from data directory.
1159 path = fGridDataDir;
923e2ca5 1160 if (!DirectoryExists(path)) {
d2a409b2 1161 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1162 return kFALSE;
1163 }
923e2ca5 1164// CdWork();
d2a409b2 1165 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1166 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1167 while (1) {
1168 ncount = 0;
1169 stage++;
1170 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1171 command = "find ";
1172 command += Form("%s -o %d ",options.Data(), nstart);
1173 command += path;
1174 command += " ";
1175 command += pattern;
1176 command += conditions;
1177 printf("command: %s\n", command.Data());
1178 TGridResult *res = gGrid->Command(command);
1179 if (res) delete res;
1180 // Write standard output to file
1181 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1182 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1183 Bool_t nullFile = kFALSE;
1184 if (!hasGrep) {
1185 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1186 } else {
1187 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1188 if (nullFile) {
1189 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1190 gSystem->Exec("rm -f __tmp*");
1191 return kFALSE;
1192 }
1193 TString line;
1194 ifstream in;
1195 in.open("__tmp__");
1196 in >> line;
1197 in.close();
1198 gSystem->Exec("rm -f __tmp__");
1199 ncount = line.Atoi();
1200 }
1201 }
1202 if (ncount == gMaxEntries) {
1203 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1204 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1205 if (!cbase) cbase = cadd;
1206 else {
1207 cbase->Add(cadd);
1208 delete cadd;
defd7a3a 1209 }
3af861ff 1210 nstart += ncount;
1211 } else {
1212 if (cbase) {
1213 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1214 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1215 cbase->Add(cadd);
1216 delete cadd;
1217 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1218 delete cbase; cbase = 0;
1219 } else {
1220 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1221 }
1222 gSystem->Exec("rm -f __tmp*");
1223 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1224 break;
1225 }
0f389141 1226 }
1227 Bool_t fileExists = FileExists(file);
1228 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1229 // Copy xml file to alien space
0f389141 1230 if (fileExists) gGrid->Rm(file);
d2a409b2 1231 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1232 if (!FileExists(file)) {
1233 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1234 return kFALSE;
1235 }
1236 // Update list of files to be processed.
1237 }
1238 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1239 return kTRUE;
1240 }
c57f56b7 1241 // Several runs
a2f5fc01 1242 Bool_t nullResult = kTRUE;
a8739e8a 1243 if (fRunNumbers.Length()) {
1244 TObjArray *arr = fRunNumbers.Tokenize(" ");
1245 TObjString *os;
1246 TIter next(arr);
1247 while ((os=(TObjString*)next())) {
3af861ff 1248 nstart = 0;
1249 stage = 0;
dc030a24 1250 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1251 if (!DirectoryExists(path)) continue;
1252// CdWork();
a8739e8a 1253 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1254 else file = Form("%s.xml", os->GetString().Data());
319593fb 1255 // If local collection file does not exist, create it via 'find' command.
3af861ff 1256 while (1) {
1257 ncount = 0;
1258 stage++;
1259 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1260 command = "find ";
1261 command += Form("%s -o %d ",options.Data(), nstart);
1262 command += path;
1263 command += pattern;
1264 command += conditions;
1265 TGridResult *res = gGrid->Command(command);
1266 if (res) delete res;
1267 // Write standard output to file
1268 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1269 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1270 Bool_t nullFile = kFALSE;
1271 if (!hasGrep) {
1272 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1273 } else {
1274 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1275 if (nullFile) {
1276 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1277 gSystem->Exec("rm -f __tmp*");
1278 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1279 break;
3af861ff 1280 }
1281 TString line;
1282 ifstream in;
1283 in.open("__tmp__");
1284 in >> line;
1285 in.close();
1286 gSystem->Exec("rm -f __tmp__");
1287 ncount = line.Atoi();
1288 }
1289 nullResult = kFALSE;
1290 }
1291 if (ncount == gMaxEntries) {
1292 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1293 if (fNrunsPerMaster > 1) {
1294 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1295 file.Data(),gMaxEntries);
1296 return kFALSE;
1297 }
1298 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1299 if (!cbase) cbase = cadd;
1300 else {
1301 cbase->Add(cadd);
1302 delete cadd;
defd7a3a 1303 }
3af861ff 1304 nstart += ncount;
1305 } else {
c89a84bc 1306 if (cbase && fNrunsPerMaster<2) {
3af861ff 1307 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1308 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1309 cbase->Add(cadd);
1310 delete cadd;
1311 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1312 delete cbase; cbase = 0;
1313 } else {
1314 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1315 }
1316 gSystem->Exec("rm -f __tmp*");
1317 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1318 break;
d3339be3 1319 }
3af861ff 1320 }
a8739e8a 1321 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1322 // Check if there is one run per master job.
1323 if (fNrunsPerMaster<2) {
1324 if (FileExists(file)) {
0f389141 1325 if (fOverwriteMode) gGrid->Rm(file);
1326 else {
1327 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1328 continue;
1329 }
319593fb 1330 }
1331 // Copy xml file to alien space
1332 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1333 if (!FileExists(file)) {
1334 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1335 delete arr;
1336 return kFALSE;
1337 }
1338 } else {
1339 nruns++;
1340 if (((nruns-1)%fNrunsPerMaster) == 0) {
1341 schunk = os->GetString();
1342 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1343 } else {
1344 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1345 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1346 cbase->Add(cadd);
1347 delete cadd;
1348 }
1349 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1350 continue;
1351 }
1352 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1353 if (FileExists(schunk)) {
1354 if (fOverwriteMode) gGrid->Rm(file);
1355 else {
1356 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1357 continue;
1358 }
319593fb 1359 }
84fcd93f 1360 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1361 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1362 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1363 if (!FileExists(schunk)) {
1364 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1365 delete arr;
1366 return kFALSE;
1367 }
d3339be3 1368 }
a8739e8a 1369 }
1370 delete arr;
a2f5fc01 1371 if (nullResult) {
d3339be3 1372 Error("CreateDataset", "No valid dataset corresponding to the query!");
1373 return kFALSE;
1374 }
a8739e8a 1375 } else {
1376 // Process a full run range.
1377 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1378 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1379 nstart = 0;
1380 stage = 0;
74013535 1381 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1382 if (!DirectoryExists(path)) continue;
1383// CdWork();
74013535 1384 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1385 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1386 else file = Form(format.Data(), irun);
0f389141 1387 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1388 if (fOverwriteMode) gGrid->Rm(file);
1389 else {
1390 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1391 continue;
1392 }
a8739e8a 1393 }
319593fb 1394 // If local collection file does not exist, create it via 'find' command.
3af861ff 1395 while (1) {
1396 ncount = 0;
1397 stage++;
1398 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1399 command = "find ";
1400 command += Form("%s -o %d ",options.Data(), nstart);
1401 command += path;
1402 command += pattern;
1403 command += conditions;
1404 TGridResult *res = gGrid->Command(command);
1405 if (res) delete res;
1406 // Write standard output to file
1407 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1408 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1409 Bool_t nullFile = kFALSE;
1410 if (!hasGrep) {
1411 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1412 } else {
1413 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1414 if (nullFile) {
1415 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1416 gSystem->Exec("rm -f __tmp*");
1604c9ed 1417 break;
3af861ff 1418 }
1419 TString line;
1420 ifstream in;
1421 in.open("__tmp__");
1422 in >> line;
1423 in.close();
1424 gSystem->Exec("rm -f __tmp__");
1425 ncount = line.Atoi();
1426 }
1427 nullResult = kFALSE;
1428 }
1429 if (ncount == gMaxEntries) {
1430 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1431 if (fNrunsPerMaster > 1) {
1432 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1433 file.Data(),gMaxEntries);
1434 return kFALSE;
1435 }
1436 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1437 if (!cbase) cbase = cadd;
1438 else {
1439 cbase->Add(cadd);
1440 delete cadd;
defd7a3a 1441 }
3af861ff 1442 nstart += ncount;
1443 } else {
c89a84bc 1444 if (cbase && fNrunsPerMaster<2) {
3af861ff 1445 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1446 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1447 cbase->Add(cadd);
1448 delete cadd;
1449 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1450 delete cbase; cbase = 0;
1451 } else {
1452 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1453 }
1454 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1455 break;
d3339be3 1456 }
319593fb 1457 }
a8739e8a 1458 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1459 // Check if there is one run per master job.
1460 if (fNrunsPerMaster<2) {
1461 if (FileExists(file)) {
0f389141 1462 if (fOverwriteMode) gGrid->Rm(file);
1463 else {
1464 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1465 continue;
1466 }
319593fb 1467 }
1468 // Copy xml file to alien space
1469 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1470 if (!FileExists(file)) {
1471 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1472 return kFALSE;
1473 }
1474 } else {
1475 nruns++;
95e5b448 1476 // Check if the collection for the chunk exist locally.
1477 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1478 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1479 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1480 else continue;
1481 }
84fcd93f 1482 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1483 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1484 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1485 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1486 } else {
1487 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1488 cbase->Add(cadd);
1489 delete cadd;
1490 }
74013535 1491 format = Form("%%s_%s.xml", fRunPrefix.Data());
1492 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1493 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1494 continue;
1495 }
904f9f5f 1496 schunk = schunk2;
319593fb 1497 if (FileExists(schunk)) {
0f389141 1498 if (fOverwriteMode) gGrid->Rm(schunk);
1499 else {
1500 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1501 continue;
1502 }
319593fb 1503 }
84fcd93f 1504 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1505 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1506 if (FileExists(schunk)) {
0f389141 1507 if (fOverwriteMode) gGrid->Rm(schunk);
1508 else {
1509 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1510 continue;
1511 }
95e5b448 1512 }
319593fb 1513 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1514 if (!FileExists(schunk)) {
1515 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1516 return kFALSE;
1517 }
1518 }
c57f56b7 1519 }
a2f5fc01 1520 if (nullResult) {
d3339be3 1521 Error("CreateDataset", "No valid dataset corresponding to the query!");
1522 return kFALSE;
1523 }
a8739e8a 1524 }
c57f56b7 1525 return kTRUE;
1526}
1527
1528//______________________________________________________________________________
1529Bool_t AliAnalysisAlien::CreateJDL()
1530{
1531// Generate a JDL file according to current settings. The name of the file is
1532// specified by fJDLName.
1533 Bool_t error = kFALSE;
1534 TObjArray *arr = 0;
1535 Bool_t copy = kTRUE;
d3b18c4c 1536 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1537 Bool_t generate = kTRUE;
1538 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1539 if (!Connect()) {
1540 Error("CreateJDL", "Alien connection required");
1541 return kFALSE;
1542 }
1543 // Check validity of alien workspace
d3b18c4c 1544 TString workdir;
1545 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1546 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1547 workdir += fGridWorkingDir;
1548 if (generate) {
1549 TObjString *os;
1550 if (!fInputFiles) {
1551 Error("CreateJDL()", "Define some input files for your analysis.");
1552 error = kTRUE;
1553 }
1554 // Compose list of input files
1555 // Check if output files were defined
1556 if (!fOutputFiles.Length()) {
1557 Error("CreateJDL", "You must define at least one output file");
1558 error = kTRUE;
1559 }
1560 // Check if an output directory was defined and valid
1561 if (!fGridOutputDir.Length()) {
1562 Error("CreateJDL", "You must define AliEn output directory");
1563 error = kTRUE;
1564 } else {
d3b18c4c 1565 if (!fProductionMode) {
1566 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1567 if (!DirectoryExists(fGridOutputDir)) {
1568 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1569 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1570 } else {
1571 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1572 // error = kTRUE;
1573 }
67a3b254 1574 } else {
1575 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1576 }
1577 gGrid->Cd(workdir);
1578 }
c57f56b7 1579 }
1580 // Exit if any error up to now
1581 if (error) return kFALSE;
1582 // Set JDL fields
0f389141 1583 if (!fUser.IsNull()) {
1584 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1585 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1586 }
1587 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1588 TString mergeExec = fExecutable;
1589 mergeExec.ReplaceAll(".sh", "_merge.sh");
1590 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1591 mergeExec.ReplaceAll(".sh", ".C");
1592 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1593 if (!fArguments.IsNull())
1594 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1595 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1596 else {
1597 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1598 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1599 }
b3e07543 1600
1f0d1ca2 1601 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1602 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1603 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1604 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1605
0f389141 1606 if (fMaxInitFailed > 0) {
c57f56b7 1607 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1608 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1609 }
1610 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1611 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1612 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1613 }
1614 if (!IsOneStageMerging()) {
f47d5cb4 1615 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1616 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1617 }
1618 if (fSplitMode.Length()) {
c57f56b7 1619 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1620 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1621 }
1622 fMergingJDL->SetValue("Split", "\"se\"");
1623 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1624 if (!fAliROOTVersion.IsNull()) {
1625 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1626 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1627 }
1628 if (!fROOTVersion.IsNull()) {
c57f56b7 1629 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1630 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1631 }
1632 if (!fAPIVersion.IsNull()) {
c57f56b7 1633 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1634 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1635 }
648174cf 1636 if (!fExternalPackages.IsNull()) {
1637 arr = fExternalPackages.Tokenize(" ");
1638 TIter next(arr);
1639 while ((os=(TObjString*)next())) {
1640 TString pkgname = os->GetString();
1641 Int_t index = pkgname.Index("::");
1642 TString pkgversion = pkgname(index+2, pkgname.Length());
1643 pkgname.Remove(index);
1644 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1645 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1646 }
1647 delete arr;
1648 }
0f389141 1649 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1650 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1651 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1652 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1653 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1654 TString analysisFile = fExecutable;
1655 analysisFile.ReplaceAll(".sh", ".root");
1656 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1657 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1658 if (fAdditionalLibs.Length()) {
1659 arr = fAdditionalLibs.Tokenize(" ");
1660 TIter next(arr);
1661 while ((os=(TObjString*)next())) {
1662 if (os->GetString().Contains(".so")) continue;
1663 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1664 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1665 }
1666 delete arr;
1667 }
4e5c5506 1668 if (fPackages) {
1669 TIter next(fPackages);
1670 TObject *obj;
0f389141 1671 while ((obj=next())) {
4e5c5506 1672 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1673 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1674 }
4e5c5506 1675 }
c57f56b7 1676 if (fOutputArchive.Length()) {
e4d9a069 1677 TString outputArchive = fOutputArchive;
1678 if (!fRegisterExcludes.IsNull()) {
1679 arr = fRegisterExcludes.Tokenize(" ");
1680 TIter next1(arr);
1681 while ((os=(TObjString*)next1())) {
1682 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1683 outputArchive.ReplaceAll(os->GetString(),"");
1684 }
1685 delete arr;
1686 }
1687 arr = outputArchive.Tokenize(" ");
c57f56b7 1688 TIter next(arr);
0f389141 1689 Bool_t first = kTRUE;
1690 const char *comment = "Files to be archived";
1691 const char *comment1 = comment;
1692 while ((os=(TObjString*)next())) {
1693 if (!first) comment = NULL;
1694 if (!os->GetString().Contains("@") && fCloseSE.Length())
1695 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1696 else
1697 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1698 first = kFALSE;
1699 }
c57f56b7 1700 delete arr;
f790bc1b 1701 // Output archive for the merging jdl
f790bc1b 1702 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1703 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1704 // Add normal output files, extra files + terminate files
1705 TString files = GetListOfFiles("outextter");
37d21c01 1706 // Do not register files in fRegisterExcludes
1707 if (!fRegisterExcludes.IsNull()) {
1708 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1709 TIter next1(arr);
1710 while ((os=(TObjString*)next1())) {
1711 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1712 files.ReplaceAll(os->GetString(),"");
1713 }
66d71516 1714 delete arr;
0f389141 1715 }
f790bc1b 1716 files.ReplaceAll(".root", "*.root");
b83f84f0 1717 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1718 } else {
95f84779 1719 TString files = fOutputArchive;
1720 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1721 outputArchive = files;
f790bc1b 1722 }
0f389141 1723 arr = outputArchive.Tokenize(" ");
705adb3e 1724 TIter next2(arr);
0f389141 1725 comment = comment1;
1726 first = kTRUE;
705adb3e 1727 while ((os=(TObjString*)next2())) {
0f389141 1728 if (!first) comment = NULL;
0cdf65a8 1729 TString currentfile = os->GetString();
0cdf65a8 1730 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1731 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1732 else
0cdf65a8 1733 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1734 first = kFALSE;
1735 }
1736 delete arr;
c57f56b7 1737 }
149d288c 1738 arr = fOutputFiles.Tokenize(",");
c57f56b7 1739 TIter next(arr);
0f389141 1740 Bool_t first = kTRUE;
f790bc1b 1741 const char *comment = "Files to be saved";
43da816a 1742 while ((os=(TObjString*)next())) {
e1eaf596 1743 // Ignore ouputs in jdl that are also in outputarchive
1744 TString sout = os->GetString();
f790bc1b 1745 sout.ReplaceAll("*", "");
1746 sout.ReplaceAll(".root", "");
e1eaf596 1747 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1748 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1749 // Ignore fRegisterExcludes
1750 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1751 if (!first) comment = NULL;
43da816a 1752 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1753 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1754 else
0f389141 1755 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1756 first = kFALSE;
f790bc1b 1757 if (fMergeExcludes.Contains(sout)) continue;
1758 if (!os->GetString().Contains("@") && fCloseSE.Length())
1759 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1760 else
1761 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1762 }
c57f56b7 1763 delete arr;
0f389141 1764 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1765 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1766 TString validationScript = fValidationScript;
0f389141 1767 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1768 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1769 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1770 if (fMasterResubmitThreshold) {
1771 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1772 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1773 }
d2a409b2 1774 // Write a jdl with 2 input parameters: collection name and output dir name.
1775 WriteJDL(copy);
c57f56b7 1776 }
1777 // Copy jdl to grid workspace
a8739e8a 1778 if (copy) {
b5e4aaa7 1779 // Check if an output directory was defined and valid
1780 if (!fGridOutputDir.Length()) {
1781 Error("CreateJDL", "You must define AliEn output directory");
1782 return kFALSE;
1783 } else {
1784 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1785 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1786 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1787 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1788 } else {
1789 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1790 return kFALSE;
1791 }
1792 }
1793 gGrid->Cd(workdir);
1794 }
648174cf 1795 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1796 TString mergeJDLName = fExecutable;
1797 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1798 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1799 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1800 if (fProductionMode) {
648174cf 1801 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1802 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1803 }
648174cf 1804 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1805 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1806 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1807 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1808 Fatal("","Terminating");
1809// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1810 if (fMergeViaJDL) {
1811 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1812// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1813 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1814 Fatal("","Terminating");
0f389141 1815 }
648174cf 1816 }
c57f56b7 1817 if (fAdditionalLibs.Length()) {
e7c71df0 1818 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1819 TObjString *os;
1820 TIter next(arr);
1821 while ((os=(TObjString*)next())) {
c57f56b7 1822 if (os->GetString().Contains(".so")) continue;
4e5c5506 1823 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1824 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1825// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1826 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1827 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1828 Fatal("","Terminating");
c57f56b7 1829 }
1830 delete arr;
1831 }
4e5c5506 1832 if (fPackages) {
1833 TIter next(fPackages);
1834 TObject *obj;
1835 while ((obj=next())) {
fdbbc7be 1836 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1837 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1838// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1839 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1840 Form("%s/%s", workdir.Data(), obj->GetName())))
1841 Fatal("","Terminating");
4e5c5506 1842 }
1843 }
c57f56b7 1844 }
1845 return kTRUE;
1846}
1847
1848//______________________________________________________________________________
d2a409b2 1849Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1850{
1851// Writes one or more JDL's corresponding to findex. If findex is negative,
1852// all run numbers are considered in one go (jdl). For non-negative indices
1853// they correspond to the indices in the array fInputFiles.
1854 if (!fInputFiles) return kFALSE;
d3b18c4c 1855 TObject *os;
1856 TString workdir;
1857 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1858 workdir += fGridWorkingDir;
f47d5cb4 1859 TString stageName = "$2";
1860 if (fProductionMode) stageName = "$4";
1861 if (!fMergeDirName.IsNull()) {
1862 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1863 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1864 } else {
1865 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1866 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1867 }
d3b18c4c 1868 if (fProductionMode) {
a8739e8a 1869 TIter next(fInputFiles);
b3e07543 1870 while ((os=next())) {
d3b18c4c 1871 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1872 }
56754bf6 1873 if (!fOutputToRunNo)
1874 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1875 else
1876 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1877 } else {
1878 if (!fRunNumbers.Length() && !fRunRange[0]) {
1879 // One jdl with no parameters in case input data is specified by name.
1880 TIter next(fInputFiles);
1881 while ((os=next()))
1882 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1883 if (!fOutputSingle.IsNull())
1884 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1885 else {
1886 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1887 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1888 }
1889 } else {
1890 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1891 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1892 if (!fOutputSingle.IsNull()) {
1893 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1894 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1895 } else {
1896 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1897 }
1898 }
a8739e8a 1899 }
1900
a8739e8a 1901 // Generate the JDL as a string
1902 TString sjdl = fGridJDL->Generate();
0f389141 1903 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1904 // Final merge jdl
1905 if (!fMergeDirName.IsNull()) {
1906 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1907 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1908 } else {
1909 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1910 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1911 }
b3e07543 1912 TString sjdl2 = fMergingJDL->Generate();
1913 Int_t index, index1;
a8739e8a 1914 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1915 sjdl.ReplaceAll("(member", "\n (member");
1916 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1917 sjdl.ReplaceAll("{", "{\n ");
1918 sjdl.ReplaceAll("};", "\n};");
1919 sjdl.ReplaceAll("{\n \n", "{\n");
1920 sjdl.ReplaceAll("\n\n", "\n");
1921 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1922 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1923 sjdl1.ReplaceAll("(member", "\n (member");
1924 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1925 sjdl1.ReplaceAll("{", "{\n ");
1926 sjdl1.ReplaceAll("};", "\n};");
1927 sjdl1.ReplaceAll("{\n \n", "{\n");
1928 sjdl1.ReplaceAll("\n\n", "\n");
1929 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1930 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1931 sjdl2.ReplaceAll("(member", "\n (member");
1932 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1933 sjdl2.ReplaceAll("{", "{\n ");
1934 sjdl2.ReplaceAll("};", "\n};");
1935 sjdl2.ReplaceAll("{\n \n", "{\n");
1936 sjdl2.ReplaceAll("\n\n", "\n");
1937 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1938 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1939 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1940 index = sjdl.Index("JDLVariables");
1941 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1942 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1943 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1944 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1945 index = fJobTag.Index(":");
1946 if (index < 0) index = fJobTag.Length();
1947 TString jobTag = fJobTag;
f47d5cb4 1948 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1949 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1950 if (fProductionMode) {
1951 sjdl1.Prepend("# Generated merging jdl (production mode) \
1952 \n# $1 = full alien path to output directory to be merged \
1953 \n# $2 = train number \
1954 \n# $3 = production (like LHC10b) \
1955 \n# $4 = merging stage \
1956 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1957 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1958 sjdl2.Prepend("# Generated merging jdl \
1959 \n# $1 = full alien path to output directory to be merged \
1960 \n# $2 = train number \
1961 \n# $3 = production (like LHC10b) \
1962 \n# $4 = merging stage \
1963 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1964 } else {
1965 sjdl1.Prepend("# Generated merging jdl \
1966 \n# $1 = full alien path to output directory to be merged \
1967 \n# $2 = merging stage \
03335324 1968 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1969 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1970 sjdl2.Prepend("# Generated merging jdl \
1971 \n# $1 = full alien path to output directory to be merged \
1972 \n# $2 = merging stage \
03335324 1973 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1974 }
0f389141 1975 index = sjdl1.Index("JDLVariables");
1976 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1977 index = sjdl2.Index("JDLVariables");
1978 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1979 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1980 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1981 index = sjdl2.Index("Split =");
1982 if (index>=0) {
1983 index1 = sjdl2.Index("\n", index);
1984 sjdl2.Remove(index, index1-index+1);
1985 }
1986 index = sjdl2.Index("SplitMaxInputFileNumber");
1987 if (index>=0) {
1988 index1 = sjdl2.Index("\n", index);
1989 sjdl2.Remove(index, index1-index+1);
1990 }
1991 index = sjdl2.Index("InputDataCollection");
1992 if (index>=0) {
1993 index1 = sjdl2.Index(";", index);
1994 sjdl2.Remove(index, index1-index+1);
1995 }
1996 index = sjdl2.Index("InputDataListFormat");
1997 if (index>=0) {
1998 index1 = sjdl2.Index("\n", index);
1999 sjdl2.Remove(index, index1-index+1);
2000 }
2001 index = sjdl2.Index("InputDataList");
2002 if (index>=0) {
2003 index1 = sjdl2.Index("\n", index);
2004 sjdl2.Remove(index, index1-index+1);
2005 }
f47d5cb4 2006 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2007 // Write jdl to file
a8739e8a 2008 ofstream out;
d2a409b2 2009 out.open(fJDLName.Data(), ios::out);
a8739e8a 2010 if (out.bad()) {
d3b18c4c 2011 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2012 return kFALSE;
2013 }
2014 out << sjdl << endl;
b3e07543 2015 out.close();
0f389141 2016 TString mergeJDLName = fExecutable;
2017 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2018 if (fMergeViaJDL) {
2019 ofstream out1;
2020 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2021 if (out1.bad()) {
d3b18c4c 2022 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2023 return kFALSE;
2024 }
2025 out1 << sjdl1 << endl;
b3e07543 2026 out1.close();
2027 ofstream out2;
2028 TString finalJDL = mergeJDLName;
2029 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2030 out2.open(finalJDL.Data(), ios::out);
2031 if (out2.bad()) {
2032 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2033 return kFALSE;
2034 }
2035 out2 << sjdl2 << endl;
2036 out2.close();
0f389141 2037 }
a8739e8a 2038
2039 // Copy jdl to grid workspace
2040 if (!copy) {
d3b18c4c 2041 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2042 } else {
d2a409b2 2043 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2044 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2045 TString finalJDL = mergeJDLName;
2046 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2047 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2048 if (fProductionMode) {
b5fe9cba 2049 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2050 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2051 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2052 }
d2a409b2 2053 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2054 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2055 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2056 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2057// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2058 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2059 Fatal("","Terminating");
0f389141 2060 if (fMergeViaJDL) {
b3e07543 2061 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2062// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2063// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2064 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2065 Fatal("","Terminating");
2066 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2067 Fatal("","Terminating");
0f389141 2068 }
a8739e8a 2069 }
2070 return kTRUE;
2071}
2072
2073//______________________________________________________________________________
5513444a 2074Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2075{
2076// Returns true if file exists.
5513444a 2077 if (!gGrid) return kFALSE;
2ea55496 2078 TString slfn = lfn;
2079 slfn.ReplaceAll("alien://","");
2080 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2081 if (!res) return kFALSE;
2082 TMap *map = dynamic_cast<TMap*>(res->At(0));
2083 if (!map) {
2084 delete res;
2085 return kFALSE;
2086 }
2087 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2088 if (!objs || !objs->GetString().Length()) {
2089 delete res;
2090 return kFALSE;
2091 }
2092 delete res;
2093 return kTRUE;
2094}
2095
2096//______________________________________________________________________________
923e2ca5 2097Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2098{
2099// Returns true if directory exists. Can be also a path.
2100 if (!gGrid) return kFALSE;
2101 // Check if dirname is a path
2102 TString dirstripped = dirname;
2103 dirstripped = dirstripped.Strip();
2104 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2105 TString dir = gSystem->BaseName(dirstripped);
2106 dir += "/";
2107 TString path = gSystem->DirName(dirstripped);
2108 TGridResult *res = gGrid->Ls(path, "-F");
2109 if (!res) return kFALSE;
2110 TIter next(res);
2111 TMap *map;
2112 TObject *obj;
2113 while ((map=dynamic_cast<TMap*>(next()))) {
2114 obj = map->GetValue("name");
2115 if (!obj) break;
2116 if (dir == obj->GetName()) {
2117 delete res;
2118 return kTRUE;
2119 }
2120 }
2121 delete res;
2122 return kFALSE;
2123}
2124
2125//______________________________________________________________________________
a2f5fc01 2126void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2127{
2128// Check input data type.
a2f5fc01 2129 isCollection = kFALSE;
2130 isXml = kFALSE;
2131 useTags = kFALSE;
c57f56b7 2132 if (!gGrid) {
2133 Error("CheckDataType", "No connection to grid");
2134 return;
2135 }
a2f5fc01 2136 isCollection = IsCollection(lfn);
c57f56b7 2137 TString msg = "\n##### file: ";
2138 msg += lfn;
a2f5fc01 2139 if (isCollection) {
c57f56b7 2140 msg += " type: raw_collection;";
2141 // special treatment for collections
a2f5fc01 2142 isXml = kFALSE;
c57f56b7 2143 // check for tag files in the collection
2144 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2145 if (!res) {
2146 msg += " using_tags: No (unknown)";
23329835 2147 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2148 return;
2149 }
2150 const char* typeStr = res->GetKey(0, "origLFN");
2151 if (!typeStr || !strlen(typeStr)) {
2152 msg += " using_tags: No (unknown)";
23329835 2153 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2154 return;
2155 }
2156 TString file = typeStr;
a2f5fc01 2157 useTags = file.Contains(".tag");
2158 if (useTags) msg += " using_tags: Yes";
c57f56b7 2159 else msg += " using_tags: No";
23329835 2160 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2161 return;
2162 }
2163 TString slfn(lfn);
2164 slfn.ToLower();
a2f5fc01 2165 isXml = slfn.Contains(".xml");
2166 if (isXml) {
c57f56b7 2167 // Open xml collection and check if there are tag files inside
2168 msg += " type: xml_collection;";
2169 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2170 if (!coll) {
2171 msg += " using_tags: No (unknown)";
23329835 2172 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2173 return;
2174 }
2175 TMap *map = coll->Next();
2176 if (!map) {
2177 msg += " using_tags: No (unknown)";
23329835 2178 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2179 return;
2180 }
2181 map = (TMap*)map->GetValue("");
2182 TString file;
2183 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2184 useTags = file.Contains(".tag");
c57f56b7 2185 delete coll;
a2f5fc01 2186 if (useTags) msg += " using_tags: Yes";
c57f56b7 2187 else msg += " using_tags: No";
23329835 2188 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2189 return;
2190 }
a2f5fc01 2191 useTags = slfn.Contains(".tag");
c57f56b7 2192 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2193 else msg += " type: unknown file;";
a2f5fc01 2194 if (useTags) msg += " using_tags: Yes";
c57f56b7 2195 else msg += " using_tags: No";
23329835 2196 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2197}
2198
2199//______________________________________________________________________________
4e5c5506 2200void AliAnalysisAlien::EnablePackage(const char *package)
2201{
2202// Enables a par file supposed to exist in the current directory.
2203 TString pkg(package);
2204 pkg.ReplaceAll(".par", "");
2205 pkg += ".par";
2206 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2207 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2208 return;
2209 }
2210 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2211 Info("EnablePackage", "AliEn plugin will use .par packages");
2212 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2213 if (!fPackages) {
2214 fPackages = new TObjArray();
2215 fPackages->SetOwner();
2216 }
2217 fPackages->Add(new TObjString(pkg));
2218}
2219
2220//______________________________________________________________________________
3bdcb562 2221TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2222{
2223// Make a tree from files having the location specified in fFileForTestMode.
2224// Inspired from JF's CreateESDChain.
2225 if (fFileForTestMode.IsNull()) {
2226 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2227 return NULL;
2228 }
2229 if (gSystem->AccessPathName(fFileForTestMode)) {
2230 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2231 return NULL;
2232 }
2233 // Open the file
2234 ifstream in;
2235 in.open(fFileForTestMode);
2236 Int_t count = 0;
2237 // Read the input list of files and add them to the chain
b505f1f8 2238 TString line;
2239 TChain *chain = new TChain(treeName);
2240 TChain *chainFriend = 0;
2241 if (!fFriendChainName.IsNull()) chainFriend = new TChain(treeName);
2242 while (in.good())
2243 {
3bdcb562 2244 in >> line;
2245 if (line.IsNull()) continue;
2246 if (count++ == fNtestFiles) break;
2247 TString esdFile(line);
2248 TFile *file = TFile::Open(esdFile);
b505f1f8 2249 if (file && !file->IsZombie()) {
2250 chain->Add(esdFile);
3bdcb562 2251 file->Close();
b505f1f8 2252 if (!fFriendChainName.IsNull()) {
4b5d9e21 2253 if (esdFile.Index("#") > -1)
2254 esdFile.Remove(esdFile.Index("#"));
2255 esdFile = gSystem->DirName(esdFile);
2256 esdFile += "/" + fFriendChainName;
b505f1f8 2257 file = TFile::Open(esdFile);
2258 if (file && !file->IsZombie()) {
2259 file->Close();
2260 chainFriend->Add(esdFile);
2261 } else {
2262 Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data());
2263 return 0;
2264 }
4b5d9e21 2265 }
3bdcb562 2266 } else {
2267 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2268 }
b505f1f8 2269 }
2270 in.close();
2271 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2272 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2273 delete chain;
b505f1f8 2274 delete chainFriend;
3bdcb562 2275 return NULL;
b505f1f8 2276 }
3bdcb562 2277// chain->ls();
b505f1f8 2278 if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend);
2279 return chain;
3bdcb562 2280}
2281
2282//______________________________________________________________________________
d2a409b2 2283const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2284{
2285// Get job status for all jobs with jobid>jobidstart.
2286 static char mstatus[20];
2287 mstatus[0] = '\0';
2288 nrunning = 0;
2289 nwaiting = 0;
2290 nerror = 0;
2291 ndone = 0;
2292 TGridJobStatusList *list = gGrid->Ps("");
2293 if (!list) return mstatus;
2294 Int_t nentries = list->GetSize();
2295 TGridJobStatus *status;
2296 Int_t pid;
2297 for (Int_t ijob=0; ijob<nentries; ijob++) {
2298 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2299 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2300 if (pid<jobidstart) continue;
2301 if (pid == lastid) {
8ddad121 2302 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2303 }
2304 switch (status->GetStatus()) {
2305 case TGridJobStatus::kWAITING:
2306 nwaiting++; break;
2307 case TGridJobStatus::kRUNNING:
2308 nrunning++; break;
2309 case TGridJobStatus::kABORTED:
2310 case TGridJobStatus::kFAIL:
2311 case TGridJobStatus::kUNKNOWN:
2312 nerror++; break;
2313 case TGridJobStatus::kDONE:
2314 ndone++;
2315 }
2316 }
2317 list->Delete();
2318 delete list;
2319 return mstatus;
2320}
2321
2322//______________________________________________________________________________
c57f56b7 2323Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2324{
2325// Returns true if file is a collection. Functionality duplicated from
2326// TAlien::Type() because we don't want to directly depend on TAlien.
2327 if (!gGrid) {
2328 Error("IsCollection", "No connection to grid");
2329 return kFALSE;
2330 }
2331 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2332 if (!res) return kFALSE;
2333 const char* typeStr = res->GetKey(0, "type");
2334 if (!typeStr || !strlen(typeStr)) return kFALSE;
2335 if (!strcmp(typeStr, "collection")) return kTRUE;
2336 delete res;
2337 return kFALSE;
2338}
2339
2340//______________________________________________________________________________
fe2d7fc2 2341Bool_t AliAnalysisAlien::IsSingleOutput() const
2342{
2343// Check if single-ouput option is on.
2344 return (!fOutputSingle.IsNull());
2345}
2346
2347//______________________________________________________________________________
16a4353c 2348void AliAnalysisAlien::Print(Option_t *) const
2349{
2350// Print current plugin settings.
84fcd93f 2351 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2352 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2353 if (mgr && mgr->IsProofMode()) {
2354 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2355 if (TestBit(AliAnalysisGrid::kTest))
2356 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2357 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2358 if (!fProofDataSet.IsNull())
2359 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2360 if (fProofReset==1)
2361 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2362 if (fProofReset>1)
2363 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2364 if (!fRootVersionForProof.IsNull())
2365 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2366 else
2367 printf("= ROOT version requested________________________ default\n");
2368 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2369 if (!fAliRootMode.IsNull())
2370 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2371 if (fNproofWorkers)
2372 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2373 if (fNproofWorkersPerSlave)
2374 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2375 if (TestSpecialBit(kClearPackages))
2376 printf("= ClearPackages requested...\n");
2377 if (fIncludePath.Data())
2378 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2379 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2380 if (fPackages && fPackages->GetEntries()) {
2381 TIter next(fPackages);
2382 TObject *obj;
2383 TString list;
2384 while ((obj=next())) list += obj->GetName();
2385 printf("= Par files to be used: ________________________ %s\n", list.Data());
2386 }
2387 if (TestSpecialBit(kProofConnectGrid))
2388 printf("= Requested PROOF connection to grid\n");
2389 return;
2390 }
e1c22e21 2391 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2392 if (fOverwriteMode) {
2393 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2394 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2395 }
348be253 2396 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2397 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2398 printf("= Production mode:______________________________ %d\n", fProductionMode);
2399 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2400 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2401 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2402 if (fUser.Length())
84fcd93f 2403 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2404 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2405 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2406 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2407 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2408 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2409 if (fRunNumbers.Length())
84fcd93f 2410 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2411 if (fRunRange[0])
74013535 2412 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2413 if (!fRunRange[0] && !fRunNumbers.Length()) {
2414 TIter next(fInputFiles);
2415 TObject *obj;
2416 TString list;
2417 while ((obj=next())) list += obj->GetName();
84fcd93f 2418 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2419 }
2420 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2421 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2422 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2423 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2424 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2425 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2426 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2427 printf("=====================================================================\n");
2428 printf("= Job price: ___________________________________ %d\n", fPrice);
2429 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2430 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2431 if (fMaxInitFailed>0)
84fcd93f 2432 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2433 if (fMasterResubmitThreshold>0)
84fcd93f 2434 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2435 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2436 if (fNrunsPerMaster>0)
84fcd93f 2437 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2438 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2439 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2440 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2441 if (fArguments.Length())
84fcd93f 2442 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2443 if (fExecutableArgs.Length())
2444 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2445 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2446 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2447 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2448 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2449 if (fDatasetName)
84fcd93f 2450 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2451 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2452 if (fIncludePath.Data())
84fcd93f 2453 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2454 if (fCloseSE.Length())
84fcd93f 2455 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2456 if (fFriendChainName.Length())
84fcd93f 2457 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2458 if (fPackages && fPackages->GetEntries()) {
16a4353c 2459 TIter next(fPackages);
2460 TObject *obj;
2461 TString list;
2462 while ((obj=next())) list += obj->GetName();
84fcd93f 2463 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2464 }
2465}
2466
2467//______________________________________________________________________________
c57f56b7 2468void AliAnalysisAlien::SetDefaults()
2469{
2470// Set default values for everything. What cannot be filled will be left empty.
2471 if (fGridJDL) delete fGridJDL;
2472 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2473 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2474 fPrice = 1;
2475 fTTL = 30000;
2476 fSplitMaxInputFileNumber = 100;
2477 fMaxInitFailed = 0;
2478 fMasterResubmitThreshold = 0;
bb885a9e 2479 fNtestFiles = 10;
149d288c 2480 fNreplicas = 2;
a8739e8a 2481 fRunRange[0] = 0;
2482 fRunRange[1] = 0;
8e3bd06a 2483 fRunPrefix = "%d";
319593fb 2484 fNrunsPerMaster = 1;
16a4353c 2485 fMaxMergeFiles = 100;
c57f56b7 2486 fRunNumbers = "";
2487 fExecutable = "analysis.sh";
0a1c1f7f 2488 fExecutableCommand = "root -b -q";
c57f56b7 2489 fArguments = "";
631c0b05 2490 fExecutableArgs = "";
c57f56b7 2491 fAnalysisMacro = "myAnalysis.C";
2492 fAnalysisSource = "";
2493 fAdditionalLibs = "";
2494 fSplitMode = "se";
2495 fAPIVersion = "";
2496 fROOTVersion = "";
2497 fAliROOTVersion = "";
2498 fUser = ""; // Your alien user name
2499 fGridWorkingDir = "";
2500 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2501 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2502 fFriendChainName = "";
c57f56b7 2503 fGridOutputDir = "output";
b83f84f0 2504 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2505 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2506 fInputFormat = "xml-single";
2507 fJDLName = "analysis.jdl";
c6cb3634 2508 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2509 fMergeExcludes = "";
0f389141 2510 fMergeViaJDL = 0;
348be253 2511 SetUseCopy(kTRUE);
2512 SetCheckCopy(kTRUE);
149d288c 2513 SetDefaultOutputs(kTRUE);
e1c22e21 2514 fOverwriteMode = 1;
c57f56b7 2515}
2516
2517//______________________________________________________________________________
b3e07543 2518Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2519{
b3e07543 2520// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2521 // First check if the result is already in the output directory.
2522 if (FileExists(Form("%s/%s",aliendir,filename))) {
2523 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2524 return kFALSE;
2525 }
b3e07543 2526 // Now check the last stage done.
2527 Int_t stage = 0;
2528 while (1) {
2529 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2530 stage++;
2531 }
2532 // Next stage of merging
2533 stage++;
2534 TString pattern = "*root_archive.zip";
2535 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2536 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2537 if (res) delete res;
2538 // Write standard output to file
2539 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2540 // Count the number of files inside
2541 ifstream ifile;
2542 ifile.open(Form("Stage_%d.xml",stage));
2543 if (!ifile.good()) {
2544 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2545 return kFALSE;
b3e07543 2546 }
2547 TString line;
2548 Int_t nfiles = 0;
2549 while (!ifile.eof()) {
2550 ifile >> line;
2551 if (line.Contains("/event")) nfiles++;
7c2cd90a 2552 }
b3e07543 2553 ifile.close();
2554 if (!nfiles) {
2555 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2556 return kFALSE;
2557 } else {
2558 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2559 }
b3e07543 2560 // Copy the file in the output directory
2561 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2562// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2563 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2564 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2565 // Check if this is the last stage to be done.
2566 Bool_t laststage = (nfiles<nperchunk);
2567 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2568 if (laststage) {
2569 printf("### Submiting final merging stage %d\n", stage);
2570 TString finalJDL = jdl;
2571 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2572 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2573 Int_t jobId = SubmitSingleJob(query);
2574 if (!jobId) return kFALSE;
2575 } else {
2576 printf("### Submiting merging stage %d\n", stage);
03335324 2577 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2578 Int_t jobId = SubmitSingleJob(query);
2579 if (!jobId) return kFALSE;
7c2cd90a 2580 }
b3e07543 2581 return kTRUE;
2582}
7c2cd90a 2583
2584//______________________________________________________________________________
105631b5 2585AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2586{
2587// Loat the analysis manager from a file.
2588 TFile *file = TFile::Open(fname);
2589 if (!file) {
2590 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2591 return 0;
2592 }
2593 TIter nextkey(file->GetListOfKeys());
2594 AliAnalysisManager *mgr = 0;
2595 TKey *key;
2596 while ((key=(TKey*)nextkey())) {
2597 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2598 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2599 }
2600 if (!mgr)
2601 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2602 return mgr;
2603}
2604
2605//______________________________________________________________________________
7c2cd90a 2606Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2607{
2608// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2609 if (!gGrid) return 0;
2610 printf("=> %s ------> ",query);
2611 TGridResult *res = gGrid->Command(query);
2612 if (!res) return 0;
2613 TString jobId = res->GetKey(0,"jobId");
2614 delete res;
2615 if (jobId.IsNull()) {
2616 printf("submission failed. Reason:\n");
2617 gGrid->Stdout();
2618 gGrid->Stderr();
2619 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2620 return 0;
2621 }
2622 printf(" Job id: %s\n", jobId.Data());
2623 return atoi(jobId);
2624}
2625
2626//______________________________________________________________________________
b3e07543 2627Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2628{
b3e07543 2629// Merge given output files from basedir. Basedir can be an alien output directory
2630// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2631// files in a group (ignored for xml input). Merging can be done in stages:
2632// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2633// stage=1 : works with an xml of all root_archive.zip in the output directory
2634// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2635 TString outputFile = output;
0f389141 2636 TString command;
a2f5fc01 2637 TString outputChunk;
2638 TString previousChunk = "";
b3e07543 2639 TObjArray *listoffiles = new TObjArray();
2640// listoffiles->SetOwner();
a2f5fc01 2641 Int_t countChunk = 0;
2642 Int_t countZero = nmaxmerge;
0f389141 2643 Bool_t merged = kTRUE;
a2f5fc01 2644 Int_t index = outputFile.Index("@");
2645 if (index > 0) outputFile.Remove(index);
7c2cd90a 2646 TString inputFile = outputFile;
b3e07543 2647 TString sbasedir = basedir;
2648 if (sbasedir.Contains(".xml")) {
2649 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2650 nmaxmerge = 9999999;
2651 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2652 if (!coll) {
2653 ::Error("MergeOutput", "Input XML collection empty.");
2654 return kFALSE;
2655 }
2656 // Iterate grid collection
2657 while (coll->Next()) {
2658 TString fname = gSystem->DirName(coll->GetTURL());
2659 fname += "/";
2660 fname += inputFile;
2661 listoffiles->Add(new TNamed(fname.Data(),""));
2662 }
2663 } else {
2664 command = Form("find %s/ *%s", basedir, inputFile.Data());
2665 printf("command: %s\n", command.Data());
2666 TGridResult *res = gGrid->Command(command);
2667 if (!res) {
2668 ::Error("MergeOutput","No result for the find command\n");
2669 delete listoffiles;
2670 return kFALSE;
2671 }
2672 TIter nextmap(res);
2673 TMap *map = 0;
2674 while ((map=(TMap*)nextmap())) {
2675 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2676 if (!objs || !objs->GetString().Length()) {
2677 // Nothing found - skip this output
2678 delete res;
2679 delete listoffiles;
2680 return kFALSE;
2681 }
2682 listoffiles->Add(new TNamed(objs->GetName(),""));
2683 }
2684 delete res;
2685 }
2686 if (!listoffiles->GetEntries()) {
7c2cd90a 2687 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2688 delete listoffiles;
0f389141 2689 return kFALSE;
2690 }
2691
2692 TFileMerger *fm = 0;
b3e07543 2693 TIter next0(listoffiles);
2694 TObjArray *listoffilestmp = new TObjArray();
2695 listoffilestmp->SetOwner();
2696 TObject *nextfile;
2697 TString snextfile;
2698 // Keep only the files at upper level
2699 Int_t countChar = 0;
2700 while ((nextfile=next0())) {
2701 snextfile = nextfile->GetName();
2702 Int_t crtCount = snextfile.CountChar('/');
2703 if (nextfile == listoffiles->First()) countChar = crtCount;
2704 if (crtCount < countChar) countChar = crtCount;
2705 }
2706 next0.Reset();
2707 while ((nextfile=next0())) {
2708 snextfile = nextfile->GetName();
2709 Int_t crtCount = snextfile.CountChar('/');
2710 if (crtCount > countChar) {
2711 delete nextfile;
2712 continue;
2713 }
2714 listoffilestmp->Add(nextfile);
2715 }
2716 delete listoffiles;
2717 listoffiles = listoffilestmp; // Now contains 'good' files
2718 listoffiles->Print();
2719 TIter next(listoffiles);
7c2cd90a 2720 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2721 outputChunk = outputFile;
2722 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2723 // Check for existent temporary merge files
e1c22e21 2724 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2725 // Preserve old merging functionality for stage 0.
2726 if (stage==0) {
2727 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2728 while (1) {
2729 // Skip as many input files as in a chunk
2ea55496 2730 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2731 nextfile = next();
2732 if (!nextfile) {
2ea55496 2733 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2734 delete listoffiles;
2ea55496 2735 return kFALSE;
2736 }
b3e07543 2737 snextfile = nextfile->GetName();
7c2cd90a 2738 }
2739 outputChunk = outputFile;
2740 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2741 countChunk++;
2742 if (gSystem->AccessPathName(outputChunk)) continue;
2743 // Merged file with chunks up to <countChunk> found
2744 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2745 previousChunk = outputChunk;
2746 break;
2747 }
2748 }
2749 countZero = nmaxmerge;
2750
b3e07543 2751 while ((nextfile=next())) {
2752 snextfile = nextfile->GetName();
2ea55496 2753 // Loop 'find' results and get next LFN
7c2cd90a 2754 if (countZero == nmaxmerge) {
2755 // First file in chunk - create file merger and add previous chunk if any.
2b895f34 2756 fm = new TFileMerger(kTRUE);
7c2cd90a 2757 fm->SetFastMethod(kTRUE);
2758 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2759 outputChunk = outputFile;
2760 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2761 }
2762 // If last file found, put merged results in the output file
b3e07543 2763 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2764 // Add file to be merged and decrement chunk counter.
b3e07543 2765 fm->AddFile(snextfile);
7c2cd90a 2766 countZero--;
b3e07543 2767 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2768 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2769 // Nothing found - skip this output
2770 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2771 merged = kFALSE;
2772 break;
7c2cd90a 2773 }
2774 fm->OutputFile(outputChunk);
2775 // Merge the outputs, then go to next chunk
2776 if (!fm->Merge()) {
2777 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2778 merged = kFALSE;
2779 break;
7c2cd90a 2780 } else {
2781 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2782 gSystem->Unlink(previousChunk);
2783 }
b3e07543 2784 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2785 countChunk++;
2786 countZero = nmaxmerge;
2787 previousChunk = outputChunk;
0f389141 2788 }
0f389141 2789 }
b3e07543 2790 delete listoffiles;
2791 delete fm;
7c2cd90a 2792 return merged;
2793 }
2794 // Merging stage different than 0.
2795 // Move to the begining of the requested chunk.
2b895f34 2796 fm = new TFileMerger(kTRUE);
7c2cd90a 2797 fm->SetFastMethod(kTRUE);
b3e07543 2798 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2799 delete listoffiles;
7c2cd90a 2800 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2801 // Nothing found - skip this output
2802 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2803 delete fm;
2804 return kFALSE;
2805 }
b3e07543 2806 fm->OutputFile(outputFile);
7c2cd90a 2807 // Merge the outputs
2808 if (!fm->Merge()) {
2809 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2810 delete fm;
2811 return kFALSE;
2812 } else {
b3e07543 2813 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2814 }
7c2cd90a 2815 delete fm;
2816 return kTRUE;
0f389141 2817}
2818
2819//______________________________________________________________________________
c57f56b7 2820Bool_t AliAnalysisAlien::MergeOutputs()
2821{
2822// Merge analysis outputs existing in the AliEn space.
2823 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2824 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2825 if (!Connect()) {
2826 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2827 return kFALSE;
0f389141 2828 }
c9e8f7fd 2829 if (fMergeViaJDL) {
2830 if (!TestBit(AliAnalysisGrid::kMerge)) {
2831 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2832 return kFALSE;
2833 }
2834 if (fProductionMode) {
2835 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2836 return kFALSE;
2837 }
0f389141 2838 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2839 if (!SubmitMerging()) return kFALSE;
0f389141 2840 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2841 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2842 return kFALSE;
c57f56b7 2843 }
2844 // Get the output path
dc030a24 2845 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2846 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2847 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2848 return kFALSE;
2849 }
2850 if (!fOutputFiles.Length()) {
2851 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2852 return kFALSE;
0f389141 2853 }
2854 // Check if fast read option was requested
e1c22e21 2855 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2856 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2857 if (fFastReadOption) {
d3339be3 2858 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2859 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2860 gEnv->SetValue("XNet.ConnectTimeout",50);
2861 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2862 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2863 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2864 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2865 }
e8b839ab 2866 // Make sure we change the temporary directory
2867 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2868 // Set temporary compilation directory to current one
2869 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2870 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2871 TIter next(list);
2872 TObjString *str;
a2f5fc01 2873 TString outputFile;
c57f56b7 2874 Bool_t merged = kTRUE;
2875 while((str=(TObjString*)next())) {
a2f5fc01 2876 outputFile = str->GetString();
2877 Int_t index = outputFile.Index("@");
2878 if (index > 0) outputFile.Remove(index);
2879 TString outputChunk = outputFile;
2880 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2881 // Skip already merged outputs
a2f5fc01 2882 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2883 if (fOverwriteMode) {
a2f5fc01 2884 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2885 gSystem->Unlink(outputFile);
2886 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2887 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2888 outputChunk.Data());
2889 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2890 }
2891 } else {
a2f5fc01 2892 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2893 continue;
2894 }
2895 } else {
a2f5fc01 2896 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2897 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2898 outputChunk.Data());
2899 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2900 }
2901 }
37d21c01 2902 if (fMergeExcludes.Contains(outputFile.Data()) ||
2903 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 2904 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2905 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2906 if (!merged) {
2907 Error("MergeOutputs", "Terminate() will NOT be executed");
2908 return kFALSE;
ff07ec61 2909 }
2910 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2911 if (fileOpened) fileOpened->Close();
c57f56b7 2912 }
0f389141 2913 return kTRUE;
c57f56b7 2914}
2915
2916//______________________________________________________________________________
bb885a9e 2917void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2918{
2919// Use the output files connected to output containers from the analysis manager
2920// rather than the files defined by SetOutputFiles
2921 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2922 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2923 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2924}
2925
2926//______________________________________________________________________________
149d288c 2927void AliAnalysisAlien::SetOutputFiles(const char *list)
2928{
2929// Manually set the output files list.
2930// Removes duplicates. Not allowed if default outputs are not disabled.
2931 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2932 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2933 return;
2934 }
2935 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2936 fOutputFiles = "";
2937 TString slist = list;
2938 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2939 TObjArray *arr = slist.Tokenize(" ");
2940 TObjString *os;
2941 TIter next(arr);
2942 TString sout;
2943 while ((os=(TObjString*)next())) {
2944 sout = os->GetString();
2945 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2946 if (fOutputFiles.Contains(sout)) continue;
2947 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2948 fOutputFiles += sout;
2949 }
2950 delete arr;
f790bc1b 2951}
149d288c 2952
2953//______________________________________________________________________________
2954void AliAnalysisAlien::SetOutputArchive(const char *list)
2955{
2956// Manually set the output archive list. Free text - you are on your own...
2957// Not allowed if default outputs are not disabled.
2958 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2959 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2960 return;
2961 }
2962 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2963 fOutputArchive = list;
2964}
2965
2966//______________________________________________________________________________
2967void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2968{
2969// Setting a prefered output SE is not allowed anymore.
2970 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2971}
2972
2973//______________________________________________________________________________
28ce6c9c 2974void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
2975{
2976// Set some PROOF special parameter.
2977 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2978 if (pair) {
2979 TObject *old = pair->Key();
2980 TObject *val = pair->Value();
2981 fProofParam.Remove(old);
2982 delete old;
2983 delete val;
2984 }
2985 fProofParam.Add(new TObjString(pname), new TObjString(value));
2986}
2987
2988//______________________________________________________________________________
2989const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
2990{
2991// Returns a special PROOF parameter.
2992 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2993 if (!pair) return 0;