AddTimeStamp was always increasing track length but accounting
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
c82bb898 51using std::ofstream;
52using std::ifstream;
53using std::ios;
54using std::endl;
c57f56b7 55ClassImp(AliAnalysisAlien)
6c395669 56#if 0
57;
58#endif
c57f56b7 59
6c395669 60namespace {
61 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
62 {
63 TString sl(Form("file:%s", loc));
64 TString sr(Form("alien://%s", rem));
65 Bool_t ret = TFile::Cp(sl, sr);
66 if (!ret) {
67 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
68 }
69 return ret;
70 }
71}
72
c57f56b7 73//______________________________________________________________________________
74AliAnalysisAlien::AliAnalysisAlien()
75 :AliAnalysisGrid(),
76 fGridJDL(NULL),
0f389141 77 fMergingJDL(NULL),
c57f56b7 78 fPrice(0),
79 fTTL(0),
80 fSplitMaxInputFileNumber(0),
81 fMaxInitFailed(0),
82 fMasterResubmitThreshold(0),
bb885a9e 83 fNtestFiles(0),
319593fb 84 fNrunsPerMaster(0),
16a4353c 85 fMaxMergeFiles(0),
b3e07543 86 fMaxMergeStages(0),
d2a409b2 87 fNsubmitted(0),
a3e84053 88 fProductionMode(0),
cd11251e 89 fOutputToRunNo(0),
0f389141 90 fMergeViaJDL(0),
91 fFastReadOption(0),
e1c22e21 92 fOverwriteMode(1),
149d288c 93 fNreplicas(2),
3bdcb562 94 fNproofWorkers(0),
27734f0e 95 fNproofWorkersPerSlave(0),
3bdcb562 96 fProofReset(0),
c57f56b7 97 fRunNumbers(),
98 fExecutable(),
0a1c1f7f 99 fExecutableCommand(),
c57f56b7 100 fArguments(),
631c0b05 101 fExecutableArgs(),
c57f56b7 102 fAnalysisMacro(),
103 fAnalysisSource(),
d3b18c4c 104 fValidationScript(),
d5c6455a 105 fAdditionalRootLibs(),
c57f56b7 106 fAdditionalLibs(),
107 fSplitMode(),
108 fAPIVersion(),
109 fROOTVersion(),
110 fAliROOTVersion(),
648174cf 111 fExternalPackages(),
c57f56b7 112 fUser(),
113 fGridWorkingDir(),
114 fGridDataDir(),
115 fDataPattern(),
116 fGridOutputDir(),
117 fOutputArchive(),
118 fOutputFiles(),
119 fInputFormat(),
e7c71df0 120 fDatasetName(),
c57f56b7 121 fJDLName(),
d3b18c4c 122 fTerminateFiles(),
bb885a9e 123 fMergeExcludes(),
37d21c01 124 fRegisterExcludes(),
f965131e 125 fIncludePath(),
bb885a9e 126 fCloseSE(),
0df6ccf2 127 fFriendChainName(),
c6cb3634 128 fJobTag(),
648174cf 129 fOutputSingle(),
5fce53f4 130 fRunPrefix(),
3bdcb562 131 fProofCluster(),
132 fProofDataSet(),
133 fFileForTestMode(),
134 fRootVersionForProof(),
135 fAliRootMode(),
f47d5cb4 136 fMergeDirName(),
4e5c5506 137 fInputFiles(0),
28ce6c9c 138 fPackages(0),
77f34eae 139 fModules(0),
28ce6c9c 140 fProofParam()
c57f56b7 141{
142// Dummy ctor.
143 SetDefaults();
144}
145
146//______________________________________________________________________________
147AliAnalysisAlien::AliAnalysisAlien(const char *name)
148 :AliAnalysisGrid(name),
149 fGridJDL(NULL),
0f389141 150 fMergingJDL(NULL),
c57f56b7 151 fPrice(0),
152 fTTL(0),
153 fSplitMaxInputFileNumber(0),
154 fMaxInitFailed(0),
155 fMasterResubmitThreshold(0),
bb885a9e 156 fNtestFiles(0),
319593fb 157 fNrunsPerMaster(0),
16a4353c 158 fMaxMergeFiles(0),
b3e07543 159 fMaxMergeStages(0),
d2a409b2 160 fNsubmitted(0),
a3e84053 161 fProductionMode(0),
cd11251e 162 fOutputToRunNo(0),
0f389141 163 fMergeViaJDL(0),
164 fFastReadOption(0),
e1c22e21 165 fOverwriteMode(1),
149d288c 166 fNreplicas(2),
3bdcb562 167 fNproofWorkers(0),
27734f0e 168 fNproofWorkersPerSlave(0),
3bdcb562 169 fProofReset(0),
c57f56b7 170 fRunNumbers(),
171 fExecutable(),
0a1c1f7f 172 fExecutableCommand(),
c57f56b7 173 fArguments(),
631c0b05 174 fExecutableArgs(),
c57f56b7 175 fAnalysisMacro(),
176 fAnalysisSource(),
d3b18c4c 177 fValidationScript(),
d5c6455a 178 fAdditionalRootLibs(),
c57f56b7 179 fAdditionalLibs(),
180 fSplitMode(),
181 fAPIVersion(),
182 fROOTVersion(),
183 fAliROOTVersion(),
648174cf 184 fExternalPackages(),
c57f56b7 185 fUser(),
186 fGridWorkingDir(),
187 fGridDataDir(),
188 fDataPattern(),
189 fGridOutputDir(),
190 fOutputArchive(),
191 fOutputFiles(),
192 fInputFormat(),
e7c71df0 193 fDatasetName(),
c57f56b7 194 fJDLName(),
d3b18c4c 195 fTerminateFiles(),
bb885a9e 196 fMergeExcludes(),
37d21c01 197 fRegisterExcludes(),
f965131e 198 fIncludePath(),
bb885a9e 199 fCloseSE(),
0df6ccf2 200 fFriendChainName(),
c6cb3634 201 fJobTag(),
648174cf 202 fOutputSingle(),
5fce53f4 203 fRunPrefix(),
3bdcb562 204 fProofCluster(),
205 fProofDataSet(),
206 fFileForTestMode(),
207 fRootVersionForProof(),
208 fAliRootMode(),
f47d5cb4 209 fMergeDirName(),
4e5c5506 210 fInputFiles(0),
28ce6c9c 211 fPackages(0),
77f34eae 212 fModules(0),
28ce6c9c 213 fProofParam()
c57f56b7 214{
215// Default ctor.
216 SetDefaults();
217}
218
219//______________________________________________________________________________
220AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
221 :AliAnalysisGrid(other),
222 fGridJDL(NULL),
0f389141 223 fMergingJDL(NULL),
c57f56b7 224 fPrice(other.fPrice),
225 fTTL(other.fTTL),
226 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
227 fMaxInitFailed(other.fMaxInitFailed),
228 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 229 fNtestFiles(other.fNtestFiles),
319593fb 230 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 231 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 232 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 233 fNsubmitted(other.fNsubmitted),
a3e84053 234 fProductionMode(other.fProductionMode),
cd11251e 235 fOutputToRunNo(other.fOutputToRunNo),
0f389141 236 fMergeViaJDL(other.fMergeViaJDL),
237 fFastReadOption(other.fFastReadOption),
238 fOverwriteMode(other.fOverwriteMode),
149d288c 239 fNreplicas(other.fNreplicas),
3bdcb562 240 fNproofWorkers(other.fNproofWorkers),
27734f0e 241 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 242 fProofReset(other.fProofReset),
c57f56b7 243 fRunNumbers(other.fRunNumbers),
244 fExecutable(other.fExecutable),
0a1c1f7f 245 fExecutableCommand(other.fExecutableCommand),
c57f56b7 246 fArguments(other.fArguments),
631c0b05 247 fExecutableArgs(other.fExecutableArgs),
c57f56b7 248 fAnalysisMacro(other.fAnalysisMacro),
249 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 250 fValidationScript(other.fValidationScript),
d5c6455a 251 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 252 fAdditionalLibs(other.fAdditionalLibs),
253 fSplitMode(other.fSplitMode),
254 fAPIVersion(other.fAPIVersion),
255 fROOTVersion(other.fROOTVersion),
256 fAliROOTVersion(other.fAliROOTVersion),
648174cf 257 fExternalPackages(other.fExternalPackages),
c57f56b7 258 fUser(other.fUser),
259 fGridWorkingDir(other.fGridWorkingDir),
260 fGridDataDir(other.fGridDataDir),
261 fDataPattern(other.fDataPattern),
262 fGridOutputDir(other.fGridOutputDir),
263 fOutputArchive(other.fOutputArchive),
264 fOutputFiles(other.fOutputFiles),
265 fInputFormat(other.fInputFormat),
e7c71df0 266 fDatasetName(other.fDatasetName),
c57f56b7 267 fJDLName(other.fJDLName),
d3b18c4c 268 fTerminateFiles(other.fTerminateFiles),
bb885a9e 269 fMergeExcludes(other.fMergeExcludes),
37d21c01 270 fRegisterExcludes(other.fRegisterExcludes),
f965131e 271 fIncludePath(other.fIncludePath),
bb885a9e 272 fCloseSE(other.fCloseSE),
0df6ccf2 273 fFriendChainName(other.fFriendChainName),
c6cb3634 274 fJobTag(other.fJobTag),
648174cf 275 fOutputSingle(other.fOutputSingle),
5fce53f4 276 fRunPrefix(other.fRunPrefix),
3bdcb562 277 fProofCluster(other.fProofCluster),
278 fProofDataSet(other.fProofDataSet),
279 fFileForTestMode(other.fFileForTestMode),
280 fRootVersionForProof(other.fRootVersionForProof),
281 fAliRootMode(other.fAliRootMode),
f47d5cb4 282 fMergeDirName(other.fMergeDirName),
4e5c5506 283 fInputFiles(0),
28ce6c9c 284 fPackages(0),
77f34eae 285 fModules(0),
28ce6c9c 286 fProofParam()
c57f56b7 287{
288// Copy ctor.
289 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 290 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 291 fRunRange[0] = other.fRunRange[0];
292 fRunRange[1] = other.fRunRange[1];
c57f56b7 293 if (other.fInputFiles) {
294 fInputFiles = new TObjArray();
295 TIter next(other.fInputFiles);
296 TObject *obj;
297 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
298 fInputFiles->SetOwner();
299 }
4e5c5506 300 if (other.fPackages) {
301 fPackages = new TObjArray();
302 TIter next(other.fPackages);
303 TObject *obj;
304 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
305 fPackages->SetOwner();
306 }
77f34eae 307 if (other.fModules) {
308 fModules = new TObjArray();
309 fModules->SetOwner();
310 TIter next(other.fModules);
311 AliAnalysisTaskCfg *mod, *crt;
312 while ((crt=(AliAnalysisTaskCfg*)next())) {
313 mod = new AliAnalysisTaskCfg(*crt);
314 fModules->Add(mod);
315 }
316 }
c57f56b7 317}
318
319//______________________________________________________________________________
320AliAnalysisAlien::~AliAnalysisAlien()
321{
322// Destructor.
77f34eae 323 delete fGridJDL;
324 delete fMergingJDL;
325 delete fInputFiles;
326 delete fPackages;
327 delete fModules;
28ce6c9c 328 fProofParam.DeleteAll();
c57f56b7 329}
330
331//______________________________________________________________________________
332AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
333{
334// Assignment.
335 if (this != &other) {
336 AliAnalysisGrid::operator=(other);
337 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 338 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 339 fPrice = other.fPrice;
340 fTTL = other.fTTL;
341 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
342 fMaxInitFailed = other.fMaxInitFailed;
343 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 344 fNtestFiles = other.fNtestFiles;
a3e84053 345 fNrunsPerMaster = other.fNrunsPerMaster;
346 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 347 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 348 fNsubmitted = other.fNsubmitted;
349 fProductionMode = other.fProductionMode;
cd11251e 350 fOutputToRunNo = other.fOutputToRunNo;
0f389141 351 fMergeViaJDL = other.fMergeViaJDL;
352 fFastReadOption = other.fFastReadOption;
353 fOverwriteMode = other.fOverwriteMode;
149d288c 354 fNreplicas = other.fNreplicas;
3bdcb562 355 fNproofWorkers = other.fNproofWorkers;
27734f0e 356 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 357 fProofReset = other.fProofReset;
c57f56b7 358 fRunNumbers = other.fRunNumbers;
359 fExecutable = other.fExecutable;
0a1c1f7f 360 fExecutableCommand = other.fExecutableCommand;
c57f56b7 361 fArguments = other.fArguments;
631c0b05 362 fExecutableArgs = other.fExecutableArgs;
c57f56b7 363 fAnalysisMacro = other.fAnalysisMacro;
364 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 365 fValidationScript = other.fValidationScript;
d5c6455a 366 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 367 fAdditionalLibs = other.fAdditionalLibs;
368 fSplitMode = other.fSplitMode;
369 fAPIVersion = other.fAPIVersion;
370 fROOTVersion = other.fROOTVersion;
371 fAliROOTVersion = other.fAliROOTVersion;
648174cf 372 fExternalPackages = other.fExternalPackages;
c57f56b7 373 fUser = other.fUser;
374 fGridWorkingDir = other.fGridWorkingDir;
375 fGridDataDir = other.fGridDataDir;
376 fDataPattern = other.fDataPattern;
377 fGridOutputDir = other.fGridOutputDir;
378 fOutputArchive = other.fOutputArchive;
379 fOutputFiles = other.fOutputFiles;
380 fInputFormat = other.fInputFormat;
e7c71df0 381 fDatasetName = other.fDatasetName;
c57f56b7 382 fJDLName = other.fJDLName;
d3b18c4c 383 fTerminateFiles = other.fTerminateFiles;
bb885a9e 384 fMergeExcludes = other.fMergeExcludes;
37d21c01 385 fRegisterExcludes = other.fRegisterExcludes;
f965131e 386 fIncludePath = other.fIncludePath;
bb885a9e 387 fCloseSE = other.fCloseSE;
0df6ccf2 388 fFriendChainName = other.fFriendChainName;
c6cb3634 389 fJobTag = other.fJobTag;
648174cf 390 fOutputSingle = other.fOutputSingle;
5fce53f4 391 fRunPrefix = other.fRunPrefix;
3bdcb562 392 fProofCluster = other.fProofCluster;
393 fProofDataSet = other.fProofDataSet;
394 fFileForTestMode = other.fFileForTestMode;
395 fRootVersionForProof = other.fRootVersionForProof;
396 fAliRootMode = other.fAliRootMode;
f47d5cb4 397 fMergeDirName = other.fMergeDirName;
c57f56b7 398 if (other.fInputFiles) {
399 fInputFiles = new TObjArray();
400 TIter next(other.fInputFiles);
401 TObject *obj;
402 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
403 fInputFiles->SetOwner();
404 }
4e5c5506 405 if (other.fPackages) {
406 fPackages = new TObjArray();
407 TIter next(other.fPackages);
408 TObject *obj;
409 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
410 fPackages->SetOwner();
411 }
77f34eae 412 if (other.fModules) {
413 fModules = new TObjArray();
414 fModules->SetOwner();
415 TIter next(other.fModules);
416 AliAnalysisTaskCfg *mod, *crt;
417 while ((crt=(AliAnalysisTaskCfg*)next())) {
418 mod = new AliAnalysisTaskCfg(*crt);
419 fModules->Add(mod);
420 }
421 }
c57f56b7 422 }
423 return *this;
424}
425
426//______________________________________________________________________________
37d21c01 427void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
428{
429// Add a single additional library to be loaded. Extension must be present.
430 TString lib(name);
431 if (!lib.Contains(".")) {
432 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
433 return;
434 }
435 if (fAdditionalLibs.Contains(name)) {
436 Warning("AddAdditionalLibrary", "Library %s already added.", name);
437 return;
438 }
439 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
440 fAdditionalLibs += lib;
441}
442
443//______________________________________________________________________________
77f34eae 444void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
445{
446// Adding a module. Checks if already existing. Becomes owned by this.
447 if (!module) return;
448 if (GetModule(module->GetName())) {
449 Error("AddModule", "A module having the same name %s already added", module->GetName());
450 return;
451 }
452 if (!fModules) {
453 fModules = new TObjArray();
454 fModules->SetOwner();
455 }
456 fModules->Add(module);
457}
458
459//______________________________________________________________________________
460void AliAnalysisAlien::AddModules(TObjArray *list)
461{
462// Adding a list of modules. Checks if already existing. Becomes owned by this.
463 TIter next(list);
464 AliAnalysisTaskCfg *module;
465 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
466}
467
468//______________________________________________________________________________
469Bool_t AliAnalysisAlien::CheckDependencies()
470{
471// Check if all dependencies are satisfied. Reorder modules if needed.
472 Int_t nmodules = GetNmodules();
473 if (!nmodules) {
474 Warning("CheckDependencies", "No modules added yet to check their dependencies");
475 return kTRUE;
476 }
477 AliAnalysisTaskCfg *mod = 0;
478 AliAnalysisTaskCfg *dep = 0;
479 TString depname;
480 Int_t i, j, k;
481 for (i=0; i<nmodules; i++) {
482 mod = (AliAnalysisTaskCfg*) fModules->At(i);
483 Int_t ndeps = mod->GetNdeps();
484 Int_t istart = i;
485 for (j=0; j<ndeps; j++) {
486 depname = mod->GetDependency(j);
487 dep = GetModule(depname);
488 if (!dep) {
489 Error("CheckDependencies","Dependency %s not added for module %s",
490 depname.Data(), mod->GetName());
491 return kFALSE;
492 }
493 if (dep->NeedsDependency(mod->GetName())) {
494 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
495 mod->GetName(), dep->GetName());
496 return kFALSE;
497 }
498 Int_t idep = fModules->IndexOf(dep);
499 // The dependency task must come first
500 if (idep>i) {
501 // Remove at idep and move all objects below up one slot
502 // down to index i included.
503 fModules->RemoveAt(idep);
3e40fd4c 504 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 505 fModules->AddAt(dep, i++);
506 }
507 //Redo from istart if dependencies were inserted
508 if (i>istart) i=istart-1;
509 }
510 }
511 return kTRUE;
512}
513
514//______________________________________________________________________________
3e40fd4c 515AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
516{
517// Create the analysis manager and optionally execute the macro in filename.
518 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
519 if (mgr) return mgr;
520 mgr = new AliAnalysisManager(name);
521 mgr->SetGridHandler((AliAnalysisGrid*)this);
522 if (strlen(filename)) {
523 TString line = gSystem->ExpandPathName(filename);
524 line.Prepend(".x ");
525 gROOT->ProcessLine(line.Data());
526 }
527 return mgr;
528}
529
530//______________________________________________________________________________
77f34eae 531Int_t AliAnalysisAlien::GetNmodules() const
532{
533// Get number of modules.
534 if (!fModules) return 0;
535 return fModules->GetEntries();
536}
537
538//______________________________________________________________________________
539AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
540{
541// Get a module by name.
542 if (!fModules) return 0;
543 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
544}
545
546//______________________________________________________________________________
547Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
548{
549// Load a given module.
550 if (mod->IsLoaded()) return kTRUE;
192ddca2 551 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
552 if (!mgr) {
553 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
554 return kFALSE;
555 }
77f34eae 556 Int_t ndeps = mod->GetNdeps();
557 TString depname;
558 for (Int_t j=0; j<ndeps; j++) {
559 depname = mod->GetDependency(j);
560 AliAnalysisTaskCfg *dep = GetModule(depname);
561 if (!dep) {
562 Error("LoadModule","Dependency %s not existing for module %s",
563 depname.Data(), mod->GetName());
564 return kFALSE;
565 }
566 if (!LoadModule(dep)) {
567 Error("LoadModule","Dependency %s for module %s could not be loaded",
568 depname.Data(), mod->GetName());
569 return kFALSE;
570 }
571 }
572 // Load libraries for the module
573 if (!mod->CheckLoadLibraries()) {
574 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
575 return kFALSE;
576 }
192ddca2 577 // Check if a custom file name was requested
578 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
579
580 // Check if a custom terminate file name was requested
581 if (strlen(mod->GetTerminateFileName())) {
582 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
583 fTerminateFiles += mod->GetTerminateFileName();
584 }
585
77f34eae 586 // Execute the macro
587 if (mod->ExecuteMacro()<0) {
588 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
589 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
590 return kFALSE;
591 }
592 // Configure dependencies
593 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
594 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
595 mod->GetConfigMacro()->GetTitle(), mod->GetName());
596 return kFALSE;
597 }
5e8c7c22 598 // Adjust extra libraries
599 Int_t nlibs = mod->GetNlibs();
600 TString lib;
601 for (Int_t i=0; i<nlibs; i++) {
602 lib = mod->GetLibrary(i);
5e8c7c22 603 lib = Form("lib%s.so", lib.Data());
c9d6bd12 604 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 605 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
606 fAdditionalLibs += lib;
607 }
77f34eae 608 return kTRUE;
609}
610
611//______________________________________________________________________________
139fe6ba 612Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
613{
614// Generate the full train.
615 fAdditionalLibs = "";
616 if (!LoadModules()) return kFALSE;
617 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
618 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 619 mgr->RunLocalInit();
139fe6ba 620 mgr->PrintStatus();
621 Int_t productionMode = fProductionMode;
622 SetProductionMode();
623 TString macro = fAnalysisMacro;
624 TString executable = fExecutable;
625 TString validation = fValidationScript;
626 TString execCommand = fExecutableCommand;
627 SetAnalysisMacro(Form("%s.C", name));
628 SetExecutable(Form("%s.sh", name));
a8e31fb0 629// SetExecutableCommand("aliroot -b -q ");
139fe6ba 630 SetValidationScript(Form("%s_validation.sh", name));
631 StartAnalysis();
632 SetProductionMode(productionMode);
633 fAnalysisMacro = macro;
634 fExecutable = executable;
635 fExecutableCommand = execCommand;
636 fValidationScript = validation;
637 return kTRUE;
638}
639
640//______________________________________________________________________________
21ca8e59 641Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 642{
643// Generate test macros for a single module or for the full train.
5e8c7c22 644 fAdditionalLibs = "";
77f34eae 645 if (strlen(modname)) {
21ca8e59 646 if (!CheckDependencies()) return kFALSE;
647 AliAnalysisTaskCfg *mod = GetModule(modname);
648 if (!mod) {
649 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
650 return kFALSE;
651 }
652 if (!LoadModule(mod)) return kFALSE;
653 } else if (!LoadModules()) return kFALSE;
654 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
655 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 656 mgr->RunLocalInit();
21ca8e59 657 mgr->PrintStatus();
658 SetLocalTest(kTRUE);
659 Int_t productionMode = fProductionMode;
660 SetProductionMode();
661 TString macro = fAnalysisMacro;
662 TString executable = fExecutable;
663 TString validation = fValidationScript;
664 TString execCommand = fExecutableCommand;
665 SetAnalysisMacro(Form("%s.C", name));
666 SetExecutable(Form("%s.sh", name));
a8e31fb0 667// SetExecutableCommand("aliroot -b -q ");
21ca8e59 668 SetValidationScript(Form("%s_validation.sh", name));
669 WriteAnalysisFile();
670 WriteAnalysisMacro();
671 WriteExecutable();
672 WriteValidationScript();
673 SetLocalTest(kFALSE);
674 SetProductionMode(productionMode);
675 fAnalysisMacro = macro;
676 fExecutable = executable;
677 fExecutableCommand = execCommand;
678 fValidationScript = validation;
77f34eae 679 return kTRUE;
680}
681
682//______________________________________________________________________________
683Bool_t AliAnalysisAlien::LoadModules()
684{
685// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 686 fAdditionalLibs = "";
21ca8e59 687 Int_t nmodules = GetNmodules();
688 if (!nmodules) {
689 Warning("LoadModules", "No module to be loaded");
690 return kTRUE;
691 }
3e40fd4c 692 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
693 if (!mgr) {
694 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
695 return kFALSE;
696 }
77f34eae 697 if (!CheckDependencies()) return kFALSE;
21ca8e59 698 nmodules = GetNmodules();
77f34eae 699 AliAnalysisTaskCfg *mod;
21ca8e59 700 for (Int_t imod=0; imod<nmodules; imod++) {
701 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 702 if (!LoadModule(mod)) return kFALSE;
703 }
77f34eae 704 return kTRUE;
705}
706
707//______________________________________________________________________________
74013535 708void AliAnalysisAlien::SetRunPrefix(const char *prefix)
709{
710// Set the run number format. Can be a prefix or a format like "%09d"
711 fRunPrefix = prefix;
712 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
713}
714
715//______________________________________________________________________________
f965131e 716void AliAnalysisAlien::AddIncludePath(const char *path)
717{
718// Add include path in the remote analysis macro.
719 TString p(path);
720 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
721 else fIncludePath += Form("-I%s ", path);
722}
723
724//______________________________________________________________________________
c57f56b7 725void AliAnalysisAlien::AddRunNumber(Int_t run)
726{
727// Add a run number to the list of runs to be processed.
728 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 729 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 730}
731
732//______________________________________________________________________________
98690895 733void AliAnalysisAlien::AddRunList(const char* runList)
734{
735// Add several runs into the list of runs; they are expected to be separated by a blank character.
736 TString sList = runList;
737 TObjArray *list = sList.Tokenize(" ");
738 Int_t n = list->GetEntries();
739 for (Int_t i = 0; i < n; i++) {
740 TObjString *os = (TObjString*)list->At(i);
741 AddRunNumber(os->GetString().Atoi());
742 }
743 delete list;
744}
745
746//______________________________________________________________________________
ee75cfc3 747void AliAnalysisAlien::AddRunNumber(const char* run)
748{
749// Add a run number to the list of runs to be processed.
7aa5d539 750 TString runs = run;
751 TObjString *os;
752 TObjArray *arr = runs.Tokenize(" ");
753 TIter next(arr);
754 TString prefix;
755 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
756 while ((os=(TObjString*)next())){
757 if (fRunNumbers.Length()) fRunNumbers += " ";
758 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
759 }
760 delete arr;
ee75cfc3 761}
762
763//______________________________________________________________________________
c57f56b7 764void AliAnalysisAlien::AddDataFile(const char *lfn)
765{
766// Adds a data file to the input to be analysed. The file should be a valid LFN
767// or point to an existing file in the alien workdir.
768 if (!fInputFiles) fInputFiles = new TObjArray();
769 fInputFiles->Add(new TObjString(lfn));
770}
648174cf 771
772//______________________________________________________________________________
773void AliAnalysisAlien::AddExternalPackage(const char *package)
774{
775// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
776 if (fExternalPackages) fExternalPackages += " ";
777 fExternalPackages += package;
778}
779
c57f56b7 780//______________________________________________________________________________
781Bool_t AliAnalysisAlien::Connect()
782{
783// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
784 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 785 if (fProductionMode) return kTRUE;
c57f56b7 786 if (!gGrid) {
787 Info("Connect", "Trying to connect to AliEn ...");
788 TGrid::Connect("alien://");
789 }
790 if (!gGrid || !gGrid->IsConnected()) {
791 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
792 return kFALSE;
793 }
794 fUser = gGrid->GetUser();
795 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
796 return kTRUE;
797}
798
799//______________________________________________________________________________
800void AliAnalysisAlien::CdWork()
801{
802// Check validity of alien workspace. Create directory if possible.
803 if (!Connect()) {
804 Error("CdWork", "Alien connection required");
805 return;
806 }
807 TString homedir = gGrid->GetHomeDirectory();
808 TString workdir = homedir + fGridWorkingDir;
923e2ca5 809 if (DirectoryExists(workdir)) {
810 gGrid->Cd(workdir);
811 return;
812 }
813 // Work directory not existing - create it
814 gGrid->Cd(homedir);
b93f8109 815 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 816 gGrid->Cd(fGridWorkingDir);
d3b18c4c 817 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 818 } else {
d3b18c4c 819 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 820 workdir.Data(), homedir.Data());
821 fGridWorkingDir = "";
822 }
c57f56b7 823}
824
825//______________________________________________________________________________
348be253 826Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
827{
828// Check if file copying is possible.
d3b18c4c 829 if (fProductionMode) return kTRUE;
b5ae4fe0 830 TString salienpath(alienpath);
831 if (salienpath.Contains(" ")) {
832 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
833 return kFALSE;
834 }
348be253 835 if (!Connect()) {
836 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
837 return kFALSE;
838 }
d3339be3 839 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
840 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 841 // Check if alien_CLOSE_SE is defined
842 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
843 if (!closeSE.IsNull()) {
844 Info("CheckFileCopy", "Your current close storage is pointing to: \
845 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
846 } else {
847 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
848 }
849 // Check if grid directory exists.
850 if (!DirectoryExists(alienpath)) {
851 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
852 return kFALSE;
853 }
b5ae4fe0 854 TString stest = "plugin_test_copy";
855 TFile f(stest, "RECREATE");
348be253 856 // User may not have write permissions to current directory
857 if (f.IsZombie()) {
858 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
859 gSystem->WorkingDirectory());
860 return kFALSE;
861 }
862 f.Close();
b5ae4fe0 863 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
864 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 865 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 866 \n# 1. Make sure you have write permissions there. If this is the case: \
867 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
868 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
869 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
870 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 871 gSystem->Unlink(stest.Data());
348be253 872 return kFALSE;
873 }
b5ae4fe0 874 gSystem->Unlink(stest.Data());
875 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 876 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 877 return kTRUE;
878}
879
880//______________________________________________________________________________
c57f56b7 881Bool_t AliAnalysisAlien::CheckInputData()
882{
883// Check validity of input data. If necessary, create xml files.
d3b18c4c 884 if (fProductionMode) return kTRUE;
a8739e8a 885 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 886 if (!fGridDataDir.Length()) {
887 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
888 return kFALSE;
889 }
9f020952 890 if (fMergeViaJDL) {
891 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
892 return kFALSE;
893 }
d2a409b2 894 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 895 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
896 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 897 return kTRUE;
c57f56b7 898 }
899 // Process declared files
a2f5fc01 900 Bool_t isCollection = kFALSE;
901 Bool_t isXml = kFALSE;
902 Bool_t useTags = kFALSE;
c57f56b7 903 Bool_t checked = kFALSE;
d3b18c4c 904 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 905 TString file;
906 TString workdir = gGrid->GetHomeDirectory();
907 workdir += fGridWorkingDir;
908 if (fInputFiles) {
909 TObjString *objstr;
910 TIter next(fInputFiles);
911 while ((objstr=(TObjString*)next())) {
912 file = workdir;
913 file += "/";
914 file += objstr->GetString();
915 // Store full lfn path
916 if (FileExists(file)) objstr->SetString(file);
917 else {
918 file = objstr->GetName();
919 if (!FileExists(objstr->GetName())) {
920 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
921 objstr->GetName(), workdir.Data());
922 return kFALSE;
923 }
924 }
925 Bool_t iscoll, isxml, usetags;
926 CheckDataType(file, iscoll, isxml, usetags);
927 if (!checked) {
928 checked = kTRUE;
a2f5fc01 929 isCollection = iscoll;
930 isXml = isxml;
931 useTags = usetags;
932 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 933 } else {
a2f5fc01 934 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 935 Error("CheckInputData", "Some conflict was found in the types of inputs");
936 return kFALSE;
937 }
938 }
939 }
940 }
941 // Process requested run numbers
a8739e8a 942 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 943 // Check validity of alien data directory
944 if (!fGridDataDir.Length()) {
945 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
946 return kFALSE;
947 }
923e2ca5 948 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 949 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
950 return kFALSE;
951 }
a2f5fc01 952 if (isCollection) {
c57f56b7 953 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
954 return kFALSE;
955 }
956
a2f5fc01 957 if (checked && !isXml) {
c57f56b7 958 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
959 return kFALSE;
960 }
961 // Check validity of run number(s)
962 TObjArray *arr;
963 TObjString *os;
74013535 964 TString format;
319593fb 965 Int_t nruns = 0;
904f9f5f 966 TString schunk, schunk2;
c57f56b7 967 TString path;
968 if (!checked) {
969 checked = kTRUE;
a2f5fc01 970 useTags = fDataPattern.Contains("tag");
971 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 972 }
a2f5fc01 973 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 974 Error("CheckInputData", "Cannot mix input files using/not using tags");
975 return kFALSE;
976 }
977 if (fRunNumbers.Length()) {
a8739e8a 978 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 979 arr = fRunNumbers.Tokenize(" ");
980 TIter next(arr);
981 while ((os=(TObjString*)next())) {
982 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 983 if (!DirectoryExists(path)) {
984 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 985 continue;
c57f56b7 986 }
987 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
988 TString msg = "\n##### file: ";
989 msg += path;
990 msg += " type: xml_collection;";
a2f5fc01 991 if (useTags) msg += " using_tags: Yes";
c57f56b7 992 else msg += " using_tags: No";
23329835 993 Info("CheckDataType", "%s", msg.Data());
319593fb 994 if (fNrunsPerMaster<2) {
d2a409b2 995 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 996 } else {
997 nruns++;
998 if (((nruns-1)%fNrunsPerMaster) == 0) {
999 schunk = os->GetString();
1000 }
1001 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1002 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1003 AddDataFile(schunk);
319593fb 1004 }
c57f56b7 1005 }
1006 delete arr;
a8739e8a 1007 } else {
1008 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1009 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1010 format = Form("%%s/%s ", fRunPrefix.Data());
1011 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1012 if (!DirectoryExists(path)) {
a8739e8a 1013 continue;
1014 }
74013535 1015 format = Form("%%s/%s.xml", fRunPrefix.Data());
1016 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1017 TString msg = "\n##### file: ";
1018 msg += path;
1019 msg += " type: xml_collection;";
a2f5fc01 1020 if (useTags) msg += " using_tags: Yes";
a8739e8a 1021 else msg += " using_tags: No";
23329835 1022 Info("CheckDataType", "%s", msg.Data());
319593fb 1023 if (fNrunsPerMaster<2) {
74013535 1024 format = Form("%s.xml", fRunPrefix.Data());
1025 AddDataFile(Form(format.Data(),irun));
319593fb 1026 } else {
1027 nruns++;
1028 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1029 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1030 }
74013535 1031 format = Form("_%s.xml", fRunPrefix.Data());
1032 schunk2 = Form(format.Data(), irun);
319593fb 1033 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1034 schunk += schunk2;
d2a409b2 1035 AddDataFile(schunk);
319593fb 1036 }
a8739e8a 1037 }
904f9f5f 1038 if (!fInputFiles) {
1039 schunk += schunk2;
1040 AddDataFile(schunk);
1041 }
c57f56b7 1042 }
1043 return kTRUE;
1044}
1045
1046//______________________________________________________________________________
43b807f6 1047Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1048{
1049// Copy data from the given grid directory according a pattern and make a local
1050// dataset.
2ba4c287 1051// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1052 if (!Connect()) {
1053 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
1054 return kFALSE;
1055 }
1056 if (!DirectoryExists(griddir)) {
1057 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
1058 return kFALSE;
1059 }
1060 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1061 printf("Running command: %s\n", command.Data());
1062 TGridResult *res = gGrid->Command(command);
1063 Int_t nfound = res->GetEntries();
1064 if (!nfound) {
1065 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
1066 return kFALSE;
1067 }
1068 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1069
1070 // archives
1071 TObjArray* additionalArchives = 0;
1072 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1073 additionalArchives = TString(archivefile).Tokenize(";");
1074 archivefile = additionalArchives->At(0)->GetName();
1075 additionalArchives->RemoveAt(0);
1076 additionalArchives->Compress();
1077 }
1078
21ca8e59 1079 // Copy files locally
1080 ofstream out;
1081 out.open(output, ios::out);
1082 TMap *map;
1083 TString turl, dirname, filename, temp;
1084 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1085 gSystem->MakeDirectory(outputdir);
1086 gSystem->ChangeDirectory(outputdir);
21ca8e59 1087 for (Int_t i=0; i<nfound; i++) {
1088 map = (TMap*)res->At(i);
1089 turl = map->GetValue("turl")->GetName();
1090 filename = gSystem->BaseName(turl.Data());
1091 dirname = gSystem->DirName(turl.Data());
1092 dirname = gSystem->BaseName(dirname.Data());
1093 gSystem->MakeDirectory(dirname);
43b807f6 1094
1095 TString source(turl);
1096 TString targetFileName(filename);
1097
1098 if (strlen(archivefile) > 0) {
1099 // TODO here the archive in which the file resides should be determined
1100 // however whereis returns only a guid, and guid2lfn does not work
1101 // Therefore we use the one provided as argument for now
1102 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1103 targetFileName = archivefile;
1104 }
1105 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
2ba4c287 1106 Bool_t success = kTRUE;
1107 if (additionalArchives)
1108 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++)
1109 {
1110 TString target;
1111 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1112 gSystem->MakeDirectory(gSystem->DirName(target));
1113 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1114 }
1115
1116 if (success) {
1117 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1118 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1119 }
21ca8e59 1120 }
1121 }
1122 gSystem->ChangeDirectory(cdir);
1123 delete res;
2ba4c287 1124 if (additionalArchives)
1125 delete additionalArchives;
21ca8e59 1126 return kTRUE;
1127}
1128
1129//______________________________________________________________________________
c57f56b7 1130Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1131{
1132// Create dataset for the grid data directory + run number.
3af861ff 1133 const Int_t gMaxEntries = 15000;
d3b18c4c 1134 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1135 if (!Connect()) {
1136 Error("CreateDataset", "Cannot create dataset with no grid connection");
1137 return kFALSE;
1138 }
1139
1140 // Cd workspace
d3b18c4c 1141 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1142 TString workdir = gGrid->GetHomeDirectory();
1143 workdir += fGridWorkingDir;
1144
1145 // Compose the 'find' command arguments
74013535 1146 TString format;
c57f56b7 1147 TString command;
1148 TString options = "-x collection ";
bb885a9e 1149 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1150 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1151 TString conditions = "";
3af861ff 1152 Int_t nstart = 0;
1153 Int_t ncount = 0;
1154 Int_t stage = 0;
c57f56b7 1155 TString file;
1156 TString path;
319593fb 1157 Int_t nruns = 0;
904f9f5f 1158 TString schunk, schunk2;
ab254fd1 1159 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1160 if (!fRunNumbers.Length() && !fRunRange[0]) {
1161 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1162 // Make a single data collection from data directory.
1163 path = fGridDataDir;
923e2ca5 1164 if (!DirectoryExists(path)) {
d2a409b2 1165 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1166 return kFALSE;
1167 }
923e2ca5 1168// CdWork();
d2a409b2 1169 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1170 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1171 while (1) {
1172 ncount = 0;
1173 stage++;
1174 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1175 command = "find ";
1176 command += Form("%s -o %d ",options.Data(), nstart);
1177 command += path;
1178 command += " ";
1179 command += pattern;
1180 command += conditions;
1181 printf("command: %s\n", command.Data());
1182 TGridResult *res = gGrid->Command(command);
1183 if (res) delete res;
1184 // Write standard output to file
1185 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1186 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1187 Bool_t nullFile = kFALSE;
1188 if (!hasGrep) {
1189 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1190 } else {
1191 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1192 if (nullFile) {
1193 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1194 gSystem->Exec("rm -f __tmp*");
1195 return kFALSE;
1196 }
1197 TString line;
1198 ifstream in;
1199 in.open("__tmp__");
1200 in >> line;
1201 in.close();
1202 gSystem->Exec("rm -f __tmp__");
1203 ncount = line.Atoi();
1204 }
1205 }
1206 if (ncount == gMaxEntries) {
1207 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1208 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1209 if (!cbase) cbase = cadd;
1210 else {
1211 cbase->Add(cadd);
1212 delete cadd;
defd7a3a 1213 }
3af861ff 1214 nstart += ncount;
1215 } else {
1216 if (cbase) {
1217 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1218 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1219 cbase->Add(cadd);
1220 delete cadd;
1221 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1222 delete cbase; cbase = 0;
1223 } else {
1224 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1225 }
1226 gSystem->Exec("rm -f __tmp*");
1227 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1228 break;
1229 }
0f389141 1230 }
1231 Bool_t fileExists = FileExists(file);
1232 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1233 // Copy xml file to alien space
0f389141 1234 if (fileExists) gGrid->Rm(file);
d2a409b2 1235 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1236 if (!FileExists(file)) {
1237 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1238 return kFALSE;
1239 }
1240 // Update list of files to be processed.
1241 }
1242 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1243 return kTRUE;
1244 }
c57f56b7 1245 // Several runs
a2f5fc01 1246 Bool_t nullResult = kTRUE;
a8739e8a 1247 if (fRunNumbers.Length()) {
1248 TObjArray *arr = fRunNumbers.Tokenize(" ");
1249 TObjString *os;
1250 TIter next(arr);
1251 while ((os=(TObjString*)next())) {
3af861ff 1252 nstart = 0;
1253 stage = 0;
dc030a24 1254 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1255 if (!DirectoryExists(path)) continue;
1256// CdWork();
a8739e8a 1257 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1258 else file = Form("%s.xml", os->GetString().Data());
319593fb 1259 // If local collection file does not exist, create it via 'find' command.
3af861ff 1260 while (1) {
1261 ncount = 0;
1262 stage++;
1263 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1264 command = "find ";
1265 command += Form("%s -o %d ",options.Data(), nstart);
1266 command += path;
1267 command += pattern;
1268 command += conditions;
1269 TGridResult *res = gGrid->Command(command);
1270 if (res) delete res;
1271 // Write standard output to file
1272 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1273 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1274 Bool_t nullFile = kFALSE;
1275 if (!hasGrep) {
1276 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1277 } else {
1278 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1279 if (nullFile) {
1280 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1281 gSystem->Exec("rm -f __tmp*");
1282 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1283 break;
3af861ff 1284 }
1285 TString line;
1286 ifstream in;
1287 in.open("__tmp__");
1288 in >> line;
1289 in.close();
1290 gSystem->Exec("rm -f __tmp__");
1291 ncount = line.Atoi();
1292 }
1293 nullResult = kFALSE;
1294 }
1295 if (ncount == gMaxEntries) {
1296 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1297 if (fNrunsPerMaster > 1) {
1298 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1299 file.Data(),gMaxEntries);
1300 return kFALSE;
1301 }
1302 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1303 if (!cbase) cbase = cadd;
1304 else {
1305 cbase->Add(cadd);
1306 delete cadd;
defd7a3a 1307 }
3af861ff 1308 nstart += ncount;
1309 } else {
c89a84bc 1310 if (cbase && fNrunsPerMaster<2) {
3af861ff 1311 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1312 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1313 cbase->Add(cadd);
1314 delete cadd;
1315 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1316 delete cbase; cbase = 0;
1317 } else {
1318 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1319 }
1320 gSystem->Exec("rm -f __tmp*");
1321 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1322 break;
d3339be3 1323 }
3af861ff 1324 }
a8739e8a 1325 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1326 // Check if there is one run per master job.
1327 if (fNrunsPerMaster<2) {
1328 if (FileExists(file)) {
0f389141 1329 if (fOverwriteMode) gGrid->Rm(file);
1330 else {
1331 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1332 continue;
1333 }
319593fb 1334 }
1335 // Copy xml file to alien space
1336 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1337 if (!FileExists(file)) {
1338 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1339 delete arr;
1340 return kFALSE;
1341 }
1342 } else {
1343 nruns++;
1344 if (((nruns-1)%fNrunsPerMaster) == 0) {
1345 schunk = os->GetString();
1346 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1347 } else {
1348 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1349 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1350 cbase->Add(cadd);
1351 delete cadd;
1352 }
1353 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1354 continue;
1355 }
1356 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1357 if (FileExists(schunk)) {
1358 if (fOverwriteMode) gGrid->Rm(file);
1359 else {
1360 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1361 continue;
1362 }
319593fb 1363 }
84fcd93f 1364 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1365 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1366 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1367 if (!FileExists(schunk)) {
1368 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1369 delete arr;
1370 return kFALSE;
1371 }
d3339be3 1372 }
a8739e8a 1373 }
1374 delete arr;
a2f5fc01 1375 if (nullResult) {
d3339be3 1376 Error("CreateDataset", "No valid dataset corresponding to the query!");
1377 return kFALSE;
1378 }
a8739e8a 1379 } else {
1380 // Process a full run range.
1381 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1382 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1383 nstart = 0;
1384 stage = 0;
74013535 1385 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1386 if (!DirectoryExists(path)) continue;
1387// CdWork();
74013535 1388 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1389 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1390 else file = Form(format.Data(), irun);
0f389141 1391 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1392 if (fOverwriteMode) gGrid->Rm(file);
1393 else {
1394 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1395 continue;
1396 }
a8739e8a 1397 }
319593fb 1398 // If local collection file does not exist, create it via 'find' command.
3af861ff 1399 while (1) {
1400 ncount = 0;
1401 stage++;
1402 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1403 command = "find ";
1404 command += Form("%s -o %d ",options.Data(), nstart);
1405 command += path;
1406 command += pattern;
1407 command += conditions;
1408 TGridResult *res = gGrid->Command(command);
1409 if (res) delete res;
1410 // Write standard output to file
1411 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1412 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1413 Bool_t nullFile = kFALSE;
1414 if (!hasGrep) {
1415 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1416 } else {
1417 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1418 if (nullFile) {
1419 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1420 gSystem->Exec("rm -f __tmp*");
1604c9ed 1421 break;
3af861ff 1422 }
1423 TString line;
1424 ifstream in;
1425 in.open("__tmp__");
1426 in >> line;
1427 in.close();
1428 gSystem->Exec("rm -f __tmp__");
1429 ncount = line.Atoi();
1430 }
1431 nullResult = kFALSE;
1432 }
1433 if (ncount == gMaxEntries) {
1434 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1435 if (fNrunsPerMaster > 1) {
1436 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1437 file.Data(),gMaxEntries);
1438 return kFALSE;
1439 }
1440 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1441 if (!cbase) cbase = cadd;
1442 else {
1443 cbase->Add(cadd);
1444 delete cadd;
defd7a3a 1445 }
3af861ff 1446 nstart += ncount;
1447 } else {
c89a84bc 1448 if (cbase && fNrunsPerMaster<2) {
3af861ff 1449 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1450 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1451 cbase->Add(cadd);
1452 delete cadd;
1453 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1454 delete cbase; cbase = 0;
1455 } else {
1456 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1457 }
1458 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1459 break;
d3339be3 1460 }
319593fb 1461 }
a8739e8a 1462 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1463 // Check if there is one run per master job.
1464 if (fNrunsPerMaster<2) {
1465 if (FileExists(file)) {
0f389141 1466 if (fOverwriteMode) gGrid->Rm(file);
1467 else {
1468 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1469 continue;
1470 }
319593fb 1471 }
1472 // Copy xml file to alien space
1473 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1474 if (!FileExists(file)) {
1475 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1476 return kFALSE;
1477 }
1478 } else {
1479 nruns++;
95e5b448 1480 // Check if the collection for the chunk exist locally.
1481 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1482 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1483 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1484 else continue;
1485 }
84fcd93f 1486 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1487 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1488 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1489 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1490 } else {
1491 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1492 cbase->Add(cadd);
1493 delete cadd;
1494 }
74013535 1495 format = Form("%%s_%s.xml", fRunPrefix.Data());
1496 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1497 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1498 continue;
1499 }
904f9f5f 1500 schunk = schunk2;
319593fb 1501 if (FileExists(schunk)) {
0f389141 1502 if (fOverwriteMode) gGrid->Rm(schunk);
1503 else {
1504 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1505 continue;
1506 }
319593fb 1507 }
84fcd93f 1508 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1509 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1510 if (FileExists(schunk)) {
0f389141 1511 if (fOverwriteMode) gGrid->Rm(schunk);
1512 else {
1513 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1514 continue;
1515 }
95e5b448 1516 }
319593fb 1517 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1518 if (!FileExists(schunk)) {
1519 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1520 return kFALSE;
1521 }
1522 }
c57f56b7 1523 }
a2f5fc01 1524 if (nullResult) {
d3339be3 1525 Error("CreateDataset", "No valid dataset corresponding to the query!");
1526 return kFALSE;
1527 }
a8739e8a 1528 }
c57f56b7 1529 return kTRUE;
1530}
1531
1532//______________________________________________________________________________
1533Bool_t AliAnalysisAlien::CreateJDL()
1534{
1535// Generate a JDL file according to current settings. The name of the file is
1536// specified by fJDLName.
1537 Bool_t error = kFALSE;
1538 TObjArray *arr = 0;
1539 Bool_t copy = kTRUE;
d3b18c4c 1540 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1541 Bool_t generate = kTRUE;
1542 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1543 if (!Connect()) {
1544 Error("CreateJDL", "Alien connection required");
1545 return kFALSE;
1546 }
1547 // Check validity of alien workspace
d3b18c4c 1548 TString workdir;
1549 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1550 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1551 workdir += fGridWorkingDir;
1552 if (generate) {
1553 TObjString *os;
1554 if (!fInputFiles) {
1555 Error("CreateJDL()", "Define some input files for your analysis.");
1556 error = kTRUE;
1557 }
1558 // Compose list of input files
1559 // Check if output files were defined
1560 if (!fOutputFiles.Length()) {
1561 Error("CreateJDL", "You must define at least one output file");
1562 error = kTRUE;
1563 }
1564 // Check if an output directory was defined and valid
1565 if (!fGridOutputDir.Length()) {
1566 Error("CreateJDL", "You must define AliEn output directory");
1567 error = kTRUE;
1568 } else {
d3b18c4c 1569 if (!fProductionMode) {
1570 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1571 if (!DirectoryExists(fGridOutputDir)) {
1572 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1573 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1574 } else {
1575 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1576 // error = kTRUE;
1577 }
67a3b254 1578 } else {
1579 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1580 }
1581 gGrid->Cd(workdir);
1582 }
c57f56b7 1583 }
1584 // Exit if any error up to now
1585 if (error) return kFALSE;
1586 // Set JDL fields
0f389141 1587 if (!fUser.IsNull()) {
1588 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1589 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1590 }
1591 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1592 TString mergeExec = fExecutable;
1593 mergeExec.ReplaceAll(".sh", "_merge.sh");
1594 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1595 mergeExec.ReplaceAll(".sh", ".C");
1596 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1597 if (!fArguments.IsNull())
1598 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1599 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1600 else {
1601 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1602 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1603 }
b3e07543 1604
1f0d1ca2 1605 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1606 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1607 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1608 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1609
0f389141 1610 if (fMaxInitFailed > 0) {
c57f56b7 1611 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1612 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1613 }
1614 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1615 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1616 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1617 }
1618 if (!IsOneStageMerging()) {
f47d5cb4 1619 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1620 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1621 }
1622 if (fSplitMode.Length()) {
c57f56b7 1623 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1624 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1625 }
1626 fMergingJDL->SetValue("Split", "\"se\"");
1627 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1628 if (!fAliROOTVersion.IsNull()) {
1629 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1630 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1631 }
1632 if (!fROOTVersion.IsNull()) {
c57f56b7 1633 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1634 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1635 }
1636 if (!fAPIVersion.IsNull()) {
c57f56b7 1637 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1638 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1639 }
648174cf 1640 if (!fExternalPackages.IsNull()) {
1641 arr = fExternalPackages.Tokenize(" ");
1642 TIter next(arr);
1643 while ((os=(TObjString*)next())) {
1644 TString pkgname = os->GetString();
1645 Int_t index = pkgname.Index("::");
1646 TString pkgversion = pkgname(index+2, pkgname.Length());
1647 pkgname.Remove(index);
1648 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1649 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1650 }
1651 delete arr;
1652 }
0f389141 1653 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1654 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1655 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1656 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1657 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1658 TString analysisFile = fExecutable;
1659 analysisFile.ReplaceAll(".sh", ".root");
1660 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1661 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1662 if (fAdditionalLibs.Length()) {
1663 arr = fAdditionalLibs.Tokenize(" ");
1664 TIter next(arr);
1665 while ((os=(TObjString*)next())) {
1666 if (os->GetString().Contains(".so")) continue;
1667 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1668 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1669 }
1670 delete arr;
1671 }
4e5c5506 1672 if (fPackages) {
1673 TIter next(fPackages);
1674 TObject *obj;
0f389141 1675 while ((obj=next())) {
4e5c5506 1676 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1677 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1678 }
4e5c5506 1679 }
c57f56b7 1680 if (fOutputArchive.Length()) {
e4d9a069 1681 TString outputArchive = fOutputArchive;
1682 if (!fRegisterExcludes.IsNull()) {
1683 arr = fRegisterExcludes.Tokenize(" ");
1684 TIter next1(arr);
1685 while ((os=(TObjString*)next1())) {
1686 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1687 outputArchive.ReplaceAll(os->GetString(),"");
1688 }
1689 delete arr;
1690 }
1691 arr = outputArchive.Tokenize(" ");
c57f56b7 1692 TIter next(arr);
0f389141 1693 Bool_t first = kTRUE;
1694 const char *comment = "Files to be archived";
1695 const char *comment1 = comment;
1696 while ((os=(TObjString*)next())) {
1697 if (!first) comment = NULL;
1698 if (!os->GetString().Contains("@") && fCloseSE.Length())
1699 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1700 else
1701 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1702 first = kFALSE;
1703 }
c57f56b7 1704 delete arr;
f790bc1b 1705 // Output archive for the merging jdl
f790bc1b 1706 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1707 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1708 // Add normal output files, extra files + terminate files
1709 TString files = GetListOfFiles("outextter");
37d21c01 1710 // Do not register files in fRegisterExcludes
1711 if (!fRegisterExcludes.IsNull()) {
1712 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1713 TIter next1(arr);
1714 while ((os=(TObjString*)next1())) {
1715 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1716 files.ReplaceAll(os->GetString(),"");
1717 }
66d71516 1718 delete arr;
0f389141 1719 }
f790bc1b 1720 files.ReplaceAll(".root", "*.root");
b83f84f0 1721 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1722 } else {
95f84779 1723 TString files = fOutputArchive;
1724 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1725 outputArchive = files;
f790bc1b 1726 }
0f389141 1727 arr = outputArchive.Tokenize(" ");
705adb3e 1728 TIter next2(arr);
0f389141 1729 comment = comment1;
1730 first = kTRUE;
705adb3e 1731 while ((os=(TObjString*)next2())) {
0f389141 1732 if (!first) comment = NULL;
0cdf65a8 1733 TString currentfile = os->GetString();
0cdf65a8 1734 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1735 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1736 else
0cdf65a8 1737 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1738 first = kFALSE;
1739 }
1740 delete arr;
c57f56b7 1741 }
149d288c 1742 arr = fOutputFiles.Tokenize(",");
c57f56b7 1743 TIter next(arr);
0f389141 1744 Bool_t first = kTRUE;
f790bc1b 1745 const char *comment = "Files to be saved";
43da816a 1746 while ((os=(TObjString*)next())) {
e1eaf596 1747 // Ignore ouputs in jdl that are also in outputarchive
1748 TString sout = os->GetString();
f790bc1b 1749 sout.ReplaceAll("*", "");
1750 sout.ReplaceAll(".root", "");
e1eaf596 1751 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1752 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1753 // Ignore fRegisterExcludes
1754 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1755 if (!first) comment = NULL;
43da816a 1756 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1757 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1758 else
0f389141 1759 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1760 first = kFALSE;
f790bc1b 1761 if (fMergeExcludes.Contains(sout)) continue;
1762 if (!os->GetString().Contains("@") && fCloseSE.Length())
1763 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1764 else
1765 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1766 }
c57f56b7 1767 delete arr;
0f389141 1768 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1769 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1770 TString validationScript = fValidationScript;
0f389141 1771 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1772 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1773 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1774 if (fMasterResubmitThreshold) {
1775 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1776 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1777 }
d2a409b2 1778 // Write a jdl with 2 input parameters: collection name and output dir name.
1779 WriteJDL(copy);
c57f56b7 1780 }
1781 // Copy jdl to grid workspace
a8739e8a 1782 if (copy) {
b5e4aaa7 1783 // Check if an output directory was defined and valid
1784 if (!fGridOutputDir.Length()) {
1785 Error("CreateJDL", "You must define AliEn output directory");
1786 return kFALSE;
1787 } else {
1788 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1789 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1790 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1791 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1792 } else {
1793 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1794 return kFALSE;
1795 }
1796 }
1797 gGrid->Cd(workdir);
1798 }
648174cf 1799 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1800 TString mergeJDLName = fExecutable;
1801 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1802 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1803 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1804 if (fProductionMode) {
648174cf 1805 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1806 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1807 }
648174cf 1808 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1809 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1810 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1811 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1812 Fatal("","Terminating");
1813// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1814 if (fMergeViaJDL) {
1815 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1816// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1817 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1818 Fatal("","Terminating");
0f389141 1819 }
648174cf 1820 }
c57f56b7 1821 if (fAdditionalLibs.Length()) {
e7c71df0 1822 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1823 TObjString *os;
1824 TIter next(arr);
1825 while ((os=(TObjString*)next())) {
c57f56b7 1826 if (os->GetString().Contains(".so")) continue;
4e5c5506 1827 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1828 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1829// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1830 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1831 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1832 Fatal("","Terminating");
c57f56b7 1833 }
1834 delete arr;
1835 }
4e5c5506 1836 if (fPackages) {
1837 TIter next(fPackages);
1838 TObject *obj;
1839 while ((obj=next())) {
fdbbc7be 1840 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1841 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1842// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1843 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1844 Form("%s/%s", workdir.Data(), obj->GetName())))
1845 Fatal("","Terminating");
4e5c5506 1846 }
1847 }
c57f56b7 1848 }
1849 return kTRUE;
1850}
1851
1852//______________________________________________________________________________
d2a409b2 1853Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1854{
1855// Writes one or more JDL's corresponding to findex. If findex is negative,
1856// all run numbers are considered in one go (jdl). For non-negative indices
1857// they correspond to the indices in the array fInputFiles.
1858 if (!fInputFiles) return kFALSE;
d3b18c4c 1859 TObject *os;
1860 TString workdir;
1861 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1862 workdir += fGridWorkingDir;
f47d5cb4 1863 TString stageName = "$2";
1864 if (fProductionMode) stageName = "$4";
1865 if (!fMergeDirName.IsNull()) {
1866 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1867 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1868 } else {
1869 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1870 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1871 }
d3b18c4c 1872 if (fProductionMode) {
a8739e8a 1873 TIter next(fInputFiles);
b3e07543 1874 while ((os=next())) {
d3b18c4c 1875 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1876 }
56754bf6 1877 if (!fOutputToRunNo)
1878 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1879 else
1880 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1881 } else {
1882 if (!fRunNumbers.Length() && !fRunRange[0]) {
1883 // One jdl with no parameters in case input data is specified by name.
1884 TIter next(fInputFiles);
1885 while ((os=next()))
1886 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1887 if (!fOutputSingle.IsNull())
1888 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1889 else {
1890 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1891 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1892 }
1893 } else {
1894 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1895 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1896 if (!fOutputSingle.IsNull()) {
1897 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1898 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1899 } else {
1900 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1901 }
1902 }
a8739e8a 1903 }
1904
a8739e8a 1905 // Generate the JDL as a string
1906 TString sjdl = fGridJDL->Generate();
0f389141 1907 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1908 // Final merge jdl
1909 if (!fMergeDirName.IsNull()) {
1910 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1911 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1912 } else {
1913 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1914 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1915 }
b3e07543 1916 TString sjdl2 = fMergingJDL->Generate();
1917 Int_t index, index1;
a8739e8a 1918 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1919 sjdl.ReplaceAll("(member", "\n (member");
1920 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1921 sjdl.ReplaceAll("{", "{\n ");
1922 sjdl.ReplaceAll("};", "\n};");
1923 sjdl.ReplaceAll("{\n \n", "{\n");
1924 sjdl.ReplaceAll("\n\n", "\n");
1925 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1926 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1927 sjdl1.ReplaceAll("(member", "\n (member");
1928 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1929 sjdl1.ReplaceAll("{", "{\n ");
1930 sjdl1.ReplaceAll("};", "\n};");
1931 sjdl1.ReplaceAll("{\n \n", "{\n");
1932 sjdl1.ReplaceAll("\n\n", "\n");
1933 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1934 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1935 sjdl2.ReplaceAll("(member", "\n (member");
1936 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1937 sjdl2.ReplaceAll("{", "{\n ");
1938 sjdl2.ReplaceAll("};", "\n};");
1939 sjdl2.ReplaceAll("{\n \n", "{\n");
1940 sjdl2.ReplaceAll("\n\n", "\n");
1941 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1942 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1943 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1944 index = sjdl.Index("JDLVariables");
1945 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1946 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1947 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1948 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1949 index = fJobTag.Index(":");
1950 if (index < 0) index = fJobTag.Length();
1951 TString jobTag = fJobTag;
f47d5cb4 1952 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1953 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1954 if (fProductionMode) {
1955 sjdl1.Prepend("# Generated merging jdl (production mode) \
1956 \n# $1 = full alien path to output directory to be merged \
1957 \n# $2 = train number \
1958 \n# $3 = production (like LHC10b) \
1959 \n# $4 = merging stage \
1960 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1961 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1962 sjdl2.Prepend("# Generated merging jdl \
1963 \n# $1 = full alien path to output directory to be merged \
1964 \n# $2 = train number \
1965 \n# $3 = production (like LHC10b) \
1966 \n# $4 = merging stage \
1967 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1968 } else {
1969 sjdl1.Prepend("# Generated merging jdl \
1970 \n# $1 = full alien path to output directory to be merged \
1971 \n# $2 = merging stage \
03335324 1972 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1973 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1974 sjdl2.Prepend("# Generated merging jdl \
1975 \n# $1 = full alien path to output directory to be merged \
1976 \n# $2 = merging stage \
03335324 1977 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1978 }
0f389141 1979 index = sjdl1.Index("JDLVariables");
1980 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1981 index = sjdl2.Index("JDLVariables");
1982 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1983 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1984 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1985 index = sjdl2.Index("Split =");
1986 if (index>=0) {
1987 index1 = sjdl2.Index("\n", index);
1988 sjdl2.Remove(index, index1-index+1);
1989 }
1990 index = sjdl2.Index("SplitMaxInputFileNumber");
1991 if (index>=0) {
1992 index1 = sjdl2.Index("\n", index);
1993 sjdl2.Remove(index, index1-index+1);
1994 }
1995 index = sjdl2.Index("InputDataCollection");
1996 if (index>=0) {
1997 index1 = sjdl2.Index(";", index);
1998 sjdl2.Remove(index, index1-index+1);
1999 }
2000 index = sjdl2.Index("InputDataListFormat");
2001 if (index>=0) {
2002 index1 = sjdl2.Index("\n", index);
2003 sjdl2.Remove(index, index1-index+1);
2004 }
2005 index = sjdl2.Index("InputDataList");
2006 if (index>=0) {
2007 index1 = sjdl2.Index("\n", index);
2008 sjdl2.Remove(index, index1-index+1);
2009 }
f47d5cb4 2010 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2011 // Write jdl to file
a8739e8a 2012 ofstream out;
d2a409b2 2013 out.open(fJDLName.Data(), ios::out);
a8739e8a 2014 if (out.bad()) {
d3b18c4c 2015 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2016 return kFALSE;
2017 }
2018 out << sjdl << endl;
b3e07543 2019 out.close();
0f389141 2020 TString mergeJDLName = fExecutable;
2021 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2022 if (fMergeViaJDL) {
2023 ofstream out1;
2024 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2025 if (out1.bad()) {
d3b18c4c 2026 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2027 return kFALSE;
2028 }
2029 out1 << sjdl1 << endl;
b3e07543 2030 out1.close();
2031 ofstream out2;
2032 TString finalJDL = mergeJDLName;
2033 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2034 out2.open(finalJDL.Data(), ios::out);
2035 if (out2.bad()) {
2036 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2037 return kFALSE;
2038 }
2039 out2 << sjdl2 << endl;
2040 out2.close();
0f389141 2041 }
a8739e8a 2042
2043 // Copy jdl to grid workspace
2044 if (!copy) {
d3b18c4c 2045 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2046 } else {
d2a409b2 2047 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2048 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2049 TString finalJDL = mergeJDLName;
2050 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2051 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2052 if (fProductionMode) {
b5fe9cba 2053 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2054 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2055 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2056 }
d2a409b2 2057 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2058 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2059 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2060 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2061// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2062 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2063 Fatal("","Terminating");
0f389141 2064 if (fMergeViaJDL) {
b3e07543 2065 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2066// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2067// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2068 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2069 Fatal("","Terminating");
2070 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2071 Fatal("","Terminating");
0f389141 2072 }
a8739e8a 2073 }
2074 return kTRUE;
2075}
2076
2077//______________________________________________________________________________
5513444a 2078Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2079{
2080// Returns true if file exists.
5513444a 2081 if (!gGrid) return kFALSE;
2ea55496 2082 TString slfn = lfn;
2083 slfn.ReplaceAll("alien://","");
2084 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2085 if (!res) return kFALSE;
2086 TMap *map = dynamic_cast<TMap*>(res->At(0));
2087 if (!map) {
2088 delete res;
2089 return kFALSE;
2090 }
2091 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2092 if (!objs || !objs->GetString().Length()) {
2093 delete res;
2094 return kFALSE;
2095 }
2096 delete res;
2097 return kTRUE;
2098}
2099
2100//______________________________________________________________________________
923e2ca5 2101Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2102{
2103// Returns true if directory exists. Can be also a path.
2104 if (!gGrid) return kFALSE;
2105 // Check if dirname is a path
2106 TString dirstripped = dirname;
2107 dirstripped = dirstripped.Strip();
2108 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2109 TString dir = gSystem->BaseName(dirstripped);
2110 dir += "/";
2111 TString path = gSystem->DirName(dirstripped);
2112 TGridResult *res = gGrid->Ls(path, "-F");
2113 if (!res) return kFALSE;
2114 TIter next(res);
2115 TMap *map;
2116 TObject *obj;
2117 while ((map=dynamic_cast<TMap*>(next()))) {
2118 obj = map->GetValue("name");
2119 if (!obj) break;
2120 if (dir == obj->GetName()) {
2121 delete res;
2122 return kTRUE;
2123 }
2124 }
2125 delete res;
2126 return kFALSE;
2127}
2128
2129//______________________________________________________________________________
a2f5fc01 2130void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2131{
2132// Check input data type.
a2f5fc01 2133 isCollection = kFALSE;
2134 isXml = kFALSE;
2135 useTags = kFALSE;
c57f56b7 2136 if (!gGrid) {
2137 Error("CheckDataType", "No connection to grid");
2138 return;
2139 }
a2f5fc01 2140 isCollection = IsCollection(lfn);
c57f56b7 2141 TString msg = "\n##### file: ";
2142 msg += lfn;
a2f5fc01 2143 if (isCollection) {
c57f56b7 2144 msg += " type: raw_collection;";
2145 // special treatment for collections
a2f5fc01 2146 isXml = kFALSE;
c57f56b7 2147 // check for tag files in the collection
2148 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2149 if (!res) {
2150 msg += " using_tags: No (unknown)";
23329835 2151 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2152 return;
2153 }
2154 const char* typeStr = res->GetKey(0, "origLFN");
2155 if (!typeStr || !strlen(typeStr)) {
2156 msg += " using_tags: No (unknown)";
23329835 2157 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2158 return;
2159 }
2160 TString file = typeStr;
a2f5fc01 2161 useTags = file.Contains(".tag");
2162 if (useTags) msg += " using_tags: Yes";
c57f56b7 2163 else msg += " using_tags: No";
23329835 2164 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2165 return;
2166 }
2167 TString slfn(lfn);
2168 slfn.ToLower();
a2f5fc01 2169 isXml = slfn.Contains(".xml");
2170 if (isXml) {
c57f56b7 2171 // Open xml collection and check if there are tag files inside
2172 msg += " type: xml_collection;";
2173 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2174 if (!coll) {
2175 msg += " using_tags: No (unknown)";
23329835 2176 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2177 return;
2178 }
2179 TMap *map = coll->Next();
2180 if (!map) {
2181 msg += " using_tags: No (unknown)";
23329835 2182 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2183 return;
2184 }
2185 map = (TMap*)map->GetValue("");
2186 TString file;
2187 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2188 useTags = file.Contains(".tag");
c57f56b7 2189 delete coll;
a2f5fc01 2190 if (useTags) msg += " using_tags: Yes";
c57f56b7 2191 else msg += " using_tags: No";
23329835 2192 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2193 return;
2194 }
a2f5fc01 2195 useTags = slfn.Contains(".tag");
c57f56b7 2196 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2197 else msg += " type: unknown file;";
a2f5fc01 2198 if (useTags) msg += " using_tags: Yes";
c57f56b7 2199 else msg += " using_tags: No";
23329835 2200 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2201}
2202
2203//______________________________________________________________________________
4e5c5506 2204void AliAnalysisAlien::EnablePackage(const char *package)
2205{
2206// Enables a par file supposed to exist in the current directory.
2207 TString pkg(package);
2208 pkg.ReplaceAll(".par", "");
2209 pkg += ".par";
2210 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2211 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2212 return;
2213 }
2214 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2215 Info("EnablePackage", "AliEn plugin will use .par packages");
2216 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2217 if (!fPackages) {
2218 fPackages = new TObjArray();
2219 fPackages->SetOwner();
2220 }
2221 fPackages->Add(new TObjString(pkg));
2222}
2223
2224//______________________________________________________________________________
3bdcb562 2225TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2226{
2227// Make a tree from files having the location specified in fFileForTestMode.
2228// Inspired from JF's CreateESDChain.
2229 if (fFileForTestMode.IsNull()) {
2230 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2231 return NULL;
2232 }
2233 if (gSystem->AccessPathName(fFileForTestMode)) {
2234 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2235 return NULL;
2236 }
2237 // Open the file
2238 ifstream in;
2239 in.open(fFileForTestMode);
2240 Int_t count = 0;
2241 // Read the input list of files and add them to the chain
b505f1f8 2242 TString line;
2243 TChain *chain = new TChain(treeName);
2244 TChain *chainFriend = 0;
2245 if (!fFriendChainName.IsNull()) chainFriend = new TChain(treeName);
2246 while (in.good())
2247 {
3bdcb562 2248 in >> line;
2249 if (line.IsNull()) continue;
2250 if (count++ == fNtestFiles) break;
2251 TString esdFile(line);
2252 TFile *file = TFile::Open(esdFile);
b505f1f8 2253 if (file && !file->IsZombie()) {
2254 chain->Add(esdFile);
3bdcb562 2255 file->Close();
b505f1f8 2256 if (!fFriendChainName.IsNull()) {
4b5d9e21 2257 if (esdFile.Index("#") > -1)
2258 esdFile.Remove(esdFile.Index("#"));
2259 esdFile = gSystem->DirName(esdFile);
2260 esdFile += "/" + fFriendChainName;
b505f1f8 2261 file = TFile::Open(esdFile);
2262 if (file && !file->IsZombie()) {
2263 file->Close();
2264 chainFriend->Add(esdFile);
2265 } else {
2266 Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data());
2267 return 0;
2268 }
4b5d9e21 2269 }
3bdcb562 2270 } else {
2271 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2272 }
b505f1f8 2273 }
2274 in.close();
2275 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2276 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2277 delete chain;
b505f1f8 2278 delete chainFriend;
3bdcb562 2279 return NULL;
b505f1f8 2280 }
3bdcb562 2281// chain->ls();
b505f1f8 2282 if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend);
2283 return chain;
3bdcb562 2284}
2285
2286//______________________________________________________________________________
d2a409b2 2287const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2288{
2289// Get job status for all jobs with jobid>jobidstart.
2290 static char mstatus[20];
2291 mstatus[0] = '\0';
2292 nrunning = 0;
2293 nwaiting = 0;
2294 nerror = 0;
2295 ndone = 0;
2296 TGridJobStatusList *list = gGrid->Ps("");
2297 if (!list) return mstatus;
2298 Int_t nentries = list->GetSize();
2299 TGridJobStatus *status;
2300 Int_t pid;
2301 for (Int_t ijob=0; ijob<nentries; ijob++) {
2302 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2303 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2304 if (pid<jobidstart) continue;
2305 if (pid == lastid) {
8ddad121 2306 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2307 }
2308 switch (status->GetStatus()) {
2309 case TGridJobStatus::kWAITING:
2310 nwaiting++; break;
2311 case TGridJobStatus::kRUNNING:
2312 nrunning++; break;
2313 case TGridJobStatus::kABORTED:
2314 case TGridJobStatus::kFAIL:
2315 case TGridJobStatus::kUNKNOWN:
2316 nerror++; break;
2317 case TGridJobStatus::kDONE:
2318 ndone++;
2319 }
2320 }
2321 list->Delete();
2322 delete list;
2323 return mstatus;
2324}
2325
2326//______________________________________________________________________________
c57f56b7 2327Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2328{
2329// Returns true if file is a collection. Functionality duplicated from
2330// TAlien::Type() because we don't want to directly depend on TAlien.
2331 if (!gGrid) {
2332 Error("IsCollection", "No connection to grid");
2333 return kFALSE;
2334 }
2335 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2336 if (!res) return kFALSE;
2337 const char* typeStr = res->GetKey(0, "type");
2338 if (!typeStr || !strlen(typeStr)) return kFALSE;
2339 if (!strcmp(typeStr, "collection")) return kTRUE;
2340 delete res;
2341 return kFALSE;
2342}
2343
2344//______________________________________________________________________________
fe2d7fc2 2345Bool_t AliAnalysisAlien::IsSingleOutput() const
2346{
2347// Check if single-ouput option is on.
2348 return (!fOutputSingle.IsNull());
2349}
2350
2351//______________________________________________________________________________
16a4353c 2352void AliAnalysisAlien::Print(Option_t *) const
2353{
2354// Print current plugin settings.
84fcd93f 2355 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2356 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2357 if (mgr && mgr->IsProofMode()) {
2358 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2359 if (TestBit(AliAnalysisGrid::kTest))
2360 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2361 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2362 if (!fProofDataSet.IsNull())
2363 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2364 if (fProofReset==1)
2365 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2366 if (fProofReset>1)
2367 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2368 if (!fRootVersionForProof.IsNull())
2369 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2370 else
2371 printf("= ROOT version requested________________________ default\n");
2372 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2373 if (!fAliRootMode.IsNull())
2374 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2375 if (fNproofWorkers)
2376 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2377 if (fNproofWorkersPerSlave)
2378 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2379 if (TestSpecialBit(kClearPackages))
2380 printf("= ClearPackages requested...\n");
2381 if (fIncludePath.Data())
2382 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2383 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2384 if (fPackages && fPackages->GetEntries()) {
2385 TIter next(fPackages);
2386 TObject *obj;
2387 TString list;
2388 while ((obj=next())) list += obj->GetName();
2389 printf("= Par files to be used: ________________________ %s\n", list.Data());
2390 }
2391 if (TestSpecialBit(kProofConnectGrid))
2392 printf("= Requested PROOF connection to grid\n");
2393 return;
2394 }
e1c22e21 2395 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2396 if (fOverwriteMode) {
2397 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2398 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2399 }
348be253 2400 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2401 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2402 printf("= Production mode:______________________________ %d\n", fProductionMode);
2403 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2404 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2405 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2406 if (fUser.Length())
84fcd93f 2407 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2408 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2409 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2410 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2411 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2412 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2413 if (fRunNumbers.Length())
84fcd93f 2414 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2415 if (fRunRange[0])
74013535 2416 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2417 if (!fRunRange[0] && !fRunNumbers.Length()) {
2418 TIter next(fInputFiles);
2419 TObject *obj;
2420 TString list;
2421 while ((obj=next())) list += obj->GetName();
84fcd93f 2422 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2423 }
2424 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2425 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2426 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2427 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2428 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2429 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2430 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2431 printf("=====================================================================\n");
2432 printf("= Job price: ___________________________________ %d\n", fPrice);
2433 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2434 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2435 if (fMaxInitFailed>0)
84fcd93f 2436 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2437 if (fMasterResubmitThreshold>0)
84fcd93f 2438 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2439 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2440 if (fNrunsPerMaster>0)
84fcd93f 2441 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2442 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2443 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2444 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2445 if (fArguments.Length())
84fcd93f 2446 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2447 if (fExecutableArgs.Length())
2448 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2449 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2450 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2451 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2452 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2453 if (fDatasetName)
84fcd93f 2454 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2455 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2456 if (fIncludePath.Data())
84fcd93f 2457 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2458 if (fCloseSE.Length())
84fcd93f 2459 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2460 if (fFriendChainName.Length())
84fcd93f 2461 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2462 if (fPackages && fPackages->GetEntries()) {
16a4353c 2463 TIter next(fPackages);
2464 TObject *obj;
2465 TString list;
2466 while ((obj=next())) list += obj->GetName();
84fcd93f 2467 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2468 }
2469}
2470
2471//______________________________________________________________________________
c57f56b7 2472void AliAnalysisAlien::SetDefaults()
2473{
2474// Set default values for everything. What cannot be filled will be left empty.
2475 if (fGridJDL) delete fGridJDL;
2476 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2477 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2478 fPrice = 1;
2479 fTTL = 30000;
2480 fSplitMaxInputFileNumber = 100;
2481 fMaxInitFailed = 0;
2482 fMasterResubmitThreshold = 0;
bb885a9e 2483 fNtestFiles = 10;
149d288c 2484 fNreplicas = 2;
a8739e8a 2485 fRunRange[0] = 0;
2486 fRunRange[1] = 0;
8e3bd06a 2487 fRunPrefix = "%d";
319593fb 2488 fNrunsPerMaster = 1;
16a4353c 2489 fMaxMergeFiles = 100;
c57f56b7 2490 fRunNumbers = "";
2491 fExecutable = "analysis.sh";
0a1c1f7f 2492 fExecutableCommand = "root -b -q";
c57f56b7 2493 fArguments = "";
631c0b05 2494 fExecutableArgs = "";
c57f56b7 2495 fAnalysisMacro = "myAnalysis.C";
2496 fAnalysisSource = "";
2497 fAdditionalLibs = "";
2498 fSplitMode = "se";
2499 fAPIVersion = "";
2500 fROOTVersion = "";
2501 fAliROOTVersion = "";
2502 fUser = ""; // Your alien user name
2503 fGridWorkingDir = "";
2504 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2505 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2506 fFriendChainName = "";
c57f56b7 2507 fGridOutputDir = "output";
b83f84f0 2508 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2509 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2510 fInputFormat = "xml-single";
2511 fJDLName = "analysis.jdl";
c6cb3634 2512 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2513 fMergeExcludes = "";
0f389141 2514 fMergeViaJDL = 0;
348be253 2515 SetUseCopy(kTRUE);
2516 SetCheckCopy(kTRUE);
149d288c 2517 SetDefaultOutputs(kTRUE);
e1c22e21 2518 fOverwriteMode = 1;
c57f56b7 2519}
2520
2521//______________________________________________________________________________
b3e07543 2522Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2523{
b3e07543 2524// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2525 // First check if the result is already in the output directory.
2526 if (FileExists(Form("%s/%s",aliendir,filename))) {
2527 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2528 return kFALSE;
2529 }
b3e07543 2530 // Now check the last stage done.
2531 Int_t stage = 0;
2532 while (1) {
2533 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2534 stage++;
2535 }
2536 // Next stage of merging
2537 stage++;
2538 TString pattern = "*root_archive.zip";
2539 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2540 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2541 if (res) delete res;
2542 // Write standard output to file
2543 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2544 // Count the number of files inside
2545 ifstream ifile;
2546 ifile.open(Form("Stage_%d.xml",stage));
2547 if (!ifile.good()) {
2548 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2549 return kFALSE;
b3e07543 2550 }
2551 TString line;
2552 Int_t nfiles = 0;
2553 while (!ifile.eof()) {
2554 ifile >> line;
2555 if (line.Contains("/event")) nfiles++;
7c2cd90a 2556 }
b3e07543 2557 ifile.close();
2558 if (!nfiles) {
2559 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2560 return kFALSE;
2561 } else {
2562 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2563 }
b3e07543 2564 // Copy the file in the output directory
2565 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2566// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2567 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2568 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2569 // Check if this is the last stage to be done.
2570 Bool_t laststage = (nfiles<nperchunk);
2571 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2572 if (laststage) {
2573 printf("### Submiting final merging stage %d\n", stage);
2574 TString finalJDL = jdl;
2575 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2576 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2577 Int_t jobId = SubmitSingleJob(query);
2578 if (!jobId) return kFALSE;
2579 } else {
2580 printf("### Submiting merging stage %d\n", stage);
03335324 2581 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2582 Int_t jobId = SubmitSingleJob(query);
2583 if (!jobId) return kFALSE;
7c2cd90a 2584 }
b3e07543 2585 return kTRUE;
2586}
7c2cd90a 2587
2588//______________________________________________________________________________
105631b5 2589AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2590{
2591// Loat the analysis manager from a file.
2592 TFile *file = TFile::Open(fname);
2593 if (!file) {
2594 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2595 return 0;
2596 }
2597 TIter nextkey(file->GetListOfKeys());
2598 AliAnalysisManager *mgr = 0;
2599 TKey *key;
2600 while ((key=(TKey*)nextkey())) {
2601 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2602 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2603 }
2604 if (!mgr)
2605 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2606 return mgr;
2607}
2608
2609//______________________________________________________________________________
7c2cd90a 2610Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2611{
2612// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2613 if (!gGrid) return 0;
2614 printf("=> %s ------> ",query);
2615 TGridResult *res = gGrid->Command(query);
2616 if (!res) return 0;
2617 TString jobId = res->GetKey(0,"jobId");
2618 delete res;
2619 if (jobId.IsNull()) {
2620 printf("submission failed. Reason:\n");
2621 gGrid->Stdout();
2622 gGrid->Stderr();
2623 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2624 return 0;
2625 }
2626 printf(" Job id: %s\n", jobId.Data());
2627 return atoi(jobId);
2628}
2629
2630//______________________________________________________________________________
b3e07543 2631Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2632{
b3e07543 2633// Merge given output files from basedir. Basedir can be an alien output directory
2634// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2635// files in a group (ignored for xml input). Merging can be done in stages:
2636// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2637// stage=1 : works with an xml of all root_archive.zip in the output directory
2638// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2639 TString outputFile = output;
0f389141 2640 TString command;
a2f5fc01 2641 TString outputChunk;
2642 TString previousChunk = "";
b3e07543 2643 TObjArray *listoffiles = new TObjArray();
2644// listoffiles->SetOwner();
a2f5fc01 2645 Int_t countChunk = 0;
2646 Int_t countZero = nmaxmerge;
0f389141 2647 Bool_t merged = kTRUE;
a2f5fc01 2648 Int_t index = outputFile.Index("@");
2649 if (index > 0) outputFile.Remove(index);
7c2cd90a 2650 TString inputFile = outputFile;
b3e07543 2651 TString sbasedir = basedir;
2652 if (sbasedir.Contains(".xml")) {
2653 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2654 nmaxmerge = 9999999;
2655 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2656 if (!coll) {
2657 ::Error("MergeOutput", "Input XML collection empty.");
2658 return kFALSE;
2659 }
2660 // Iterate grid collection
2661 while (coll->Next()) {
2662 TString fname = gSystem->DirName(coll->GetTURL());
2663 fname += "/";
2664 fname += inputFile;
2665 listoffiles->Add(new TNamed(fname.Data(),""));
2666 }
2667 } else {
2668 command = Form("find %s/ *%s", basedir, inputFile.Data());
2669 printf("command: %s\n", command.Data());
2670 TGridResult *res = gGrid->Command(command);
2671 if (!res) {
2672 ::Error("MergeOutput","No result for the find command\n");
2673 delete listoffiles;
2674 return kFALSE;
2675 }
2676 TIter nextmap(res);
2677 TMap *map = 0;
2678 while ((map=(TMap*)nextmap())) {
2679 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2680 if (!objs || !objs->GetString().Length()) {
2681 // Nothing found - skip this output
2682 delete res;
2683 delete listoffiles;
2684 return kFALSE;
2685 }
2686 listoffiles->Add(new TNamed(objs->GetName(),""));
2687 }
2688 delete res;
2689 }
2690 if (!listoffiles->GetEntries()) {
7c2cd90a 2691 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2692 delete listoffiles;
0f389141 2693 return kFALSE;
2694 }
2695
2696 TFileMerger *fm = 0;
b3e07543 2697 TIter next0(listoffiles);
2698 TObjArray *listoffilestmp = new TObjArray();
2699 listoffilestmp->SetOwner();
2700 TObject *nextfile;
2701 TString snextfile;
2702 // Keep only the files at upper level
2703 Int_t countChar = 0;
2704 while ((nextfile=next0())) {
2705 snextfile = nextfile->GetName();
2706 Int_t crtCount = snextfile.CountChar('/');
2707 if (nextfile == listoffiles->First()) countChar = crtCount;
2708 if (crtCount < countChar) countChar = crtCount;
2709 }
2710 next0.Reset();
2711 while ((nextfile=next0())) {
2712 snextfile = nextfile->GetName();
2713 Int_t crtCount = snextfile.CountChar('/');
2714 if (crtCount > countChar) {
2715 delete nextfile;
2716 continue;
2717 }
2718 listoffilestmp->Add(nextfile);
2719 }
2720 delete listoffiles;
2721 listoffiles = listoffilestmp; // Now contains 'good' files
2722 listoffiles->Print();
2723 TIter next(listoffiles);
7c2cd90a 2724 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2725 outputChunk = outputFile;
2726 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2727 // Check for existent temporary merge files
e1c22e21 2728 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2729 // Preserve old merging functionality for stage 0.
2730 if (stage==0) {
2731 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2732 while (1) {
2733 // Skip as many input files as in a chunk
2ea55496 2734 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2735 nextfile = next();
2736 if (!nextfile) {
2ea55496 2737 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2738 delete listoffiles;
2ea55496 2739 return kFALSE;
2740 }
b3e07543 2741 snextfile = nextfile->GetName();
7c2cd90a 2742 }
2743 outputChunk = outputFile;
2744 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2745 countChunk++;
2746 if (gSystem->AccessPathName(outputChunk)) continue;
2747 // Merged file with chunks up to <countChunk> found
2748 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2749 previousChunk = outputChunk;
2750 break;
2751 }
2752 }
2753 countZero = nmaxmerge;
2754
b3e07543 2755 while ((nextfile=next())) {
2756 snextfile = nextfile->GetName();
2ea55496 2757 // Loop 'find' results and get next LFN
7c2cd90a 2758 if (countZero == nmaxmerge) {
2759 // First file in chunk - create file merger and add previous chunk if any.
2b895f34 2760 fm = new TFileMerger(kTRUE);
7c2cd90a 2761 fm->SetFastMethod(kTRUE);
2762 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2763 outputChunk = outputFile;
2764 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2765 }
2766 // If last file found, put merged results in the output file
b3e07543 2767 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2768 // Add file to be merged and decrement chunk counter.
b3e07543 2769 fm->AddFile(snextfile);
7c2cd90a 2770 countZero--;
b3e07543 2771 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2772 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2773 // Nothing found - skip this output
2774 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2775 merged = kFALSE;
2776 break;
7c2cd90a 2777 }
2778 fm->OutputFile(outputChunk);
2779 // Merge the outputs, then go to next chunk
2780 if (!fm->Merge()) {
2781 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2782 merged = kFALSE;
2783 break;
7c2cd90a 2784 } else {
2785 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2786 gSystem->Unlink(previousChunk);
2787 }
b3e07543 2788 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2789 countChunk++;
2790 countZero = nmaxmerge;
2791 previousChunk = outputChunk;
0f389141 2792 }
0f389141 2793 }
b3e07543 2794 delete listoffiles;
2795 delete fm;
7c2cd90a 2796 return merged;
2797 }
2798 // Merging stage different than 0.
2799 // Move to the begining of the requested chunk.
2b895f34 2800 fm = new TFileMerger(kTRUE);
7c2cd90a 2801 fm->SetFastMethod(kTRUE);
b3e07543 2802 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2803 delete listoffiles;
7c2cd90a 2804 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2805 // Nothing found - skip this output
2806 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2807 delete fm;
2808 return kFALSE;
2809 }
b3e07543 2810 fm->OutputFile(outputFile);
7c2cd90a 2811 // Merge the outputs
2812 if (!fm->Merge()) {
2813 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2814 delete fm;
2815 return kFALSE;
2816 } else {
b3e07543 2817 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2818 }
7c2cd90a 2819 delete fm;
2820 return kTRUE;
0f389141 2821}
2822
2823//______________________________________________________________________________
c57f56b7 2824Bool_t AliAnalysisAlien::MergeOutputs()
2825{
2826// Merge analysis outputs existing in the AliEn space.
2827 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2828 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2829 if (!Connect()) {
2830 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2831 return kFALSE;
0f389141 2832 }
c9e8f7fd 2833 if (fMergeViaJDL) {
2834 if (!TestBit(AliAnalysisGrid::kMerge)) {
2835 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2836 return kFALSE;
2837 }
2838 if (fProductionMode) {
2839 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2840 return kFALSE;
2841 }
0f389141 2842 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2843 if (!SubmitMerging()) return kFALSE;
0f389141 2844 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2845 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2846 return kFALSE;
c57f56b7 2847 }
2848 // Get the output path
dc030a24 2849 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2850 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2851 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2852 return kFALSE;
2853 }
2854 if (!fOutputFiles.Length()) {
2855 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2856 return kFALSE;
0f389141 2857 }
2858 // Check if fast read option was requested
e1c22e21 2859 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2860 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2861 if (fFastReadOption) {
d3339be3 2862 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2863 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2864 gEnv->SetValue("XNet.ConnectTimeout",50);
2865 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2866 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2867 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2868 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2869 }
e8b839ab 2870 // Make sure we change the temporary directory
2871 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2872 // Set temporary compilation directory to current one
2873 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2874 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2875 TIter next(list);
2876 TObjString *str;
a2f5fc01 2877 TString outputFile;
c57f56b7 2878 Bool_t merged = kTRUE;
2879 while((str=(TObjString*)next())) {
a2f5fc01 2880 outputFile = str->GetString();
2881 Int_t index = outputFile.Index("@");
2882 if (index > 0) outputFile.Remove(index);
2883 TString outputChunk = outputFile;
2884 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2885 // Skip already merged outputs
a2f5fc01 2886 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2887 if (fOverwriteMode) {
a2f5fc01 2888 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2889 gSystem->Unlink(outputFile);
2890 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2891 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2892 outputChunk.Data());
2893 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2894 }
2895 } else {
a2f5fc01 2896 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2897 continue;
2898 }
2899 } else {
a2f5fc01 2900 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2901 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2902 outputChunk.Data());
2903 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2904 }
2905 }
37d21c01 2906 if (fMergeExcludes.Contains(outputFile.Data()) ||
2907 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 2908 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2909 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2910 if (!merged) {
2911 Error("MergeOutputs", "Terminate() will NOT be executed");
2912 return kFALSE;
ff07ec61 2913 }
2914 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2915 if (fileOpened) fileOpened->Close();
c57f56b7 2916 }
0f389141 2917 return kTRUE;
c57f56b7 2918}
2919
2920//______________________________________________________________________________
bb885a9e 2921void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2922{
2923// Use the output files connected to output containers from the analysis manager
2924// rather than the files defined by SetOutputFiles
2925 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2926 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2927 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2928}
2929
2930//______________________________________________________________________________
149d288c 2931void AliAnalysisAlien::SetOutputFiles(const char *list)
2932{
2933// Manually set the output files list.
2934// Removes duplicates. Not allowed if default outputs are not disabled.
2935 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2936 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2937 return;
2938 }
2939 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2940 fOutputFiles = "";
2941 TString slist = list;
2942 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2943 TObjArray *arr = slist.Tokenize(" ");
2944 TObjString *os;
2945 TIter next(arr);
2946 TString sout;
2947 while ((os=(TObjString*)next())) {
2948 sout = os->GetString();
2949 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2950 if (fOutputFiles.Contains(sout)) continue;
2951 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2952 fOutputFiles += sout;
2953 }
2954 delete arr;
f790bc1b 2955}
149d288c 2956
2957//______________________________________________________________________________
2958void AliAnalysisAlien::SetOutputArchive(const char *list)
2959{
2960// Manually set the output archive list. Free text - you are on your own...
2961// Not allowed if default outputs are not disabled.
2962 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2963 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2964 return;
2965 }
2966 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2967 fOutputArchive = list;
2968}
2969
2970//______________________________________________________________________________
2971void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2972{
2973// Setting a prefered output SE is not allowed anymore.
2974 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2975}
2976
2977//______________________________________________________________________________
28ce6c9c 2978void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
2979{
2980// Set some PROOF special parameter.
2981 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2982 if (pair) {
2983 TObject *old = pair->Key();
2984 TObject *val = pair->Value();
2985 fProofParam.Remove(old);
2986 delete old;
2987 delete val;
2988 }
2989 fProofParam.Add(new TObjString(pname), new TObjString(value));
2990}
2991
2992//______________________________________________________________________________