Added separate flag to mark that the LocalInit phase was done + method AliAnalysisMan...
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
51ClassImp(AliAnalysisAlien)
6c395669 52#if 0
53;
54#endif
c57f56b7 55
6c395669 56namespace {
57 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
58 {
59 TString sl(Form("file:%s", loc));
60 TString sr(Form("alien://%s", rem));
61 Bool_t ret = TFile::Cp(sl, sr);
62 if (!ret) {
63 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
64 }
65 return ret;
66 }
67}
68
c57f56b7 69//______________________________________________________________________________
70AliAnalysisAlien::AliAnalysisAlien()
71 :AliAnalysisGrid(),
72 fGridJDL(NULL),
0f389141 73 fMergingJDL(NULL),
c57f56b7 74 fPrice(0),
75 fTTL(0),
76 fSplitMaxInputFileNumber(0),
77 fMaxInitFailed(0),
78 fMasterResubmitThreshold(0),
bb885a9e 79 fNtestFiles(0),
319593fb 80 fNrunsPerMaster(0),
16a4353c 81 fMaxMergeFiles(0),
b3e07543 82 fMaxMergeStages(0),
d2a409b2 83 fNsubmitted(0),
a3e84053 84 fProductionMode(0),
cd11251e 85 fOutputToRunNo(0),
0f389141 86 fMergeViaJDL(0),
87 fFastReadOption(0),
e1c22e21 88 fOverwriteMode(1),
149d288c 89 fNreplicas(2),
3bdcb562 90 fNproofWorkers(0),
27734f0e 91 fNproofWorkersPerSlave(0),
3bdcb562 92 fProofReset(0),
c57f56b7 93 fRunNumbers(),
94 fExecutable(),
0a1c1f7f 95 fExecutableCommand(),
c57f56b7 96 fArguments(),
631c0b05 97 fExecutableArgs(),
c57f56b7 98 fAnalysisMacro(),
99 fAnalysisSource(),
d3b18c4c 100 fValidationScript(),
d5c6455a 101 fAdditionalRootLibs(),
c57f56b7 102 fAdditionalLibs(),
103 fSplitMode(),
104 fAPIVersion(),
105 fROOTVersion(),
106 fAliROOTVersion(),
648174cf 107 fExternalPackages(),
c57f56b7 108 fUser(),
109 fGridWorkingDir(),
110 fGridDataDir(),
111 fDataPattern(),
112 fGridOutputDir(),
113 fOutputArchive(),
114 fOutputFiles(),
115 fInputFormat(),
e7c71df0 116 fDatasetName(),
c57f56b7 117 fJDLName(),
d3b18c4c 118 fTerminateFiles(),
bb885a9e 119 fMergeExcludes(),
37d21c01 120 fRegisterExcludes(),
f965131e 121 fIncludePath(),
bb885a9e 122 fCloseSE(),
0df6ccf2 123 fFriendChainName(),
c6cb3634 124 fJobTag(),
648174cf 125 fOutputSingle(),
5fce53f4 126 fRunPrefix(),
3bdcb562 127 fProofCluster(),
128 fProofDataSet(),
129 fFileForTestMode(),
130 fRootVersionForProof(),
131 fAliRootMode(),
f47d5cb4 132 fMergeDirName(),
4e5c5506 133 fInputFiles(0),
28ce6c9c 134 fPackages(0),
77f34eae 135 fModules(0),
28ce6c9c 136 fProofParam()
c57f56b7 137{
138// Dummy ctor.
139 SetDefaults();
140}
141
142//______________________________________________________________________________
143AliAnalysisAlien::AliAnalysisAlien(const char *name)
144 :AliAnalysisGrid(name),
145 fGridJDL(NULL),
0f389141 146 fMergingJDL(NULL),
c57f56b7 147 fPrice(0),
148 fTTL(0),
149 fSplitMaxInputFileNumber(0),
150 fMaxInitFailed(0),
151 fMasterResubmitThreshold(0),
bb885a9e 152 fNtestFiles(0),
319593fb 153 fNrunsPerMaster(0),
16a4353c 154 fMaxMergeFiles(0),
b3e07543 155 fMaxMergeStages(0),
d2a409b2 156 fNsubmitted(0),
a3e84053 157 fProductionMode(0),
cd11251e 158 fOutputToRunNo(0),
0f389141 159 fMergeViaJDL(0),
160 fFastReadOption(0),
e1c22e21 161 fOverwriteMode(1),
149d288c 162 fNreplicas(2),
3bdcb562 163 fNproofWorkers(0),
27734f0e 164 fNproofWorkersPerSlave(0),
3bdcb562 165 fProofReset(0),
c57f56b7 166 fRunNumbers(),
167 fExecutable(),
0a1c1f7f 168 fExecutableCommand(),
c57f56b7 169 fArguments(),
631c0b05 170 fExecutableArgs(),
c57f56b7 171 fAnalysisMacro(),
172 fAnalysisSource(),
d3b18c4c 173 fValidationScript(),
d5c6455a 174 fAdditionalRootLibs(),
c57f56b7 175 fAdditionalLibs(),
176 fSplitMode(),
177 fAPIVersion(),
178 fROOTVersion(),
179 fAliROOTVersion(),
648174cf 180 fExternalPackages(),
c57f56b7 181 fUser(),
182 fGridWorkingDir(),
183 fGridDataDir(),
184 fDataPattern(),
185 fGridOutputDir(),
186 fOutputArchive(),
187 fOutputFiles(),
188 fInputFormat(),
e7c71df0 189 fDatasetName(),
c57f56b7 190 fJDLName(),
d3b18c4c 191 fTerminateFiles(),
bb885a9e 192 fMergeExcludes(),
37d21c01 193 fRegisterExcludes(),
f965131e 194 fIncludePath(),
bb885a9e 195 fCloseSE(),
0df6ccf2 196 fFriendChainName(),
c6cb3634 197 fJobTag(),
648174cf 198 fOutputSingle(),
5fce53f4 199 fRunPrefix(),
3bdcb562 200 fProofCluster(),
201 fProofDataSet(),
202 fFileForTestMode(),
203 fRootVersionForProof(),
204 fAliRootMode(),
f47d5cb4 205 fMergeDirName(),
4e5c5506 206 fInputFiles(0),
28ce6c9c 207 fPackages(0),
77f34eae 208 fModules(0),
28ce6c9c 209 fProofParam()
c57f56b7 210{
211// Default ctor.
212 SetDefaults();
213}
214
215//______________________________________________________________________________
216AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
217 :AliAnalysisGrid(other),
218 fGridJDL(NULL),
0f389141 219 fMergingJDL(NULL),
c57f56b7 220 fPrice(other.fPrice),
221 fTTL(other.fTTL),
222 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
223 fMaxInitFailed(other.fMaxInitFailed),
224 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 225 fNtestFiles(other.fNtestFiles),
319593fb 226 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 227 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 228 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 229 fNsubmitted(other.fNsubmitted),
a3e84053 230 fProductionMode(other.fProductionMode),
cd11251e 231 fOutputToRunNo(other.fOutputToRunNo),
0f389141 232 fMergeViaJDL(other.fMergeViaJDL),
233 fFastReadOption(other.fFastReadOption),
234 fOverwriteMode(other.fOverwriteMode),
149d288c 235 fNreplicas(other.fNreplicas),
3bdcb562 236 fNproofWorkers(other.fNproofWorkers),
27734f0e 237 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 238 fProofReset(other.fProofReset),
c57f56b7 239 fRunNumbers(other.fRunNumbers),
240 fExecutable(other.fExecutable),
0a1c1f7f 241 fExecutableCommand(other.fExecutableCommand),
c57f56b7 242 fArguments(other.fArguments),
631c0b05 243 fExecutableArgs(other.fExecutableArgs),
c57f56b7 244 fAnalysisMacro(other.fAnalysisMacro),
245 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 246 fValidationScript(other.fValidationScript),
d5c6455a 247 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 248 fAdditionalLibs(other.fAdditionalLibs),
249 fSplitMode(other.fSplitMode),
250 fAPIVersion(other.fAPIVersion),
251 fROOTVersion(other.fROOTVersion),
252 fAliROOTVersion(other.fAliROOTVersion),
648174cf 253 fExternalPackages(other.fExternalPackages),
c57f56b7 254 fUser(other.fUser),
255 fGridWorkingDir(other.fGridWorkingDir),
256 fGridDataDir(other.fGridDataDir),
257 fDataPattern(other.fDataPattern),
258 fGridOutputDir(other.fGridOutputDir),
259 fOutputArchive(other.fOutputArchive),
260 fOutputFiles(other.fOutputFiles),
261 fInputFormat(other.fInputFormat),
e7c71df0 262 fDatasetName(other.fDatasetName),
c57f56b7 263 fJDLName(other.fJDLName),
d3b18c4c 264 fTerminateFiles(other.fTerminateFiles),
bb885a9e 265 fMergeExcludes(other.fMergeExcludes),
37d21c01 266 fRegisterExcludes(other.fRegisterExcludes),
f965131e 267 fIncludePath(other.fIncludePath),
bb885a9e 268 fCloseSE(other.fCloseSE),
0df6ccf2 269 fFriendChainName(other.fFriendChainName),
c6cb3634 270 fJobTag(other.fJobTag),
648174cf 271 fOutputSingle(other.fOutputSingle),
5fce53f4 272 fRunPrefix(other.fRunPrefix),
3bdcb562 273 fProofCluster(other.fProofCluster),
274 fProofDataSet(other.fProofDataSet),
275 fFileForTestMode(other.fFileForTestMode),
276 fRootVersionForProof(other.fRootVersionForProof),
277 fAliRootMode(other.fAliRootMode),
f47d5cb4 278 fMergeDirName(other.fMergeDirName),
4e5c5506 279 fInputFiles(0),
28ce6c9c 280 fPackages(0),
77f34eae 281 fModules(0),
28ce6c9c 282 fProofParam()
c57f56b7 283{
284// Copy ctor.
285 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 286 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 287 fRunRange[0] = other.fRunRange[0];
288 fRunRange[1] = other.fRunRange[1];
c57f56b7 289 if (other.fInputFiles) {
290 fInputFiles = new TObjArray();
291 TIter next(other.fInputFiles);
292 TObject *obj;
293 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
294 fInputFiles->SetOwner();
295 }
4e5c5506 296 if (other.fPackages) {
297 fPackages = new TObjArray();
298 TIter next(other.fPackages);
299 TObject *obj;
300 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
301 fPackages->SetOwner();
302 }
77f34eae 303 if (other.fModules) {
304 fModules = new TObjArray();
305 fModules->SetOwner();
306 TIter next(other.fModules);
307 AliAnalysisTaskCfg *mod, *crt;
308 while ((crt=(AliAnalysisTaskCfg*)next())) {
309 mod = new AliAnalysisTaskCfg(*crt);
310 fModules->Add(mod);
311 }
312 }
c57f56b7 313}
314
315//______________________________________________________________________________
316AliAnalysisAlien::~AliAnalysisAlien()
317{
318// Destructor.
77f34eae 319 delete fGridJDL;
320 delete fMergingJDL;
321 delete fInputFiles;
322 delete fPackages;
323 delete fModules;
28ce6c9c 324 fProofParam.DeleteAll();
c57f56b7 325}
326
327//______________________________________________________________________________
328AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
329{
330// Assignment.
331 if (this != &other) {
332 AliAnalysisGrid::operator=(other);
333 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 334 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 335 fPrice = other.fPrice;
336 fTTL = other.fTTL;
337 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
338 fMaxInitFailed = other.fMaxInitFailed;
339 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 340 fNtestFiles = other.fNtestFiles;
a3e84053 341 fNrunsPerMaster = other.fNrunsPerMaster;
342 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 343 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 344 fNsubmitted = other.fNsubmitted;
345 fProductionMode = other.fProductionMode;
cd11251e 346 fOutputToRunNo = other.fOutputToRunNo;
0f389141 347 fMergeViaJDL = other.fMergeViaJDL;
348 fFastReadOption = other.fFastReadOption;
349 fOverwriteMode = other.fOverwriteMode;
149d288c 350 fNreplicas = other.fNreplicas;
3bdcb562 351 fNproofWorkers = other.fNproofWorkers;
27734f0e 352 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 353 fProofReset = other.fProofReset;
c57f56b7 354 fRunNumbers = other.fRunNumbers;
355 fExecutable = other.fExecutable;
0a1c1f7f 356 fExecutableCommand = other.fExecutableCommand;
c57f56b7 357 fArguments = other.fArguments;
631c0b05 358 fExecutableArgs = other.fExecutableArgs;
c57f56b7 359 fAnalysisMacro = other.fAnalysisMacro;
360 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 361 fValidationScript = other.fValidationScript;
d5c6455a 362 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 363 fAdditionalLibs = other.fAdditionalLibs;
364 fSplitMode = other.fSplitMode;
365 fAPIVersion = other.fAPIVersion;
366 fROOTVersion = other.fROOTVersion;
367 fAliROOTVersion = other.fAliROOTVersion;
648174cf 368 fExternalPackages = other.fExternalPackages;
c57f56b7 369 fUser = other.fUser;
370 fGridWorkingDir = other.fGridWorkingDir;
371 fGridDataDir = other.fGridDataDir;
372 fDataPattern = other.fDataPattern;
373 fGridOutputDir = other.fGridOutputDir;
374 fOutputArchive = other.fOutputArchive;
375 fOutputFiles = other.fOutputFiles;
376 fInputFormat = other.fInputFormat;
e7c71df0 377 fDatasetName = other.fDatasetName;
c57f56b7 378 fJDLName = other.fJDLName;
d3b18c4c 379 fTerminateFiles = other.fTerminateFiles;
bb885a9e 380 fMergeExcludes = other.fMergeExcludes;
37d21c01 381 fRegisterExcludes = other.fRegisterExcludes;
f965131e 382 fIncludePath = other.fIncludePath;
bb885a9e 383 fCloseSE = other.fCloseSE;
0df6ccf2 384 fFriendChainName = other.fFriendChainName;
c6cb3634 385 fJobTag = other.fJobTag;
648174cf 386 fOutputSingle = other.fOutputSingle;
5fce53f4 387 fRunPrefix = other.fRunPrefix;
3bdcb562 388 fProofCluster = other.fProofCluster;
389 fProofDataSet = other.fProofDataSet;
390 fFileForTestMode = other.fFileForTestMode;
391 fRootVersionForProof = other.fRootVersionForProof;
392 fAliRootMode = other.fAliRootMode;
f47d5cb4 393 fMergeDirName = other.fMergeDirName;
c57f56b7 394 if (other.fInputFiles) {
395 fInputFiles = new TObjArray();
396 TIter next(other.fInputFiles);
397 TObject *obj;
398 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
399 fInputFiles->SetOwner();
400 }
4e5c5506 401 if (other.fPackages) {
402 fPackages = new TObjArray();
403 TIter next(other.fPackages);
404 TObject *obj;
405 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
406 fPackages->SetOwner();
407 }
77f34eae 408 if (other.fModules) {
409 fModules = new TObjArray();
410 fModules->SetOwner();
411 TIter next(other.fModules);
412 AliAnalysisTaskCfg *mod, *crt;
413 while ((crt=(AliAnalysisTaskCfg*)next())) {
414 mod = new AliAnalysisTaskCfg(*crt);
415 fModules->Add(mod);
416 }
417 }
c57f56b7 418 }
419 return *this;
420}
421
422//______________________________________________________________________________
37d21c01 423void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
424{
425// Add a single additional library to be loaded. Extension must be present.
426 TString lib(name);
427 if (!lib.Contains(".")) {
428 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
429 return;
430 }
431 if (fAdditionalLibs.Contains(name)) {
432 Warning("AddAdditionalLibrary", "Library %s already added.", name);
433 return;
434 }
435 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
436 fAdditionalLibs += lib;
437}
438
439//______________________________________________________________________________
77f34eae 440void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
441{
442// Adding a module. Checks if already existing. Becomes owned by this.
443 if (!module) return;
444 if (GetModule(module->GetName())) {
445 Error("AddModule", "A module having the same name %s already added", module->GetName());
446 return;
447 }
448 if (!fModules) {
449 fModules = new TObjArray();
450 fModules->SetOwner();
451 }
452 fModules->Add(module);
453}
454
455//______________________________________________________________________________
456void AliAnalysisAlien::AddModules(TObjArray *list)
457{
458// Adding a list of modules. Checks if already existing. Becomes owned by this.
459 TIter next(list);
460 AliAnalysisTaskCfg *module;
461 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
462}
463
464//______________________________________________________________________________
465Bool_t AliAnalysisAlien::CheckDependencies()
466{
467// Check if all dependencies are satisfied. Reorder modules if needed.
468 Int_t nmodules = GetNmodules();
469 if (!nmodules) {
470 Warning("CheckDependencies", "No modules added yet to check their dependencies");
471 return kTRUE;
472 }
473 AliAnalysisTaskCfg *mod = 0;
474 AliAnalysisTaskCfg *dep = 0;
475 TString depname;
476 Int_t i, j, k;
477 for (i=0; i<nmodules; i++) {
478 mod = (AliAnalysisTaskCfg*) fModules->At(i);
479 Int_t ndeps = mod->GetNdeps();
480 Int_t istart = i;
481 for (j=0; j<ndeps; j++) {
482 depname = mod->GetDependency(j);
483 dep = GetModule(depname);
484 if (!dep) {
485 Error("CheckDependencies","Dependency %s not added for module %s",
486 depname.Data(), mod->GetName());
487 return kFALSE;
488 }
489 if (dep->NeedsDependency(mod->GetName())) {
490 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
491 mod->GetName(), dep->GetName());
492 return kFALSE;
493 }
494 Int_t idep = fModules->IndexOf(dep);
495 // The dependency task must come first
496 if (idep>i) {
497 // Remove at idep and move all objects below up one slot
498 // down to index i included.
499 fModules->RemoveAt(idep);
3e40fd4c 500 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 501 fModules->AddAt(dep, i++);
502 }
503 //Redo from istart if dependencies were inserted
504 if (i>istart) i=istart-1;
505 }
506 }
507 return kTRUE;
508}
509
510//______________________________________________________________________________
3e40fd4c 511AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
512{
513// Create the analysis manager and optionally execute the macro in filename.
514 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
515 if (mgr) return mgr;
516 mgr = new AliAnalysisManager(name);
517 mgr->SetGridHandler((AliAnalysisGrid*)this);
518 if (strlen(filename)) {
519 TString line = gSystem->ExpandPathName(filename);
520 line.Prepend(".x ");
521 gROOT->ProcessLine(line.Data());
522 }
523 return mgr;
524}
525
526//______________________________________________________________________________
77f34eae 527Int_t AliAnalysisAlien::GetNmodules() const
528{
529// Get number of modules.
530 if (!fModules) return 0;
531 return fModules->GetEntries();
532}
533
534//______________________________________________________________________________
535AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
536{
537// Get a module by name.
538 if (!fModules) return 0;
539 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
540}
541
542//______________________________________________________________________________
543Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
544{
545// Load a given module.
546 if (mod->IsLoaded()) return kTRUE;
547 Int_t ndeps = mod->GetNdeps();
548 TString depname;
549 for (Int_t j=0; j<ndeps; j++) {
550 depname = mod->GetDependency(j);
551 AliAnalysisTaskCfg *dep = GetModule(depname);
552 if (!dep) {
553 Error("LoadModule","Dependency %s not existing for module %s",
554 depname.Data(), mod->GetName());
555 return kFALSE;
556 }
557 if (!LoadModule(dep)) {
558 Error("LoadModule","Dependency %s for module %s could not be loaded",
559 depname.Data(), mod->GetName());
560 return kFALSE;
561 }
562 }
563 // Load libraries for the module
564 if (!mod->CheckLoadLibraries()) {
565 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
566 return kFALSE;
567 }
568 // Execute the macro
569 if (mod->ExecuteMacro()<0) {
570 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
571 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
572 return kFALSE;
573 }
574 // Configure dependencies
575 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
576 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
577 mod->GetConfigMacro()->GetTitle(), mod->GetName());
578 return kFALSE;
579 }
5e8c7c22 580 // Adjust extra libraries
581 Int_t nlibs = mod->GetNlibs();
582 TString lib;
583 for (Int_t i=0; i<nlibs; i++) {
584 lib = mod->GetLibrary(i);
585 if (fAdditionalLibs.Contains(lib)) continue;
586 lib = Form("lib%s.so", lib.Data());
587 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
588 fAdditionalLibs += lib;
589 }
77f34eae 590 return kTRUE;
591}
592
593//______________________________________________________________________________
139fe6ba 594Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
595{
596// Generate the full train.
597 fAdditionalLibs = "";
598 if (!LoadModules()) return kFALSE;
599 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
600 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 601 mgr->RunLocalInit();
139fe6ba 602 mgr->PrintStatus();
603 Int_t productionMode = fProductionMode;
604 SetProductionMode();
605 TString macro = fAnalysisMacro;
606 TString executable = fExecutable;
607 TString validation = fValidationScript;
608 TString execCommand = fExecutableCommand;
609 SetAnalysisMacro(Form("%s.C", name));
610 SetExecutable(Form("%s.sh", name));
611 SetExecutableCommand("aliroot -b -q ");
612 SetValidationScript(Form("%s_validation.sh", name));
613 StartAnalysis();
614 SetProductionMode(productionMode);
615 fAnalysisMacro = macro;
616 fExecutable = executable;
617 fExecutableCommand = execCommand;
618 fValidationScript = validation;
619 return kTRUE;
620}
621
622//______________________________________________________________________________
21ca8e59 623Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 624{
625// Generate test macros for a single module or for the full train.
5e8c7c22 626 fAdditionalLibs = "";
77f34eae 627 if (strlen(modname)) {
21ca8e59 628 if (!CheckDependencies()) return kFALSE;
629 AliAnalysisTaskCfg *mod = GetModule(modname);
630 if (!mod) {
631 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
632 return kFALSE;
633 }
634 if (!LoadModule(mod)) return kFALSE;
635 } else if (!LoadModules()) return kFALSE;
636 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
637 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 638 mgr->RunLocalInit();
21ca8e59 639 mgr->PrintStatus();
640 SetLocalTest(kTRUE);
641 Int_t productionMode = fProductionMode;
642 SetProductionMode();
643 TString macro = fAnalysisMacro;
644 TString executable = fExecutable;
645 TString validation = fValidationScript;
646 TString execCommand = fExecutableCommand;
647 SetAnalysisMacro(Form("%s.C", name));
648 SetExecutable(Form("%s.sh", name));
649 SetExecutableCommand("aliroot -b -q ");
650 SetValidationScript(Form("%s_validation.sh", name));
651 WriteAnalysisFile();
652 WriteAnalysisMacro();
653 WriteExecutable();
654 WriteValidationScript();
655 SetLocalTest(kFALSE);
656 SetProductionMode(productionMode);
657 fAnalysisMacro = macro;
658 fExecutable = executable;
659 fExecutableCommand = execCommand;
660 fValidationScript = validation;
77f34eae 661 return kTRUE;
662}
663
664//______________________________________________________________________________
665Bool_t AliAnalysisAlien::LoadModules()
666{
667// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 668 fAdditionalLibs = "";
21ca8e59 669 Int_t nmodules = GetNmodules();
670 if (!nmodules) {
671 Warning("LoadModules", "No module to be loaded");
672 return kTRUE;
673 }
3e40fd4c 674 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
675 if (!mgr) {
676 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
677 return kFALSE;
678 }
77f34eae 679 if (!CheckDependencies()) return kFALSE;
21ca8e59 680 nmodules = GetNmodules();
77f34eae 681 AliAnalysisTaskCfg *mod;
21ca8e59 682 for (Int_t imod=0; imod<nmodules; imod++) {
683 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 684 if (!LoadModule(mod)) return kFALSE;
685 }
77f34eae 686 return kTRUE;
687}
688
689//______________________________________________________________________________
74013535 690void AliAnalysisAlien::SetRunPrefix(const char *prefix)
691{
692// Set the run number format. Can be a prefix or a format like "%09d"
693 fRunPrefix = prefix;
694 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
695}
696
697//______________________________________________________________________________
f965131e 698void AliAnalysisAlien::AddIncludePath(const char *path)
699{
700// Add include path in the remote analysis macro.
701 TString p(path);
702 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
703 else fIncludePath += Form("-I%s ", path);
704}
705
706//______________________________________________________________________________
c57f56b7 707void AliAnalysisAlien::AddRunNumber(Int_t run)
708{
709// Add a run number to the list of runs to be processed.
710 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 711 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 712}
713
714//______________________________________________________________________________
98690895 715void AliAnalysisAlien::AddRunList(const char* runList)
716{
717// Add several runs into the list of runs; they are expected to be separated by a blank character.
718 TString sList = runList;
719 TObjArray *list = sList.Tokenize(" ");
720 Int_t n = list->GetEntries();
721 for (Int_t i = 0; i < n; i++) {
722 TObjString *os = (TObjString*)list->At(i);
723 AddRunNumber(os->GetString().Atoi());
724 }
725 delete list;
726}
727
728//______________________________________________________________________________
ee75cfc3 729void AliAnalysisAlien::AddRunNumber(const char* run)
730{
731// Add a run number to the list of runs to be processed.
7aa5d539 732 TString runs = run;
733 TObjString *os;
734 TObjArray *arr = runs.Tokenize(" ");
735 TIter next(arr);
736 TString prefix;
737 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
738 while ((os=(TObjString*)next())){
739 if (fRunNumbers.Length()) fRunNumbers += " ";
740 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
741 }
742 delete arr;
ee75cfc3 743}
744
745//______________________________________________________________________________
c57f56b7 746void AliAnalysisAlien::AddDataFile(const char *lfn)
747{
748// Adds a data file to the input to be analysed. The file should be a valid LFN
749// or point to an existing file in the alien workdir.
750 if (!fInputFiles) fInputFiles = new TObjArray();
751 fInputFiles->Add(new TObjString(lfn));
752}
648174cf 753
754//______________________________________________________________________________
755void AliAnalysisAlien::AddExternalPackage(const char *package)
756{
757// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
758 if (fExternalPackages) fExternalPackages += " ";
759 fExternalPackages += package;
760}
761
c57f56b7 762//______________________________________________________________________________
763Bool_t AliAnalysisAlien::Connect()
764{
765// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
766 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 767 if (fProductionMode) return kTRUE;
c57f56b7 768 if (!gGrid) {
769 Info("Connect", "Trying to connect to AliEn ...");
770 TGrid::Connect("alien://");
771 }
772 if (!gGrid || !gGrid->IsConnected()) {
773 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
774 return kFALSE;
775 }
776 fUser = gGrid->GetUser();
777 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
778 return kTRUE;
779}
780
781//______________________________________________________________________________
782void AliAnalysisAlien::CdWork()
783{
784// Check validity of alien workspace. Create directory if possible.
785 if (!Connect()) {
786 Error("CdWork", "Alien connection required");
787 return;
788 }
789 TString homedir = gGrid->GetHomeDirectory();
790 TString workdir = homedir + fGridWorkingDir;
923e2ca5 791 if (DirectoryExists(workdir)) {
792 gGrid->Cd(workdir);
793 return;
794 }
795 // Work directory not existing - create it
796 gGrid->Cd(homedir);
b93f8109 797 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 798 gGrid->Cd(fGridWorkingDir);
d3b18c4c 799 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 800 } else {
d3b18c4c 801 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 802 workdir.Data(), homedir.Data());
803 fGridWorkingDir = "";
804 }
c57f56b7 805}
806
807//______________________________________________________________________________
348be253 808Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
809{
810// Check if file copying is possible.
d3b18c4c 811 if (fProductionMode) return kTRUE;
348be253 812 if (!Connect()) {
813 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
814 return kFALSE;
815 }
d3339be3 816 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
817 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 818 // Check if alien_CLOSE_SE is defined
819 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
820 if (!closeSE.IsNull()) {
821 Info("CheckFileCopy", "Your current close storage is pointing to: \
822 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
823 } else {
824 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
825 }
826 // Check if grid directory exists.
827 if (!DirectoryExists(alienpath)) {
828 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
829 return kFALSE;
830 }
d3339be3 831 TFile f("plugin_test_copy", "RECREATE");
348be253 832 // User may not have write permissions to current directory
833 if (f.IsZombie()) {
834 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
835 gSystem->WorkingDirectory());
836 return kFALSE;
837 }
838 f.Close();
d3339be3 839 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
840 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
841 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 842 \n# 1. Make sure you have write permissions there. If this is the case: \
843 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
844 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
845 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
846 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
847 gSystem->Unlink(f.GetName());
848 return kFALSE;
849 }
850 gSystem->Unlink(f.GetName());
d3339be3 851 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
852 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 853 return kTRUE;
854}
855
856//______________________________________________________________________________
c57f56b7 857Bool_t AliAnalysisAlien::CheckInputData()
858{
859// Check validity of input data. If necessary, create xml files.
d3b18c4c 860 if (fProductionMode) return kTRUE;
a8739e8a 861 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 862 if (!fGridDataDir.Length()) {
863 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
864 return kFALSE;
865 }
9f020952 866 if (fMergeViaJDL) {
867 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
868 return kFALSE;
869 }
d2a409b2 870 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 871 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
872 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 873 return kTRUE;
c57f56b7 874 }
875 // Process declared files
a2f5fc01 876 Bool_t isCollection = kFALSE;
877 Bool_t isXml = kFALSE;
878 Bool_t useTags = kFALSE;
c57f56b7 879 Bool_t checked = kFALSE;
d3b18c4c 880 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 881 TString file;
882 TString workdir = gGrid->GetHomeDirectory();
883 workdir += fGridWorkingDir;
884 if (fInputFiles) {
885 TObjString *objstr;
886 TIter next(fInputFiles);
887 while ((objstr=(TObjString*)next())) {
888 file = workdir;
889 file += "/";
890 file += objstr->GetString();
891 // Store full lfn path
892 if (FileExists(file)) objstr->SetString(file);
893 else {
894 file = objstr->GetName();
895 if (!FileExists(objstr->GetName())) {
896 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
897 objstr->GetName(), workdir.Data());
898 return kFALSE;
899 }
900 }
901 Bool_t iscoll, isxml, usetags;
902 CheckDataType(file, iscoll, isxml, usetags);
903 if (!checked) {
904 checked = kTRUE;
a2f5fc01 905 isCollection = iscoll;
906 isXml = isxml;
907 useTags = usetags;
908 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 909 } else {
a2f5fc01 910 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 911 Error("CheckInputData", "Some conflict was found in the types of inputs");
912 return kFALSE;
913 }
914 }
915 }
916 }
917 // Process requested run numbers
a8739e8a 918 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 919 // Check validity of alien data directory
920 if (!fGridDataDir.Length()) {
921 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
922 return kFALSE;
923 }
923e2ca5 924 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 925 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
926 return kFALSE;
927 }
a2f5fc01 928 if (isCollection) {
c57f56b7 929 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
930 return kFALSE;
931 }
932
a2f5fc01 933 if (checked && !isXml) {
c57f56b7 934 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
935 return kFALSE;
936 }
937 // Check validity of run number(s)
938 TObjArray *arr;
939 TObjString *os;
74013535 940 TString format;
319593fb 941 Int_t nruns = 0;
904f9f5f 942 TString schunk, schunk2;
c57f56b7 943 TString path;
944 if (!checked) {
945 checked = kTRUE;
a2f5fc01 946 useTags = fDataPattern.Contains("tag");
947 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 948 }
a2f5fc01 949 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 950 Error("CheckInputData", "Cannot mix input files using/not using tags");
951 return kFALSE;
952 }
953 if (fRunNumbers.Length()) {
a8739e8a 954 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 955 arr = fRunNumbers.Tokenize(" ");
956 TIter next(arr);
957 while ((os=(TObjString*)next())) {
958 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 959 if (!DirectoryExists(path)) {
960 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 961 continue;
c57f56b7 962 }
963 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
964 TString msg = "\n##### file: ";
965 msg += path;
966 msg += " type: xml_collection;";
a2f5fc01 967 if (useTags) msg += " using_tags: Yes";
c57f56b7 968 else msg += " using_tags: No";
23329835 969 Info("CheckDataType", "%s", msg.Data());
319593fb 970 if (fNrunsPerMaster<2) {
d2a409b2 971 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 972 } else {
973 nruns++;
974 if (((nruns-1)%fNrunsPerMaster) == 0) {
975 schunk = os->GetString();
976 }
977 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
978 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 979 AddDataFile(schunk);
319593fb 980 }
c57f56b7 981 }
982 delete arr;
a8739e8a 983 } else {
984 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
985 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 986 format = Form("%%s/%s ", fRunPrefix.Data());
987 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 988 if (!DirectoryExists(path)) {
a8739e8a 989 continue;
990 }
74013535 991 format = Form("%%s/%s.xml", fRunPrefix.Data());
992 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 993 TString msg = "\n##### file: ";
994 msg += path;
995 msg += " type: xml_collection;";
a2f5fc01 996 if (useTags) msg += " using_tags: Yes";
a8739e8a 997 else msg += " using_tags: No";
23329835 998 Info("CheckDataType", "%s", msg.Data());
319593fb 999 if (fNrunsPerMaster<2) {
74013535 1000 format = Form("%s.xml", fRunPrefix.Data());
1001 AddDataFile(Form(format.Data(),irun));
319593fb 1002 } else {
1003 nruns++;
1004 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1005 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1006 }
74013535 1007 format = Form("_%s.xml", fRunPrefix.Data());
1008 schunk2 = Form(format.Data(), irun);
319593fb 1009 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1010 schunk += schunk2;
d2a409b2 1011 AddDataFile(schunk);
319593fb 1012 }
a8739e8a 1013 }
904f9f5f 1014 if (!fInputFiles) {
1015 schunk += schunk2;
1016 AddDataFile(schunk);
1017 }
c57f56b7 1018 }
1019 return kTRUE;
1020}
1021
1022//______________________________________________________________________________
43b807f6 1023Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1024{
1025// Copy data from the given grid directory according a pattern and make a local
1026// dataset.
1027 if (!Connect()) {
1028 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
1029 return kFALSE;
1030 }
1031 if (!DirectoryExists(griddir)) {
1032 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
1033 return kFALSE;
1034 }
1035 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1036 printf("Running command: %s\n", command.Data());
1037 TGridResult *res = gGrid->Command(command);
1038 Int_t nfound = res->GetEntries();
1039 if (!nfound) {
1040 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
1041 return kFALSE;
1042 }
1043 printf("... found %d files. Copying locally ...\n", nfound);
1044 // Copy files locally
1045 ofstream out;
1046 out.open(output, ios::out);
1047 TMap *map;
1048 TString turl, dirname, filename, temp;
1049 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1050 gSystem->MakeDirectory(outputdir);
1051 gSystem->ChangeDirectory(outputdir);
21ca8e59 1052 for (Int_t i=0; i<nfound; i++) {
1053 map = (TMap*)res->At(i);
1054 turl = map->GetValue("turl")->GetName();
1055 filename = gSystem->BaseName(turl.Data());
1056 dirname = gSystem->DirName(turl.Data());
1057 dirname = gSystem->BaseName(dirname.Data());
1058 gSystem->MakeDirectory(dirname);
43b807f6 1059
1060 TString source(turl);
1061 TString targetFileName(filename);
1062
1063 if (strlen(archivefile) > 0) {
1064 // TODO here the archive in which the file resides should be determined
1065 // however whereis returns only a guid, and guid2lfn does not work
1066 // Therefore we use the one provided as argument for now
1067 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1068 targetFileName = archivefile;
1069 }
1070 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
1071 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1072 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
21ca8e59 1073 }
1074 }
1075 gSystem->ChangeDirectory(cdir);
1076 delete res;
1077 return kTRUE;
1078}
1079
1080//______________________________________________________________________________
c57f56b7 1081Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1082{
1083// Create dataset for the grid data directory + run number.
3af861ff 1084 const Int_t gMaxEntries = 15000;
d3b18c4c 1085 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1086 if (!Connect()) {
1087 Error("CreateDataset", "Cannot create dataset with no grid connection");
1088 return kFALSE;
1089 }
1090
1091 // Cd workspace
d3b18c4c 1092 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1093 TString workdir = gGrid->GetHomeDirectory();
1094 workdir += fGridWorkingDir;
1095
1096 // Compose the 'find' command arguments
74013535 1097 TString format;
c57f56b7 1098 TString command;
1099 TString options = "-x collection ";
bb885a9e 1100 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1101 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1102 TString conditions = "";
3af861ff 1103 Int_t nstart = 0;
1104 Int_t ncount = 0;
1105 Int_t stage = 0;
c57f56b7 1106 TString file;
1107 TString path;
319593fb 1108 Int_t nruns = 0;
904f9f5f 1109 TString schunk, schunk2;
ab254fd1 1110 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1111 if (!fRunNumbers.Length() && !fRunRange[0]) {
1112 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1113 // Make a single data collection from data directory.
1114 path = fGridDataDir;
923e2ca5 1115 if (!DirectoryExists(path)) {
d2a409b2 1116 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1117 return kFALSE;
1118 }
923e2ca5 1119// CdWork();
d2a409b2 1120 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1121 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1122 while (1) {
1123 ncount = 0;
1124 stage++;
1125 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1126 command = "find ";
1127 command += Form("%s -o %d ",options.Data(), nstart);
1128 command += path;
1129 command += " ";
1130 command += pattern;
1131 command += conditions;
1132 printf("command: %s\n", command.Data());
1133 TGridResult *res = gGrid->Command(command);
1134 if (res) delete res;
1135 // Write standard output to file
1136 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1137 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1138 Bool_t nullFile = kFALSE;
1139 if (!hasGrep) {
1140 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1141 } else {
1142 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1143 if (nullFile) {
1144 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1145 gSystem->Exec("rm -f __tmp*");
1146 return kFALSE;
1147 }
1148 TString line;
1149 ifstream in;
1150 in.open("__tmp__");
1151 in >> line;
1152 in.close();
1153 gSystem->Exec("rm -f __tmp__");
1154 ncount = line.Atoi();
1155 }
1156 }
1157 if (ncount == gMaxEntries) {
1158 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1159 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1160 if (!cbase) cbase = cadd;
1161 else {
1162 cbase->Add(cadd);
1163 delete cadd;
defd7a3a 1164 }
3af861ff 1165 nstart += ncount;
1166 } else {
1167 if (cbase) {
1168 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1169 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1170 cbase->Add(cadd);
1171 delete cadd;
1172 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1173 delete cbase; cbase = 0;
1174 } else {
1175 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1176 }
1177 gSystem->Exec("rm -f __tmp*");
1178 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1179 break;
1180 }
0f389141 1181 }
1182 Bool_t fileExists = FileExists(file);
1183 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1184 // Copy xml file to alien space
0f389141 1185 if (fileExists) gGrid->Rm(file);
d2a409b2 1186 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1187 if (!FileExists(file)) {
1188 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1189 return kFALSE;
1190 }
1191 // Update list of files to be processed.
1192 }
1193 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1194 return kTRUE;
1195 }
c57f56b7 1196 // Several runs
a2f5fc01 1197 Bool_t nullResult = kTRUE;
a8739e8a 1198 if (fRunNumbers.Length()) {
1199 TObjArray *arr = fRunNumbers.Tokenize(" ");
1200 TObjString *os;
1201 TIter next(arr);
1202 while ((os=(TObjString*)next())) {
3af861ff 1203 nstart = 0;
1204 stage = 0;
dc030a24 1205 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1206 if (!DirectoryExists(path)) continue;
1207// CdWork();
a8739e8a 1208 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1209 else file = Form("%s.xml", os->GetString().Data());
319593fb 1210 // If local collection file does not exist, create it via 'find' command.
3af861ff 1211 while (1) {
1212 ncount = 0;
1213 stage++;
1214 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1215 command = "find ";
1216 command += Form("%s -o %d ",options.Data(), nstart);
1217 command += path;
1218 command += pattern;
1219 command += conditions;
1220 TGridResult *res = gGrid->Command(command);
1221 if (res) delete res;
1222 // Write standard output to file
1223 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1224 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1225 Bool_t nullFile = kFALSE;
1226 if (!hasGrep) {
1227 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1228 } else {
1229 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1230 if (nullFile) {
1231 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1232 gSystem->Exec("rm -f __tmp*");
1233 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1234 break;
3af861ff 1235 }
1236 TString line;
1237 ifstream in;
1238 in.open("__tmp__");
1239 in >> line;
1240 in.close();
1241 gSystem->Exec("rm -f __tmp__");
1242 ncount = line.Atoi();
1243 }
1244 nullResult = kFALSE;
1245 }
1246 if (ncount == gMaxEntries) {
1247 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1248 if (fNrunsPerMaster > 1) {
1249 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1250 file.Data(),gMaxEntries);
1251 return kFALSE;
1252 }
1253 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1254 if (!cbase) cbase = cadd;
1255 else {
1256 cbase->Add(cadd);
1257 delete cadd;
defd7a3a 1258 }
3af861ff 1259 nstart += ncount;
1260 } else {
c89a84bc 1261 if (cbase && fNrunsPerMaster<2) {
3af861ff 1262 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1263 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1264 cbase->Add(cadd);
1265 delete cadd;
1266 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1267 delete cbase; cbase = 0;
1268 } else {
1269 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1270 }
1271 gSystem->Exec("rm -f __tmp*");
1272 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1273 break;
d3339be3 1274 }
3af861ff 1275 }
a8739e8a 1276 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1277 // Check if there is one run per master job.
1278 if (fNrunsPerMaster<2) {
1279 if (FileExists(file)) {
0f389141 1280 if (fOverwriteMode) gGrid->Rm(file);
1281 else {
1282 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1283 continue;
1284 }
319593fb 1285 }
1286 // Copy xml file to alien space
1287 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1288 if (!FileExists(file)) {
1289 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1290 delete arr;
1291 return kFALSE;
1292 }
1293 } else {
1294 nruns++;
1295 if (((nruns-1)%fNrunsPerMaster) == 0) {
1296 schunk = os->GetString();
1297 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1298 } else {
1299 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1300 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1301 cbase->Add(cadd);
1302 delete cadd;
1303 }
1304 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1305 continue;
1306 }
1307 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1308 if (FileExists(schunk)) {
1309 if (fOverwriteMode) gGrid->Rm(file);
1310 else {
1311 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1312 continue;
1313 }
319593fb 1314 }
84fcd93f 1315 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1316 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1317 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1318 if (!FileExists(schunk)) {
1319 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1320 delete arr;
1321 return kFALSE;
1322 }
d3339be3 1323 }
a8739e8a 1324 }
1325 delete arr;
a2f5fc01 1326 if (nullResult) {
d3339be3 1327 Error("CreateDataset", "No valid dataset corresponding to the query!");
1328 return kFALSE;
1329 }
a8739e8a 1330 } else {
1331 // Process a full run range.
1332 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1333 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1334 nstart = 0;
1335 stage = 0;
74013535 1336 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1337 if (!DirectoryExists(path)) continue;
1338// CdWork();
74013535 1339 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1340 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1341 else file = Form(format.Data(), irun);
0f389141 1342 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1343 if (fOverwriteMode) gGrid->Rm(file);
1344 else {
1345 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1346 continue;
1347 }
a8739e8a 1348 }
319593fb 1349 // If local collection file does not exist, create it via 'find' command.
3af861ff 1350 while (1) {
1351 ncount = 0;
1352 stage++;
1353 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1354 command = "find ";
1355 command += Form("%s -o %d ",options.Data(), nstart);
1356 command += path;
1357 command += pattern;
1358 command += conditions;
1359 TGridResult *res = gGrid->Command(command);
1360 if (res) delete res;
1361 // Write standard output to file
1362 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1363 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1364 Bool_t nullFile = kFALSE;
1365 if (!hasGrep) {
1366 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1367 } else {
1368 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1369 if (nullFile) {
1370 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1371 gSystem->Exec("rm -f __tmp*");
1604c9ed 1372 break;
3af861ff 1373 }
1374 TString line;
1375 ifstream in;
1376 in.open("__tmp__");
1377 in >> line;
1378 in.close();
1379 gSystem->Exec("rm -f __tmp__");
1380 ncount = line.Atoi();
1381 }
1382 nullResult = kFALSE;
1383 }
1384 if (ncount == gMaxEntries) {
1385 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1386 if (fNrunsPerMaster > 1) {
1387 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1388 file.Data(),gMaxEntries);
1389 return kFALSE;
1390 }
1391 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1392 if (!cbase) cbase = cadd;
1393 else {
1394 cbase->Add(cadd);
1395 delete cadd;
defd7a3a 1396 }
3af861ff 1397 nstart += ncount;
1398 } else {
c89a84bc 1399 if (cbase && fNrunsPerMaster<2) {
3af861ff 1400 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1401 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1402 cbase->Add(cadd);
1403 delete cadd;
1404 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1405 delete cbase; cbase = 0;
1406 } else {
1407 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1408 }
1409 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1410 break;
d3339be3 1411 }
319593fb 1412 }
a8739e8a 1413 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1414 // Check if there is one run per master job.
1415 if (fNrunsPerMaster<2) {
1416 if (FileExists(file)) {
0f389141 1417 if (fOverwriteMode) gGrid->Rm(file);
1418 else {
1419 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1420 continue;
1421 }
319593fb 1422 }
1423 // Copy xml file to alien space
1424 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1425 if (!FileExists(file)) {
1426 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1427 return kFALSE;
1428 }
1429 } else {
1430 nruns++;
95e5b448 1431 // Check if the collection for the chunk exist locally.
1432 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1433 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1434 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1435 else continue;
1436 }
84fcd93f 1437 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1438 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1439 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1440 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1441 } else {
1442 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1443 cbase->Add(cadd);
1444 delete cadd;
1445 }
74013535 1446 format = Form("%%s_%s.xml", fRunPrefix.Data());
1447 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1448 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1449 continue;
1450 }
904f9f5f 1451 schunk = schunk2;
319593fb 1452 if (FileExists(schunk)) {
0f389141 1453 if (fOverwriteMode) gGrid->Rm(schunk);
1454 else {
1455 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1456 continue;
1457 }
319593fb 1458 }
84fcd93f 1459 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1460 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1461 if (FileExists(schunk)) {
0f389141 1462 if (fOverwriteMode) gGrid->Rm(schunk);
1463 else {
1464 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1465 continue;
1466 }
95e5b448 1467 }
319593fb 1468 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1469 if (!FileExists(schunk)) {
1470 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1471 return kFALSE;
1472 }
1473 }
c57f56b7 1474 }
a2f5fc01 1475 if (nullResult) {
d3339be3 1476 Error("CreateDataset", "No valid dataset corresponding to the query!");
1477 return kFALSE;
1478 }
a8739e8a 1479 }
c57f56b7 1480 return kTRUE;
1481}
1482
1483//______________________________________________________________________________
1484Bool_t AliAnalysisAlien::CreateJDL()
1485{
1486// Generate a JDL file according to current settings. The name of the file is
1487// specified by fJDLName.
1488 Bool_t error = kFALSE;
1489 TObjArray *arr = 0;
1490 Bool_t copy = kTRUE;
d3b18c4c 1491 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1492 Bool_t generate = kTRUE;
1493 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1494 if (!Connect()) {
1495 Error("CreateJDL", "Alien connection required");
1496 return kFALSE;
1497 }
1498 // Check validity of alien workspace
d3b18c4c 1499 TString workdir;
1500 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1501 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1502 workdir += fGridWorkingDir;
1503 if (generate) {
1504 TObjString *os;
1505 if (!fInputFiles) {
1506 Error("CreateJDL()", "Define some input files for your analysis.");
1507 error = kTRUE;
1508 }
1509 // Compose list of input files
1510 // Check if output files were defined
1511 if (!fOutputFiles.Length()) {
1512 Error("CreateJDL", "You must define at least one output file");
1513 error = kTRUE;
1514 }
1515 // Check if an output directory was defined and valid
1516 if (!fGridOutputDir.Length()) {
1517 Error("CreateJDL", "You must define AliEn output directory");
1518 error = kTRUE;
1519 } else {
d3b18c4c 1520 if (!fProductionMode) {
1521 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1522 if (!DirectoryExists(fGridOutputDir)) {
1523 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1524 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1525 } else {
1526 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1527 // error = kTRUE;
1528 }
67a3b254 1529 } else {
1530 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1531 }
1532 gGrid->Cd(workdir);
1533 }
c57f56b7 1534 }
1535 // Exit if any error up to now
1536 if (error) return kFALSE;
1537 // Set JDL fields
0f389141 1538 if (!fUser.IsNull()) {
1539 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1540 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1541 }
1542 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1543 TString mergeExec = fExecutable;
1544 mergeExec.ReplaceAll(".sh", "_merge.sh");
1545 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1546 mergeExec.ReplaceAll(".sh", ".C");
1547 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1548 if (!fArguments.IsNull())
1549 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1550 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1551 else {
1552 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1553 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1554 }
b3e07543 1555
1f0d1ca2 1556 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1557 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1558 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1559 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1560
0f389141 1561 if (fMaxInitFailed > 0) {
c57f56b7 1562 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1563 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1564 }
1565 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1566 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1567 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1568 }
1569 if (!IsOneStageMerging()) {
f47d5cb4 1570 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1571 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1572 }
1573 if (fSplitMode.Length()) {
c57f56b7 1574 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1575 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1576 }
1577 fMergingJDL->SetValue("Split", "\"se\"");
1578 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1579 if (!fAliROOTVersion.IsNull()) {
1580 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1581 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1582 }
1583 if (!fROOTVersion.IsNull()) {
c57f56b7 1584 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1585 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1586 }
1587 if (!fAPIVersion.IsNull()) {
c57f56b7 1588 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1589 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1590 }
648174cf 1591 if (!fExternalPackages.IsNull()) {
1592 arr = fExternalPackages.Tokenize(" ");
1593 TIter next(arr);
1594 while ((os=(TObjString*)next())) {
1595 TString pkgname = os->GetString();
1596 Int_t index = pkgname.Index("::");
1597 TString pkgversion = pkgname(index+2, pkgname.Length());
1598 pkgname.Remove(index);
1599 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1600 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1601 }
1602 delete arr;
1603 }
0f389141 1604 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1605 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1606 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1607 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1608 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1609 TString analysisFile = fExecutable;
1610 analysisFile.ReplaceAll(".sh", ".root");
1611 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1612 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1613 if (fAdditionalLibs.Length()) {
1614 arr = fAdditionalLibs.Tokenize(" ");
1615 TIter next(arr);
1616 while ((os=(TObjString*)next())) {
1617 if (os->GetString().Contains(".so")) continue;
1618 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1619 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1620 }
1621 delete arr;
1622 }
4e5c5506 1623 if (fPackages) {
1624 TIter next(fPackages);
1625 TObject *obj;
0f389141 1626 while ((obj=next())) {
4e5c5506 1627 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1628 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1629 }
4e5c5506 1630 }
c57f56b7 1631 if (fOutputArchive.Length()) {
1632 arr = fOutputArchive.Tokenize(" ");
1633 TIter next(arr);
0f389141 1634 Bool_t first = kTRUE;
1635 const char *comment = "Files to be archived";
1636 const char *comment1 = comment;
1637 while ((os=(TObjString*)next())) {
1638 if (!first) comment = NULL;
1639 if (!os->GetString().Contains("@") && fCloseSE.Length())
1640 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1641 else
1642 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1643 first = kFALSE;
1644 }
c57f56b7 1645 delete arr;
f790bc1b 1646 // Output archive for the merging jdl
1647 TString outputArchive;
1648 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1649 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1650 // Add normal output files, extra files + terminate files
1651 TString files = GetListOfFiles("outextter");
37d21c01 1652 // Do not register files in fRegisterExcludes
1653 if (!fRegisterExcludes.IsNull()) {
1654 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1655 TIter next1(arr);
1656 while ((os=(TObjString*)next1())) {
1657 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1658 files.ReplaceAll(os->GetString(),"");
1659 }
66d71516 1660 delete arr;
0f389141 1661 }
f790bc1b 1662 files.ReplaceAll(".root", "*.root");
b83f84f0 1663 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1664 } else {
95f84779 1665 TString files = fOutputArchive;
1666 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1667 outputArchive = files;
f790bc1b 1668 }
0f389141 1669 arr = outputArchive.Tokenize(" ");
705adb3e 1670 TIter next2(arr);
0f389141 1671 comment = comment1;
1672 first = kTRUE;
705adb3e 1673 while ((os=(TObjString*)next2())) {
0f389141 1674 if (!first) comment = NULL;
0cdf65a8 1675 TString currentfile = os->GetString();
0cdf65a8 1676 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1677 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1678 else
0cdf65a8 1679 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1680 first = kFALSE;
1681 }
1682 delete arr;
c57f56b7 1683 }
149d288c 1684 arr = fOutputFiles.Tokenize(",");
c57f56b7 1685 TIter next(arr);
0f389141 1686 Bool_t first = kTRUE;
f790bc1b 1687 const char *comment = "Files to be saved";
43da816a 1688 while ((os=(TObjString*)next())) {
e1eaf596 1689 // Ignore ouputs in jdl that are also in outputarchive
1690 TString sout = os->GetString();
f790bc1b 1691 sout.ReplaceAll("*", "");
1692 sout.ReplaceAll(".root", "");
e1eaf596 1693 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1694 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1695 // Ignore fRegisterExcludes
1696 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1697 if (!first) comment = NULL;
43da816a 1698 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1699 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1700 else
0f389141 1701 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1702 first = kFALSE;
f790bc1b 1703 if (fMergeExcludes.Contains(sout)) continue;
1704 if (!os->GetString().Contains("@") && fCloseSE.Length())
1705 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1706 else
1707 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1708 }
c57f56b7 1709 delete arr;
0f389141 1710 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1711 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1712 TString validationScript = fValidationScript;
0f389141 1713 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1714 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1715 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1716 if (fMasterResubmitThreshold) {
1717 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1718 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1719 }
d2a409b2 1720 // Write a jdl with 2 input parameters: collection name and output dir name.
1721 WriteJDL(copy);
c57f56b7 1722 }
1723 // Copy jdl to grid workspace
a8739e8a 1724 if (copy) {
b5e4aaa7 1725 // Check if an output directory was defined and valid
1726 if (!fGridOutputDir.Length()) {
1727 Error("CreateJDL", "You must define AliEn output directory");
1728 return kFALSE;
1729 } else {
1730 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1731 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1732 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1733 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1734 } else {
1735 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1736 return kFALSE;
1737 }
1738 }
1739 gGrid->Cd(workdir);
1740 }
648174cf 1741 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1742 TString mergeJDLName = fExecutable;
1743 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1744 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1745 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1746 if (fProductionMode) {
648174cf 1747 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1748 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1749 }
648174cf 1750 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1751 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1752 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1753 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1754 Fatal("","Terminating");
1755// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1756 if (fMergeViaJDL) {
1757 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1758// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1759 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1760 Fatal("","Terminating");
0f389141 1761 }
648174cf 1762 }
c57f56b7 1763 if (fAdditionalLibs.Length()) {
e7c71df0 1764 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1765 TObjString *os;
1766 TIter next(arr);
1767 while ((os=(TObjString*)next())) {
c57f56b7 1768 if (os->GetString().Contains(".so")) continue;
4e5c5506 1769 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1770 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1771// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1772 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1773 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1774 Fatal("","Terminating");
c57f56b7 1775 }
1776 delete arr;
1777 }
4e5c5506 1778 if (fPackages) {
1779 TIter next(fPackages);
1780 TObject *obj;
1781 while ((obj=next())) {
fdbbc7be 1782 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1783 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1784// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1785 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1786 Form("%s/%s", workdir.Data(), obj->GetName())))
1787 Fatal("","Terminating");
4e5c5506 1788 }
1789 }
c57f56b7 1790 }
1791 return kTRUE;
1792}
1793
1794//______________________________________________________________________________
d2a409b2 1795Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1796{
1797// Writes one or more JDL's corresponding to findex. If findex is negative,
1798// all run numbers are considered in one go (jdl). For non-negative indices
1799// they correspond to the indices in the array fInputFiles.
1800 if (!fInputFiles) return kFALSE;
d3b18c4c 1801 TObject *os;
1802 TString workdir;
1803 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1804 workdir += fGridWorkingDir;
f47d5cb4 1805 TString stageName = "$2";
1806 if (fProductionMode) stageName = "$4";
1807 if (!fMergeDirName.IsNull()) {
1808 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1809 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1810 } else {
1811 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1812 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1813 }
d3b18c4c 1814 if (fProductionMode) {
a8739e8a 1815 TIter next(fInputFiles);
b3e07543 1816 while ((os=next())) {
d3b18c4c 1817 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1818 }
56754bf6 1819 if (!fOutputToRunNo)
1820 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1821 else
1822 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1823 } else {
1824 if (!fRunNumbers.Length() && !fRunRange[0]) {
1825 // One jdl with no parameters in case input data is specified by name.
1826 TIter next(fInputFiles);
1827 while ((os=next()))
1828 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1829 if (!fOutputSingle.IsNull())
1830 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1831 else {
1832 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1833 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1834 }
1835 } else {
1836 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1837 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1838 if (!fOutputSingle.IsNull()) {
1839 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1840 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1841 } else {
1842 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1843 }
1844 }
a8739e8a 1845 }
1846
a8739e8a 1847 // Generate the JDL as a string
1848 TString sjdl = fGridJDL->Generate();
0f389141 1849 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1850 // Final merge jdl
1851 if (!fMergeDirName.IsNull()) {
1852 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1853 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1854 } else {
1855 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1856 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1857 }
b3e07543 1858 TString sjdl2 = fMergingJDL->Generate();
1859 Int_t index, index1;
a8739e8a 1860 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1861 sjdl.ReplaceAll("(member", "\n (member");
1862 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1863 sjdl.ReplaceAll("{", "{\n ");
1864 sjdl.ReplaceAll("};", "\n};");
1865 sjdl.ReplaceAll("{\n \n", "{\n");
1866 sjdl.ReplaceAll("\n\n", "\n");
1867 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1868 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1869 sjdl1.ReplaceAll("(member", "\n (member");
1870 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1871 sjdl1.ReplaceAll("{", "{\n ");
1872 sjdl1.ReplaceAll("};", "\n};");
1873 sjdl1.ReplaceAll("{\n \n", "{\n");
1874 sjdl1.ReplaceAll("\n\n", "\n");
1875 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1876 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1877 sjdl2.ReplaceAll("(member", "\n (member");
1878 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1879 sjdl2.ReplaceAll("{", "{\n ");
1880 sjdl2.ReplaceAll("};", "\n};");
1881 sjdl2.ReplaceAll("{\n \n", "{\n");
1882 sjdl2.ReplaceAll("\n\n", "\n");
1883 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1884 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1885 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1886 index = sjdl.Index("JDLVariables");
1887 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1888 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1889 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1890 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1891 index = fJobTag.Index(":");
1892 if (index < 0) index = fJobTag.Length();
1893 TString jobTag = fJobTag;
f47d5cb4 1894 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1895 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1896 if (fProductionMode) {
1897 sjdl1.Prepend("# Generated merging jdl (production mode) \
1898 \n# $1 = full alien path to output directory to be merged \
1899 \n# $2 = train number \
1900 \n# $3 = production (like LHC10b) \
1901 \n# $4 = merging stage \
1902 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1903 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1904 sjdl2.Prepend("# Generated merging jdl \
1905 \n# $1 = full alien path to output directory to be merged \
1906 \n# $2 = train number \
1907 \n# $3 = production (like LHC10b) \
1908 \n# $4 = merging stage \
1909 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1910 } else {
1911 sjdl1.Prepend("# Generated merging jdl \
1912 \n# $1 = full alien path to output directory to be merged \
1913 \n# $2 = merging stage \
03335324 1914 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1915 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1916 sjdl2.Prepend("# Generated merging jdl \
1917 \n# $1 = full alien path to output directory to be merged \
1918 \n# $2 = merging stage \
03335324 1919 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1920 }
0f389141 1921 index = sjdl1.Index("JDLVariables");
1922 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1923 index = sjdl2.Index("JDLVariables");
1924 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1925 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1926 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1927 index = sjdl2.Index("Split =");
1928 if (index>=0) {
1929 index1 = sjdl2.Index("\n", index);
1930 sjdl2.Remove(index, index1-index+1);
1931 }
1932 index = sjdl2.Index("SplitMaxInputFileNumber");
1933 if (index>=0) {
1934 index1 = sjdl2.Index("\n", index);
1935 sjdl2.Remove(index, index1-index+1);
1936 }
1937 index = sjdl2.Index("InputDataCollection");
1938 if (index>=0) {
1939 index1 = sjdl2.Index(";", index);
1940 sjdl2.Remove(index, index1-index+1);
1941 }
1942 index = sjdl2.Index("InputDataListFormat");
1943 if (index>=0) {
1944 index1 = sjdl2.Index("\n", index);
1945 sjdl2.Remove(index, index1-index+1);
1946 }
1947 index = sjdl2.Index("InputDataList");
1948 if (index>=0) {
1949 index1 = sjdl2.Index("\n", index);
1950 sjdl2.Remove(index, index1-index+1);
1951 }
f47d5cb4 1952 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1953 // Write jdl to file
a8739e8a 1954 ofstream out;
d2a409b2 1955 out.open(fJDLName.Data(), ios::out);
a8739e8a 1956 if (out.bad()) {
d3b18c4c 1957 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1958 return kFALSE;
1959 }
1960 out << sjdl << endl;
b3e07543 1961 out.close();
0f389141 1962 TString mergeJDLName = fExecutable;
1963 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1964 if (fMergeViaJDL) {
1965 ofstream out1;
1966 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1967 if (out1.bad()) {
d3b18c4c 1968 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1969 return kFALSE;
1970 }
1971 out1 << sjdl1 << endl;
b3e07543 1972 out1.close();
1973 ofstream out2;
1974 TString finalJDL = mergeJDLName;
1975 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1976 out2.open(finalJDL.Data(), ios::out);
1977 if (out2.bad()) {
1978 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1979 return kFALSE;
1980 }
1981 out2 << sjdl2 << endl;
1982 out2.close();
0f389141 1983 }
a8739e8a 1984
1985 // Copy jdl to grid workspace
1986 if (!copy) {
d3b18c4c 1987 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1988 } else {
d2a409b2 1989 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1990 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1991 TString finalJDL = mergeJDLName;
1992 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1993 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1994 if (fProductionMode) {
b5fe9cba 1995 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1996 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1997 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1998 }
d2a409b2 1999 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2000 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2001 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2002 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2003// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2004 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2005 Fatal("","Terminating");
0f389141 2006 if (fMergeViaJDL) {
b3e07543 2007 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2008// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2009// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2010 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2011 Fatal("","Terminating");
2012 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2013 Fatal("","Terminating");
0f389141 2014 }
a8739e8a 2015 }
2016 return kTRUE;
2017}
2018
2019//______________________________________________________________________________
5513444a 2020Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2021{
2022// Returns true if file exists.
5513444a 2023 if (!gGrid) return kFALSE;
2ea55496 2024 TString slfn = lfn;
2025 slfn.ReplaceAll("alien://","");
2026 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2027 if (!res) return kFALSE;
2028 TMap *map = dynamic_cast<TMap*>(res->At(0));
2029 if (!map) {
2030 delete res;
2031 return kFALSE;
2032 }
2033 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2034 if (!objs || !objs->GetString().Length()) {
2035 delete res;
2036 return kFALSE;
2037 }
2038 delete res;
2039 return kTRUE;
2040}
2041
2042//______________________________________________________________________________
923e2ca5 2043Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2044{
2045// Returns true if directory exists. Can be also a path.
2046 if (!gGrid) return kFALSE;
2047 // Check if dirname is a path
2048 TString dirstripped = dirname;
2049 dirstripped = dirstripped.Strip();
2050 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2051 TString dir = gSystem->BaseName(dirstripped);
2052 dir += "/";
2053 TString path = gSystem->DirName(dirstripped);
2054 TGridResult *res = gGrid->Ls(path, "-F");
2055 if (!res) return kFALSE;
2056 TIter next(res);
2057 TMap *map;
2058 TObject *obj;
2059 while ((map=dynamic_cast<TMap*>(next()))) {
2060 obj = map->GetValue("name");
2061 if (!obj) break;
2062 if (dir == obj->GetName()) {
2063 delete res;
2064 return kTRUE;
2065 }
2066 }
2067 delete res;
2068 return kFALSE;
2069}
2070
2071//______________________________________________________________________________
a2f5fc01 2072void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2073{
2074// Check input data type.
a2f5fc01 2075 isCollection = kFALSE;
2076 isXml = kFALSE;
2077 useTags = kFALSE;
c57f56b7 2078 if (!gGrid) {
2079 Error("CheckDataType", "No connection to grid");
2080 return;
2081 }
a2f5fc01 2082 isCollection = IsCollection(lfn);
c57f56b7 2083 TString msg = "\n##### file: ";
2084 msg += lfn;
a2f5fc01 2085 if (isCollection) {
c57f56b7 2086 msg += " type: raw_collection;";
2087 // special treatment for collections
a2f5fc01 2088 isXml = kFALSE;
c57f56b7 2089 // check for tag files in the collection
2090 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2091 if (!res) {
2092 msg += " using_tags: No (unknown)";
23329835 2093 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2094 return;
2095 }
2096 const char* typeStr = res->GetKey(0, "origLFN");
2097 if (!typeStr || !strlen(typeStr)) {
2098 msg += " using_tags: No (unknown)";
23329835 2099 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2100 return;
2101 }
2102 TString file = typeStr;
a2f5fc01 2103 useTags = file.Contains(".tag");
2104 if (useTags) msg += " using_tags: Yes";
c57f56b7 2105 else msg += " using_tags: No";
23329835 2106 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2107 return;
2108 }
2109 TString slfn(lfn);
2110 slfn.ToLower();
a2f5fc01 2111 isXml = slfn.Contains(".xml");
2112 if (isXml) {
c57f56b7 2113 // Open xml collection and check if there are tag files inside
2114 msg += " type: xml_collection;";
2115 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2116 if (!coll) {
2117 msg += " using_tags: No (unknown)";
23329835 2118 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2119 return;
2120 }
2121 TMap *map = coll->Next();
2122 if (!map) {
2123 msg += " using_tags: No (unknown)";
23329835 2124 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2125 return;
2126 }
2127 map = (TMap*)map->GetValue("");
2128 TString file;
2129 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2130 useTags = file.Contains(".tag");
c57f56b7 2131 delete coll;
a2f5fc01 2132 if (useTags) msg += " using_tags: Yes";
c57f56b7 2133 else msg += " using_tags: No";
23329835 2134 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2135 return;
2136 }
a2f5fc01 2137 useTags = slfn.Contains(".tag");
c57f56b7 2138 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2139 else msg += " type: unknown file;";
a2f5fc01 2140 if (useTags) msg += " using_tags: Yes";
c57f56b7 2141 else msg += " using_tags: No";
23329835 2142 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2143}
2144
2145//______________________________________________________________________________
4e5c5506 2146void AliAnalysisAlien::EnablePackage(const char *package)
2147{
2148// Enables a par file supposed to exist in the current directory.
2149 TString pkg(package);
2150 pkg.ReplaceAll(".par", "");
2151 pkg += ".par";
2152 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2153 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2154 return;
2155 }
2156 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2157 Info("EnablePackage", "AliEn plugin will use .par packages");
2158 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2159 if (!fPackages) {
2160 fPackages = new TObjArray();
2161 fPackages->SetOwner();
2162 }
2163 fPackages->Add(new TObjString(pkg));
2164}
2165
2166//______________________________________________________________________________
3bdcb562 2167TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2168{
2169// Make a tree from files having the location specified in fFileForTestMode.
2170// Inspired from JF's CreateESDChain.
2171 if (fFileForTestMode.IsNull()) {
2172 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2173 return NULL;
2174 }
2175 if (gSystem->AccessPathName(fFileForTestMode)) {
2176 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2177 return NULL;
2178 }
2179 // Open the file
2180 ifstream in;
2181 in.open(fFileForTestMode);
2182 Int_t count = 0;
2183 // Read the input list of files and add them to the chain
2184 TString line;
2185 TChain *chain = new TChain(treeName);
2186 while (in.good())
2187 {
2188 in >> line;
2189 if (line.IsNull()) continue;
2190 if (count++ == fNtestFiles) break;
2191 TString esdFile(line);
2192 TFile *file = TFile::Open(esdFile);
2193 if (file) {
2194 if (!file->IsZombie()) chain->Add(esdFile);
2195 file->Close();
2196 } else {
2197 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2198 }
2199 }
2200 in.close();
2201 if (!chain->GetListOfFiles()->GetEntries()) {
2202 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2203 delete chain;
2204 return NULL;
2205 }
2206// chain->ls();
2207 return chain;
2208}
2209
2210//______________________________________________________________________________
d2a409b2 2211const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2212{
2213// Get job status for all jobs with jobid>jobidstart.
2214 static char mstatus[20];
2215 mstatus[0] = '\0';
2216 nrunning = 0;
2217 nwaiting = 0;
2218 nerror = 0;
2219 ndone = 0;
2220 TGridJobStatusList *list = gGrid->Ps("");
2221 if (!list) return mstatus;
2222 Int_t nentries = list->GetSize();
2223 TGridJobStatus *status;
2224 Int_t pid;
2225 for (Int_t ijob=0; ijob<nentries; ijob++) {
2226 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2227 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2228 if (pid<jobidstart) continue;
2229 if (pid == lastid) {
8ddad121 2230 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2231 }
2232 switch (status->GetStatus()) {
2233 case TGridJobStatus::kWAITING:
2234 nwaiting++; break;
2235 case TGridJobStatus::kRUNNING:
2236 nrunning++; break;
2237 case TGridJobStatus::kABORTED:
2238 case TGridJobStatus::kFAIL:
2239 case TGridJobStatus::kUNKNOWN:
2240 nerror++; break;
2241 case TGridJobStatus::kDONE:
2242 ndone++;
2243 }
2244 }
2245 list->Delete();
2246 delete list;
2247 return mstatus;
2248}
2249
2250//______________________________________________________________________________
c57f56b7 2251Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2252{
2253// Returns true if file is a collection. Functionality duplicated from
2254// TAlien::Type() because we don't want to directly depend on TAlien.
2255 if (!gGrid) {
2256 Error("IsCollection", "No connection to grid");
2257 return kFALSE;
2258 }
2259 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2260 if (!res) return kFALSE;
2261 const char* typeStr = res->GetKey(0, "type");
2262 if (!typeStr || !strlen(typeStr)) return kFALSE;
2263 if (!strcmp(typeStr, "collection")) return kTRUE;
2264 delete res;
2265 return kFALSE;
2266}
2267
2268//______________________________________________________________________________
fe2d7fc2 2269Bool_t AliAnalysisAlien::IsSingleOutput() const
2270{
2271// Check if single-ouput option is on.
2272 return (!fOutputSingle.IsNull());
2273}
2274
2275//______________________________________________________________________________
16a4353c 2276void AliAnalysisAlien::Print(Option_t *) const
2277{
2278// Print current plugin settings.
84fcd93f 2279 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2280 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2281 if (mgr && mgr->IsProofMode()) {
2282 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2283 if (TestBit(AliAnalysisGrid::kTest))
2284 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2285 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2286 if (!fProofDataSet.IsNull())
2287 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2288 if (fProofReset==1)
2289 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2290 if (fProofReset>1)
2291 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2292 if (!fRootVersionForProof.IsNull())
2293 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2294 else
2295 printf("= ROOT version requested________________________ default\n");
2296 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2297 if (!fAliRootMode.IsNull())
2298 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2299 if (fNproofWorkers)
2300 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2301 if (fNproofWorkersPerSlave)
2302 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2303 if (TestSpecialBit(kClearPackages))
2304 printf("= ClearPackages requested...\n");
2305 if (fIncludePath.Data())
2306 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2307 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2308 if (fPackages && fPackages->GetEntries()) {
2309 TIter next(fPackages);
2310 TObject *obj;
2311 TString list;
2312 while ((obj=next())) list += obj->GetName();
2313 printf("= Par files to be used: ________________________ %s\n", list.Data());
2314 }
2315 if (TestSpecialBit(kProofConnectGrid))
2316 printf("= Requested PROOF connection to grid\n");
2317 return;
2318 }
e1c22e21 2319 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2320 if (fOverwriteMode) {
2321 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2322 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2323 }
348be253 2324 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2325 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2326 printf("= Production mode:______________________________ %d\n", fProductionMode);
2327 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2328 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2329 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2330 if (fUser.Length())
84fcd93f 2331 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2332 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2333 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2334 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2335 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2336 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2337 if (fRunNumbers.Length())
84fcd93f 2338 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2339 if (fRunRange[0])
74013535 2340 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2341 if (!fRunRange[0] && !fRunNumbers.Length()) {
2342 TIter next(fInputFiles);
2343 TObject *obj;
2344 TString list;
2345 while ((obj=next())) list += obj->GetName();
84fcd93f 2346 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2347 }
2348 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2349 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2350 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2351 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2352 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2353 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2354 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2355 printf("=====================================================================\n");
2356 printf("= Job price: ___________________________________ %d\n", fPrice);
2357 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2358 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2359 if (fMaxInitFailed>0)
84fcd93f 2360 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2361 if (fMasterResubmitThreshold>0)
84fcd93f 2362 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2363 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2364 if (fNrunsPerMaster>0)
84fcd93f 2365 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2366 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2367 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2368 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2369 if (fArguments.Length())
84fcd93f 2370 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2371 if (fExecutableArgs.Length())
2372 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2373 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2374 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2375 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2376 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2377 if (fDatasetName)
84fcd93f 2378 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2379 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2380 if (fIncludePath.Data())
84fcd93f 2381 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2382 if (fCloseSE.Length())
84fcd93f 2383 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2384 if (fFriendChainName.Length())
84fcd93f 2385 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2386 if (fPackages && fPackages->GetEntries()) {
16a4353c 2387 TIter next(fPackages);
2388 TObject *obj;
2389 TString list;
2390 while ((obj=next())) list += obj->GetName();
84fcd93f 2391 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2392 }
2393}
2394
2395//______________________________________________________________________________
c57f56b7 2396void AliAnalysisAlien::SetDefaults()
2397{
2398// Set default values for everything. What cannot be filled will be left empty.
2399 if (fGridJDL) delete fGridJDL;
2400 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2401 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2402 fPrice = 1;
2403 fTTL = 30000;
2404 fSplitMaxInputFileNumber = 100;
2405 fMaxInitFailed = 0;
2406 fMasterResubmitThreshold = 0;
bb885a9e 2407 fNtestFiles = 10;
149d288c 2408 fNreplicas = 2;
a8739e8a 2409 fRunRange[0] = 0;
2410 fRunRange[1] = 0;
8e3bd06a 2411 fRunPrefix = "%d";
319593fb 2412 fNrunsPerMaster = 1;
16a4353c 2413 fMaxMergeFiles = 100;
c57f56b7 2414 fRunNumbers = "";
2415 fExecutable = "analysis.sh";
0a1c1f7f 2416 fExecutableCommand = "root -b -q";
c57f56b7 2417 fArguments = "";
631c0b05 2418 fExecutableArgs = "";
c57f56b7 2419 fAnalysisMacro = "myAnalysis.C";
2420 fAnalysisSource = "";
2421 fAdditionalLibs = "";
2422 fSplitMode = "se";
2423 fAPIVersion = "";
2424 fROOTVersion = "";
2425 fAliROOTVersion = "";
2426 fUser = ""; // Your alien user name
2427 fGridWorkingDir = "";
2428 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2429 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2430 fFriendChainName = "";
c57f56b7 2431 fGridOutputDir = "output";
b83f84f0 2432 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2433 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2434 fInputFormat = "xml-single";
2435 fJDLName = "analysis.jdl";
c6cb3634 2436 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2437 fMergeExcludes = "";
0f389141 2438 fMergeViaJDL = 0;
348be253 2439 SetUseCopy(kTRUE);
2440 SetCheckCopy(kTRUE);
149d288c 2441 SetDefaultOutputs(kTRUE);
e1c22e21 2442 fOverwriteMode = 1;
c57f56b7 2443}
2444
2445//______________________________________________________________________________
b3e07543 2446Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2447{
b3e07543 2448// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2449 // First check if the result is already in the output directory.
2450 if (FileExists(Form("%s/%s",aliendir,filename))) {
2451 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2452 return kFALSE;
2453 }
b3e07543 2454 // Now check the last stage done.
2455 Int_t stage = 0;
2456 while (1) {
2457 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2458 stage++;
2459 }
2460 // Next stage of merging
2461 stage++;
2462 TString pattern = "*root_archive.zip";
2463 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2464 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2465 if (res) delete res;
2466 // Write standard output to file
2467 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2468 // Count the number of files inside
2469 ifstream ifile;
2470 ifile.open(Form("Stage_%d.xml",stage));
2471 if (!ifile.good()) {
2472 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2473 return kFALSE;
b3e07543 2474 }
2475 TString line;
2476 Int_t nfiles = 0;
2477 while (!ifile.eof()) {
2478 ifile >> line;
2479 if (line.Contains("/event")) nfiles++;
7c2cd90a 2480 }
b3e07543 2481 ifile.close();
2482 if (!nfiles) {
2483 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2484 return kFALSE;
2485 } else {
2486 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2487 }
b3e07543 2488 // Copy the file in the output directory
2489 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2490// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2491 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2492 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2493 // Check if this is the last stage to be done.
2494 Bool_t laststage = (nfiles<nperchunk);
2495 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2496 if (laststage) {
2497 printf("### Submiting final merging stage %d\n", stage);
2498 TString finalJDL = jdl;
2499 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2500 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2501 Int_t jobId = SubmitSingleJob(query);
2502 if (!jobId) return kFALSE;
2503 } else {
2504 printf("### Submiting merging stage %d\n", stage);
03335324 2505 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2506 Int_t jobId = SubmitSingleJob(query);
2507 if (!jobId) return kFALSE;
7c2cd90a 2508 }
b3e07543 2509 return kTRUE;
2510}
7c2cd90a 2511
2512//______________________________________________________________________________
105631b5 2513AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2514{
2515// Loat the analysis manager from a file.
2516 TFile *file = TFile::Open(fname);
2517 if (!file) {
2518 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2519 return 0;
2520 }
2521 TIter nextkey(file->GetListOfKeys());
2522 AliAnalysisManager *mgr = 0;
2523 TKey *key;
2524 while ((key=(TKey*)nextkey())) {
2525 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2526 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2527 }
2528 if (!mgr)
2529 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2530 return mgr;
2531}
2532
2533//______________________________________________________________________________
7c2cd90a 2534Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2535{
2536// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2537 if (!gGrid) return 0;
2538 printf("=> %s ------> ",query);
2539 TGridResult *res = gGrid->Command(query);
2540 if (!res) return 0;
2541 TString jobId = res->GetKey(0,"jobId");
2542 delete res;
2543 if (jobId.IsNull()) {
2544 printf("submission failed. Reason:\n");
2545 gGrid->Stdout();
2546 gGrid->Stderr();
2547 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2548 return 0;
2549 }
2550 printf(" Job id: %s\n", jobId.Data());
2551 return atoi(jobId);
2552}
2553
2554//______________________________________________________________________________
b3e07543 2555Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2556{
b3e07543 2557// Merge given output files from basedir. Basedir can be an alien output directory
2558// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2559// files in a group (ignored for xml input). Merging can be done in stages:
2560// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2561// stage=1 : works with an xml of all root_archive.zip in the output directory
2562// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2563 TString outputFile = output;
0f389141 2564 TString command;
a2f5fc01 2565 TString outputChunk;
2566 TString previousChunk = "";
b3e07543 2567 TObjArray *listoffiles = new TObjArray();
2568// listoffiles->SetOwner();
a2f5fc01 2569 Int_t countChunk = 0;
2570 Int_t countZero = nmaxmerge;
0f389141 2571 Bool_t merged = kTRUE;
a2f5fc01 2572 Int_t index = outputFile.Index("@");
2573 if (index > 0) outputFile.Remove(index);
7c2cd90a 2574 TString inputFile = outputFile;
b3e07543 2575 TString sbasedir = basedir;
2576 if (sbasedir.Contains(".xml")) {
2577 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2578 nmaxmerge = 9999999;
2579 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2580 if (!coll) {
2581 ::Error("MergeOutput", "Input XML collection empty.");
2582 return kFALSE;
2583 }
2584 // Iterate grid collection
2585 while (coll->Next()) {
2586 TString fname = gSystem->DirName(coll->GetTURL());
2587 fname += "/";
2588 fname += inputFile;
2589 listoffiles->Add(new TNamed(fname.Data(),""));
2590 }
2591 } else {
2592 command = Form("find %s/ *%s", basedir, inputFile.Data());
2593 printf("command: %s\n", command.Data());
2594 TGridResult *res = gGrid->Command(command);
2595 if (!res) {
2596 ::Error("MergeOutput","No result for the find command\n");
2597 delete listoffiles;
2598 return kFALSE;
2599 }
2600 TIter nextmap(res);
2601 TMap *map = 0;
2602 while ((map=(TMap*)nextmap())) {
2603 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2604 if (!objs || !objs->GetString().Length()) {
2605 // Nothing found - skip this output
2606 delete res;
2607 delete listoffiles;
2608 return kFALSE;
2609 }
2610 listoffiles->Add(new TNamed(objs->GetName(),""));
2611 }
2612 delete res;
2613 }
2614 if (!listoffiles->GetEntries()) {
7c2cd90a 2615 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2616 delete listoffiles;
0f389141 2617 return kFALSE;
2618 }
2619
2620 TFileMerger *fm = 0;
b3e07543 2621 TIter next0(listoffiles);
2622 TObjArray *listoffilestmp = new TObjArray();
2623 listoffilestmp->SetOwner();
2624 TObject *nextfile;
2625 TString snextfile;
2626 // Keep only the files at upper level
2627 Int_t countChar = 0;
2628 while ((nextfile=next0())) {
2629 snextfile = nextfile->GetName();
2630 Int_t crtCount = snextfile.CountChar('/');
2631 if (nextfile == listoffiles->First()) countChar = crtCount;
2632 if (crtCount < countChar) countChar = crtCount;
2633 }
2634 next0.Reset();
2635 while ((nextfile=next0())) {
2636 snextfile = nextfile->GetName();
2637 Int_t crtCount = snextfile.CountChar('/');
2638 if (crtCount > countChar) {
2639 delete nextfile;
2640 continue;
2641 }
2642 listoffilestmp->Add(nextfile);
2643 }
2644 delete listoffiles;
2645 listoffiles = listoffilestmp; // Now contains 'good' files
2646 listoffiles->Print();
2647 TIter next(listoffiles);
7c2cd90a 2648 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2649 outputChunk = outputFile;
2650 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2651 // Check for existent temporary merge files
e1c22e21 2652 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2653 // Preserve old merging functionality for stage 0.
2654 if (stage==0) {
2655 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2656 while (1) {
2657 // Skip as many input files as in a chunk
2ea55496 2658 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2659 nextfile = next();
2660 if (!nextfile) {
2ea55496 2661 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2662 delete listoffiles;
2ea55496 2663 return kFALSE;
2664 }
b3e07543 2665 snextfile = nextfile->GetName();
7c2cd90a 2666 }
2667 outputChunk = outputFile;
2668 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2669 countChunk++;
2670 if (gSystem->AccessPathName(outputChunk)) continue;
2671 // Merged file with chunks up to <countChunk> found
2672 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2673 previousChunk = outputChunk;
2674 break;
2675 }
2676 }
2677 countZero = nmaxmerge;
2678
b3e07543 2679 while ((nextfile=next())) {
2680 snextfile = nextfile->GetName();
2ea55496 2681 // Loop 'find' results and get next LFN
7c2cd90a 2682 if (countZero == nmaxmerge) {
2683 // First file in chunk - create file merger and add previous chunk if any.
2b895f34 2684 fm = new TFileMerger(kTRUE);
7c2cd90a 2685 fm->SetFastMethod(kTRUE);
2686 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2687 outputChunk = outputFile;
2688 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2689 }
2690 // If last file found, put merged results in the output file
b3e07543 2691 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2692 // Add file to be merged and decrement chunk counter.
b3e07543 2693 fm->AddFile(snextfile);
7c2cd90a 2694 countZero--;
b3e07543 2695 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2696 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2697 // Nothing found - skip this output
2698 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2699 merged = kFALSE;
2700 break;
7c2cd90a 2701 }
2702 fm->OutputFile(outputChunk);
2703 // Merge the outputs, then go to next chunk
2704 if (!fm->Merge()) {
2705 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2706 merged = kFALSE;
2707 break;
7c2cd90a 2708 } else {
2709 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2710 gSystem->Unlink(previousChunk);
2711 }
b3e07543 2712 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2713 countChunk++;
2714 countZero = nmaxmerge;
2715 previousChunk = outputChunk;
0f389141 2716 }
0f389141 2717 }
b3e07543 2718 delete listoffiles;
2719 delete fm;
7c2cd90a 2720 return merged;
2721 }
2722 // Merging stage different than 0.
2723 // Move to the begining of the requested chunk.
2b895f34 2724 fm = new TFileMerger(kTRUE);
7c2cd90a 2725 fm->SetFastMethod(kTRUE);
b3e07543 2726 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2727 delete listoffiles;
7c2cd90a 2728 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2729 // Nothing found - skip this output
2730 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2731 delete fm;
2732 return kFALSE;
2733 }
b3e07543 2734 fm->OutputFile(outputFile);
7c2cd90a 2735 // Merge the outputs
2736 if (!fm->Merge()) {
2737 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2738 delete fm;
2739 return kFALSE;
2740 } else {
b3e07543 2741 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2742 }
7c2cd90a 2743 delete fm;
2744 return kTRUE;
0f389141 2745}
2746
2747//______________________________________________________________________________
c57f56b7 2748Bool_t AliAnalysisAlien::MergeOutputs()
2749{
2750// Merge analysis outputs existing in the AliEn space.
2751 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2752 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2753 if (!Connect()) {
2754 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2755 return kFALSE;
0f389141 2756 }
c9e8f7fd 2757 if (fMergeViaJDL) {
2758 if (!TestBit(AliAnalysisGrid::kMerge)) {
2759 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2760 return kFALSE;
2761 }
2762 if (fProductionMode) {
2763 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2764 return kFALSE;
2765 }
0f389141 2766 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2767 if (!SubmitMerging()) return kFALSE;
0f389141 2768 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2769 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2770 return kFALSE;
c57f56b7 2771 }
2772 // Get the output path
dc030a24 2773 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2774 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2775 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2776 return kFALSE;
2777 }
2778 if (!fOutputFiles.Length()) {
2779 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2780 return kFALSE;
0f389141 2781 }
2782 // Check if fast read option was requested
e1c22e21 2783 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2784 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2785 if (fFastReadOption) {
d3339be3 2786 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2787 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2788 gEnv->SetValue("XNet.ConnectTimeout",50);
2789 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2790 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2791 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2792 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2793 }
e8b839ab 2794 // Make sure we change the temporary directory
2795 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2796 // Set temporary compilation directory to current one
2797 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2798 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2799 TIter next(list);
2800 TObjString *str;
a2f5fc01 2801 TString outputFile;
c57f56b7 2802 Bool_t merged = kTRUE;
2803 while((str=(TObjString*)next())) {
a2f5fc01 2804 outputFile = str->GetString();
2805 Int_t index = outputFile.Index("@");
2806 if (index > 0) outputFile.Remove(index);
2807 TString outputChunk = outputFile;
2808 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2809 // Skip already merged outputs
a2f5fc01 2810 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2811 if (fOverwriteMode) {
a2f5fc01 2812 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2813 gSystem->Unlink(outputFile);
2814 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2815 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2816 outputChunk.Data());
2817 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2818 }
2819 } else {
a2f5fc01 2820 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2821 continue;
2822 }
2823 } else {
a2f5fc01 2824 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2825 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2826 outputChunk.Data());
2827 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2828 }
2829 }
37d21c01 2830 if (fMergeExcludes.Contains(outputFile.Data()) ||
2831 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 2832 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2833 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2834 if (!merged) {
2835 Error("MergeOutputs", "Terminate() will NOT be executed");
2836 return kFALSE;
ff07ec61 2837 }
2838 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2839 if (fileOpened) fileOpened->Close();
c57f56b7 2840 }
0f389141 2841 return kTRUE;
c57f56b7 2842}
2843
2844//______________________________________________________________________________
bb885a9e 2845void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2846{
2847// Use the output files connected to output containers from the analysis manager
2848// rather than the files defined by SetOutputFiles
2849 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2850 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2851 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2852}
2853
2854//______________________________________________________________________________
149d288c 2855void AliAnalysisAlien::SetOutputFiles(const char *list)
2856{
2857// Manually set the output files list.
2858// Removes duplicates. Not allowed if default outputs are not disabled.
2859 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2860 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2861 return;
2862 }
2863 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2864 fOutputFiles = "";
2865 TString slist = list;
2866 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2867 TObjArray *arr = slist.Tokenize(" ");
2868 TObjString *os;
2869 TIter next(arr);
2870 TString sout;
2871 while ((os=(TObjString*)next())) {
2872 sout = os->GetString();
2873 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2874 if (fOutputFiles.Contains(sout)) continue;
2875 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2876 fOutputFiles += sout;
2877 }
2878 delete arr;
f790bc1b 2879}
149d288c 2880
2881//______________________________________________________________________________
2882void AliAnalysisAlien::SetOutputArchive(const char *list)
2883{
2884// Manually set the output archive list. Free text - you are on your own...
2885// Not allowed if default outputs are not disabled.
2886 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2887 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2888 return;
2889 }
2890 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2891 fOutputArchive = list;
2892}
2893
2894//______________________________________________________________________________
2895void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2896{
2897// Setting a prefered output SE is not allowed anymore.
2898 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2899}
2900
2901//______________________________________________________________________________
28ce6c9c 2902void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
2903{
2904// Set some PROOF special parameter.
2905 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2906 if (pair) {
2907 TObject *old = pair->Key();
2908 TObject *val = pair->Value();
2909 fProofParam.Remove(old);
2910 delete old;
2911 delete val;
2912 }
2913 fProofParam.Add(new TObjString(pname), new TObjString(value));
2914}
2915
2916//______________________________________________________________________________
2917const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
2918{
2919// Returns a special PROOF parameter.
2920 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2921 if (!pair) return 0;
2922 return pair->Value()->GetName();
2923}
2924
2925//______________________________________________________________________________
5513444a 2926Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2927{
2928// Start remote grid analysis.
3bdcb562 2929 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2930 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2931 if (!mgr || !mgr->IsInitialized()) {
2932 Error("StartAnalysis", "You need an initialized analysis manager for this");
2933 return kFALSE;
2934 }
2935 // Are we in PROOF mode ?
2936 if (mgr->IsProofMode()) {
baa34407 2937 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
2938 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2939 if (fProofCluster.IsNull()) {
2940 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2941 return kFALSE;
2942 }
2943 if (fProofDataSet.IsNull() && !testMode) {
2944 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2945 return kFALSE;
2946 }
2947 // Set the needed environment
2948 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2949 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2950 if (fProofReset && !testMode) {
2951 if (fProofReset==1) {
2952 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2953 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2954 } else {
2955 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2956 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2957 }
2958 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2959 return kFALSE;
2960 }
baa34407 2961
2962 if (!testMode) {
2963 // Check if there is an old active session
2964 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
2965 if (nsessions) {
2966 Error("StartAnalysis","You have to reset your old session first\n");
2967 return kFALSE;
2968 }
53911ca6 2969 }
3bdcb562 2970 // Do we need to change the ROOT version ? The success of this cannot be checked.
2971 if (!fRootVersionForProof.IsNull() && !testMode) {
2972 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2973 fProofCluster.Data(), fRootVersionForProof.Data()));
2974 }
2975 // Connect to PROOF and check the status
2976 Long_t proof = 0;
27734f0e 2977 TString sworkers;
2978 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2979 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2980 if (!testMode) {
27734f0e 2981 if (!sworkers.IsNull())
2982 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2983 else
2984 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2985 } else {
2986 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2987 if (!proof) {
2988 Error("StartAnalysis", "Could not start PROOF in test mode");
2989 return kFALSE;
2990 }
2991 }
2992 if (!proof) {
2993 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2994 return kFALSE;
2995 }
27734f0e 2996 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2997 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 2998 // Set proof special parameters if any
2999 TIter nextpp(&fProofParam);
3000 TObject *proofparam;
3001 while ((proofparam=nextpp())) {
3002 TString svalue = GetProofParameter(proofparam->GetName());
3003 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3004 }
3bdcb562 3005 // Is dataset existing ?
3006 if (!testMode) {
3007 TString dataset = fProofDataSet;
3008 Int_t index = dataset.Index("#");
3009 if (index>=0) dataset.Remove(index);
830acc4c 3010// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"