]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Bringing CMakeLists under svn maintenance
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
c82bb898 51using std::ofstream;
52using std::ifstream;
53using std::ios;
54using std::endl;
c57f56b7 55ClassImp(AliAnalysisAlien)
6c395669 56#if 0
57;
58#endif
c57f56b7 59
6c395669 60namespace {
61 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
62 {
63 TString sl(Form("file:%s", loc));
64 TString sr(Form("alien://%s", rem));
65 Bool_t ret = TFile::Cp(sl, sr);
66 if (!ret) {
67 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
68 }
69 return ret;
70 }
71}
72
c57f56b7 73//______________________________________________________________________________
74AliAnalysisAlien::AliAnalysisAlien()
75 :AliAnalysisGrid(),
76 fGridJDL(NULL),
0f389141 77 fMergingJDL(NULL),
c57f56b7 78 fPrice(0),
79 fTTL(0),
80 fSplitMaxInputFileNumber(0),
81 fMaxInitFailed(0),
82 fMasterResubmitThreshold(0),
bb885a9e 83 fNtestFiles(0),
319593fb 84 fNrunsPerMaster(0),
16a4353c 85 fMaxMergeFiles(0),
b3e07543 86 fMaxMergeStages(0),
d2a409b2 87 fNsubmitted(0),
a3e84053 88 fProductionMode(0),
cd11251e 89 fOutputToRunNo(0),
0f389141 90 fMergeViaJDL(0),
91 fFastReadOption(0),
e1c22e21 92 fOverwriteMode(1),
149d288c 93 fNreplicas(2),
3bdcb562 94 fNproofWorkers(0),
27734f0e 95 fNproofWorkersPerSlave(0),
3bdcb562 96 fProofReset(0),
c57f56b7 97 fRunNumbers(),
98 fExecutable(),
0a1c1f7f 99 fExecutableCommand(),
c57f56b7 100 fArguments(),
631c0b05 101 fExecutableArgs(),
c57f56b7 102 fAnalysisMacro(),
103 fAnalysisSource(),
d3b18c4c 104 fValidationScript(),
d5c6455a 105 fAdditionalRootLibs(),
c57f56b7 106 fAdditionalLibs(),
107 fSplitMode(),
108 fAPIVersion(),
109 fROOTVersion(),
110 fAliROOTVersion(),
648174cf 111 fExternalPackages(),
c57f56b7 112 fUser(),
113 fGridWorkingDir(),
114 fGridDataDir(),
115 fDataPattern(),
116 fGridOutputDir(),
117 fOutputArchive(),
118 fOutputFiles(),
119 fInputFormat(),
e7c71df0 120 fDatasetName(),
c57f56b7 121 fJDLName(),
d3b18c4c 122 fTerminateFiles(),
bb885a9e 123 fMergeExcludes(),
37d21c01 124 fRegisterExcludes(),
f965131e 125 fIncludePath(),
bb885a9e 126 fCloseSE(),
0df6ccf2 127 fFriendChainName(),
c6cb3634 128 fJobTag(),
648174cf 129 fOutputSingle(),
5fce53f4 130 fRunPrefix(),
3bdcb562 131 fProofCluster(),
132 fProofDataSet(),
133 fFileForTestMode(),
134 fRootVersionForProof(),
135 fAliRootMode(),
f47d5cb4 136 fMergeDirName(),
4e5c5506 137 fInputFiles(0),
28ce6c9c 138 fPackages(0),
77f34eae 139 fModules(0),
28ce6c9c 140 fProofParam()
c57f56b7 141{
142// Dummy ctor.
143 SetDefaults();
144}
145
146//______________________________________________________________________________
147AliAnalysisAlien::AliAnalysisAlien(const char *name)
148 :AliAnalysisGrid(name),
149 fGridJDL(NULL),
0f389141 150 fMergingJDL(NULL),
c57f56b7 151 fPrice(0),
152 fTTL(0),
153 fSplitMaxInputFileNumber(0),
154 fMaxInitFailed(0),
155 fMasterResubmitThreshold(0),
bb885a9e 156 fNtestFiles(0),
319593fb 157 fNrunsPerMaster(0),
16a4353c 158 fMaxMergeFiles(0),
b3e07543 159 fMaxMergeStages(0),
d2a409b2 160 fNsubmitted(0),
a3e84053 161 fProductionMode(0),
cd11251e 162 fOutputToRunNo(0),
0f389141 163 fMergeViaJDL(0),
164 fFastReadOption(0),
e1c22e21 165 fOverwriteMode(1),
149d288c 166 fNreplicas(2),
3bdcb562 167 fNproofWorkers(0),
27734f0e 168 fNproofWorkersPerSlave(0),
3bdcb562 169 fProofReset(0),
c57f56b7 170 fRunNumbers(),
171 fExecutable(),
0a1c1f7f 172 fExecutableCommand(),
c57f56b7 173 fArguments(),
631c0b05 174 fExecutableArgs(),
c57f56b7 175 fAnalysisMacro(),
176 fAnalysisSource(),
d3b18c4c 177 fValidationScript(),
d5c6455a 178 fAdditionalRootLibs(),
c57f56b7 179 fAdditionalLibs(),
180 fSplitMode(),
181 fAPIVersion(),
182 fROOTVersion(),
183 fAliROOTVersion(),
648174cf 184 fExternalPackages(),
c57f56b7 185 fUser(),
186 fGridWorkingDir(),
187 fGridDataDir(),
188 fDataPattern(),
189 fGridOutputDir(),
190 fOutputArchive(),
191 fOutputFiles(),
192 fInputFormat(),
e7c71df0 193 fDatasetName(),
c57f56b7 194 fJDLName(),
d3b18c4c 195 fTerminateFiles(),
bb885a9e 196 fMergeExcludes(),
37d21c01 197 fRegisterExcludes(),
f965131e 198 fIncludePath(),
bb885a9e 199 fCloseSE(),
0df6ccf2 200 fFriendChainName(),
c6cb3634 201 fJobTag(),
648174cf 202 fOutputSingle(),
5fce53f4 203 fRunPrefix(),
3bdcb562 204 fProofCluster(),
205 fProofDataSet(),
206 fFileForTestMode(),
207 fRootVersionForProof(),
208 fAliRootMode(),
f47d5cb4 209 fMergeDirName(),
4e5c5506 210 fInputFiles(0),
28ce6c9c 211 fPackages(0),
77f34eae 212 fModules(0),
28ce6c9c 213 fProofParam()
c57f56b7 214{
215// Default ctor.
216 SetDefaults();
217}
218
219//______________________________________________________________________________
220AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
221 :AliAnalysisGrid(other),
222 fGridJDL(NULL),
0f389141 223 fMergingJDL(NULL),
c57f56b7 224 fPrice(other.fPrice),
225 fTTL(other.fTTL),
226 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
227 fMaxInitFailed(other.fMaxInitFailed),
228 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 229 fNtestFiles(other.fNtestFiles),
319593fb 230 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 231 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 232 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 233 fNsubmitted(other.fNsubmitted),
a3e84053 234 fProductionMode(other.fProductionMode),
cd11251e 235 fOutputToRunNo(other.fOutputToRunNo),
0f389141 236 fMergeViaJDL(other.fMergeViaJDL),
237 fFastReadOption(other.fFastReadOption),
238 fOverwriteMode(other.fOverwriteMode),
149d288c 239 fNreplicas(other.fNreplicas),
3bdcb562 240 fNproofWorkers(other.fNproofWorkers),
27734f0e 241 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 242 fProofReset(other.fProofReset),
c57f56b7 243 fRunNumbers(other.fRunNumbers),
244 fExecutable(other.fExecutable),
0a1c1f7f 245 fExecutableCommand(other.fExecutableCommand),
c57f56b7 246 fArguments(other.fArguments),
631c0b05 247 fExecutableArgs(other.fExecutableArgs),
c57f56b7 248 fAnalysisMacro(other.fAnalysisMacro),
249 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 250 fValidationScript(other.fValidationScript),
d5c6455a 251 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 252 fAdditionalLibs(other.fAdditionalLibs),
253 fSplitMode(other.fSplitMode),
254 fAPIVersion(other.fAPIVersion),
255 fROOTVersion(other.fROOTVersion),
256 fAliROOTVersion(other.fAliROOTVersion),
648174cf 257 fExternalPackages(other.fExternalPackages),
c57f56b7 258 fUser(other.fUser),
259 fGridWorkingDir(other.fGridWorkingDir),
260 fGridDataDir(other.fGridDataDir),
261 fDataPattern(other.fDataPattern),
262 fGridOutputDir(other.fGridOutputDir),
263 fOutputArchive(other.fOutputArchive),
264 fOutputFiles(other.fOutputFiles),
265 fInputFormat(other.fInputFormat),
e7c71df0 266 fDatasetName(other.fDatasetName),
c57f56b7 267 fJDLName(other.fJDLName),
d3b18c4c 268 fTerminateFiles(other.fTerminateFiles),
bb885a9e 269 fMergeExcludes(other.fMergeExcludes),
37d21c01 270 fRegisterExcludes(other.fRegisterExcludes),
f965131e 271 fIncludePath(other.fIncludePath),
bb885a9e 272 fCloseSE(other.fCloseSE),
0df6ccf2 273 fFriendChainName(other.fFriendChainName),
c6cb3634 274 fJobTag(other.fJobTag),
648174cf 275 fOutputSingle(other.fOutputSingle),
5fce53f4 276 fRunPrefix(other.fRunPrefix),
3bdcb562 277 fProofCluster(other.fProofCluster),
278 fProofDataSet(other.fProofDataSet),
279 fFileForTestMode(other.fFileForTestMode),
280 fRootVersionForProof(other.fRootVersionForProof),
281 fAliRootMode(other.fAliRootMode),
f47d5cb4 282 fMergeDirName(other.fMergeDirName),
4e5c5506 283 fInputFiles(0),
28ce6c9c 284 fPackages(0),
77f34eae 285 fModules(0),
28ce6c9c 286 fProofParam()
c57f56b7 287{
288// Copy ctor.
289 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 290 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 291 fRunRange[0] = other.fRunRange[0];
292 fRunRange[1] = other.fRunRange[1];
c57f56b7 293 if (other.fInputFiles) {
294 fInputFiles = new TObjArray();
295 TIter next(other.fInputFiles);
296 TObject *obj;
297 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
298 fInputFiles->SetOwner();
299 }
4e5c5506 300 if (other.fPackages) {
301 fPackages = new TObjArray();
302 TIter next(other.fPackages);
303 TObject *obj;
304 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
305 fPackages->SetOwner();
306 }
77f34eae 307 if (other.fModules) {
308 fModules = new TObjArray();
309 fModules->SetOwner();
310 TIter next(other.fModules);
311 AliAnalysisTaskCfg *mod, *crt;
312 while ((crt=(AliAnalysisTaskCfg*)next())) {
313 mod = new AliAnalysisTaskCfg(*crt);
314 fModules->Add(mod);
315 }
316 }
c57f56b7 317}
318
319//______________________________________________________________________________
320AliAnalysisAlien::~AliAnalysisAlien()
321{
322// Destructor.
77f34eae 323 delete fGridJDL;
324 delete fMergingJDL;
325 delete fInputFiles;
326 delete fPackages;
327 delete fModules;
28ce6c9c 328 fProofParam.DeleteAll();
c57f56b7 329}
330
331//______________________________________________________________________________
332AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
333{
334// Assignment.
335 if (this != &other) {
336 AliAnalysisGrid::operator=(other);
337 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 338 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 339 fPrice = other.fPrice;
340 fTTL = other.fTTL;
341 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
342 fMaxInitFailed = other.fMaxInitFailed;
343 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 344 fNtestFiles = other.fNtestFiles;
a3e84053 345 fNrunsPerMaster = other.fNrunsPerMaster;
346 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 347 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 348 fNsubmitted = other.fNsubmitted;
349 fProductionMode = other.fProductionMode;
cd11251e 350 fOutputToRunNo = other.fOutputToRunNo;
0f389141 351 fMergeViaJDL = other.fMergeViaJDL;
352 fFastReadOption = other.fFastReadOption;
353 fOverwriteMode = other.fOverwriteMode;
149d288c 354 fNreplicas = other.fNreplicas;
3bdcb562 355 fNproofWorkers = other.fNproofWorkers;
27734f0e 356 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 357 fProofReset = other.fProofReset;
c57f56b7 358 fRunNumbers = other.fRunNumbers;
359 fExecutable = other.fExecutable;
0a1c1f7f 360 fExecutableCommand = other.fExecutableCommand;
c57f56b7 361 fArguments = other.fArguments;
631c0b05 362 fExecutableArgs = other.fExecutableArgs;
c57f56b7 363 fAnalysisMacro = other.fAnalysisMacro;
364 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 365 fValidationScript = other.fValidationScript;
d5c6455a 366 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 367 fAdditionalLibs = other.fAdditionalLibs;
368 fSplitMode = other.fSplitMode;
369 fAPIVersion = other.fAPIVersion;
370 fROOTVersion = other.fROOTVersion;
371 fAliROOTVersion = other.fAliROOTVersion;
648174cf 372 fExternalPackages = other.fExternalPackages;
c57f56b7 373 fUser = other.fUser;
374 fGridWorkingDir = other.fGridWorkingDir;
375 fGridDataDir = other.fGridDataDir;
376 fDataPattern = other.fDataPattern;
377 fGridOutputDir = other.fGridOutputDir;
378 fOutputArchive = other.fOutputArchive;
379 fOutputFiles = other.fOutputFiles;
380 fInputFormat = other.fInputFormat;
e7c71df0 381 fDatasetName = other.fDatasetName;
c57f56b7 382 fJDLName = other.fJDLName;
d3b18c4c 383 fTerminateFiles = other.fTerminateFiles;
bb885a9e 384 fMergeExcludes = other.fMergeExcludes;
37d21c01 385 fRegisterExcludes = other.fRegisterExcludes;
f965131e 386 fIncludePath = other.fIncludePath;
bb885a9e 387 fCloseSE = other.fCloseSE;
0df6ccf2 388 fFriendChainName = other.fFriendChainName;
c6cb3634 389 fJobTag = other.fJobTag;
648174cf 390 fOutputSingle = other.fOutputSingle;
5fce53f4 391 fRunPrefix = other.fRunPrefix;
3bdcb562 392 fProofCluster = other.fProofCluster;
393 fProofDataSet = other.fProofDataSet;
394 fFileForTestMode = other.fFileForTestMode;
395 fRootVersionForProof = other.fRootVersionForProof;
396 fAliRootMode = other.fAliRootMode;
f47d5cb4 397 fMergeDirName = other.fMergeDirName;
c57f56b7 398 if (other.fInputFiles) {
399 fInputFiles = new TObjArray();
400 TIter next(other.fInputFiles);
401 TObject *obj;
402 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
403 fInputFiles->SetOwner();
404 }
4e5c5506 405 if (other.fPackages) {
406 fPackages = new TObjArray();
407 TIter next(other.fPackages);
408 TObject *obj;
409 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
410 fPackages->SetOwner();
411 }
77f34eae 412 if (other.fModules) {
413 fModules = new TObjArray();
414 fModules->SetOwner();
415 TIter next(other.fModules);
416 AliAnalysisTaskCfg *mod, *crt;
417 while ((crt=(AliAnalysisTaskCfg*)next())) {
418 mod = new AliAnalysisTaskCfg(*crt);
419 fModules->Add(mod);
420 }
421 }
c57f56b7 422 }
423 return *this;
424}
425
37d21c01 426//______________________________________________________________________________
427void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
428{
429// Add a single additional library to be loaded. Extension must be present.
430 TString lib(name);
431 if (!lib.Contains(".")) {
432 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
433 return;
434 }
435 if (fAdditionalLibs.Contains(name)) {
436 Warning("AddAdditionalLibrary", "Library %s already added.", name);
437 return;
438 }
439 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
440 fAdditionalLibs += lib;
441}
442
77f34eae 443//______________________________________________________________________________
444void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
445{
446// Adding a module. Checks if already existing. Becomes owned by this.
447 if (!module) return;
448 if (GetModule(module->GetName())) {
449 Error("AddModule", "A module having the same name %s already added", module->GetName());
450 return;
451 }
452 if (!fModules) {
453 fModules = new TObjArray();
454 fModules->SetOwner();
455 }
456 fModules->Add(module);
457}
458
459//______________________________________________________________________________
460void AliAnalysisAlien::AddModules(TObjArray *list)
461{
462// Adding a list of modules. Checks if already existing. Becomes owned by this.
463 TIter next(list);
464 AliAnalysisTaskCfg *module;
465 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
466}
467
468//______________________________________________________________________________
469Bool_t AliAnalysisAlien::CheckDependencies()
470{
471// Check if all dependencies are satisfied. Reorder modules if needed.
472 Int_t nmodules = GetNmodules();
473 if (!nmodules) {
474 Warning("CheckDependencies", "No modules added yet to check their dependencies");
475 return kTRUE;
476 }
477 AliAnalysisTaskCfg *mod = 0;
478 AliAnalysisTaskCfg *dep = 0;
479 TString depname;
480 Int_t i, j, k;
481 for (i=0; i<nmodules; i++) {
482 mod = (AliAnalysisTaskCfg*) fModules->At(i);
483 Int_t ndeps = mod->GetNdeps();
484 Int_t istart = i;
485 for (j=0; j<ndeps; j++) {
486 depname = mod->GetDependency(j);
487 dep = GetModule(depname);
488 if (!dep) {
489 Error("CheckDependencies","Dependency %s not added for module %s",
490 depname.Data(), mod->GetName());
491 return kFALSE;
492 }
493 if (dep->NeedsDependency(mod->GetName())) {
494 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
495 mod->GetName(), dep->GetName());
496 return kFALSE;
497 }
498 Int_t idep = fModules->IndexOf(dep);
499 // The dependency task must come first
500 if (idep>i) {
501 // Remove at idep and move all objects below up one slot
502 // down to index i included.
503 fModules->RemoveAt(idep);
3e40fd4c 504 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 505 fModules->AddAt(dep, i++);
506 }
507 //Redo from istart if dependencies were inserted
508 if (i>istart) i=istart-1;
509 }
510 }
511 return kTRUE;
512}
513
3e40fd4c 514//______________________________________________________________________________
515AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
516{
517// Create the analysis manager and optionally execute the macro in filename.
518 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
519 if (mgr) return mgr;
520 mgr = new AliAnalysisManager(name);
521 mgr->SetGridHandler((AliAnalysisGrid*)this);
522 if (strlen(filename)) {
523 TString line = gSystem->ExpandPathName(filename);
524 line.Prepend(".x ");
525 gROOT->ProcessLine(line.Data());
526 }
527 return mgr;
528}
529
77f34eae 530//______________________________________________________________________________
531Int_t AliAnalysisAlien::GetNmodules() const
532{
533// Get number of modules.
534 if (!fModules) return 0;
535 return fModules->GetEntries();
536}
537
538//______________________________________________________________________________
539AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
540{
541// Get a module by name.
542 if (!fModules) return 0;
543 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
544}
545
546//______________________________________________________________________________
547Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
548{
549// Load a given module.
550 if (mod->IsLoaded()) return kTRUE;
192ddca2 551 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
552 if (!mgr) {
553 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
554 return kFALSE;
555 }
77f34eae 556 Int_t ndeps = mod->GetNdeps();
557 TString depname;
558 for (Int_t j=0; j<ndeps; j++) {
559 depname = mod->GetDependency(j);
560 AliAnalysisTaskCfg *dep = GetModule(depname);
561 if (!dep) {
562 Error("LoadModule","Dependency %s not existing for module %s",
563 depname.Data(), mod->GetName());
564 return kFALSE;
565 }
566 if (!LoadModule(dep)) {
567 Error("LoadModule","Dependency %s for module %s could not be loaded",
568 depname.Data(), mod->GetName());
569 return kFALSE;
570 }
571 }
572 // Load libraries for the module
573 if (!mod->CheckLoadLibraries()) {
574 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
575 return kFALSE;
576 }
192ddca2 577 // Check if a custom file name was requested
578 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
579
580 // Check if a custom terminate file name was requested
581 if (strlen(mod->GetTerminateFileName())) {
582 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
583 fTerminateFiles += mod->GetTerminateFileName();
584 }
585
77f34eae 586 // Execute the macro
587 if (mod->ExecuteMacro()<0) {
588 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
589 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
590 return kFALSE;
591 }
592 // Configure dependencies
593 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
594 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
595 mod->GetConfigMacro()->GetTitle(), mod->GetName());
596 return kFALSE;
597 }
5e8c7c22 598 // Adjust extra libraries
599 Int_t nlibs = mod->GetNlibs();
600 TString lib;
601 for (Int_t i=0; i<nlibs; i++) {
602 lib = mod->GetLibrary(i);
5e8c7c22 603 lib = Form("lib%s.so", lib.Data());
c9d6bd12 604 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 605 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
606 fAdditionalLibs += lib;
607 }
77f34eae 608 return kTRUE;
609}
610
139fe6ba 611//______________________________________________________________________________
612Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
613{
614// Generate the full train.
615 fAdditionalLibs = "";
616 if (!LoadModules()) return kFALSE;
617 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
618 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 619 mgr->RunLocalInit();
139fe6ba 620 mgr->PrintStatus();
621 Int_t productionMode = fProductionMode;
622 SetProductionMode();
623 TString macro = fAnalysisMacro;
624 TString executable = fExecutable;
625 TString validation = fValidationScript;
626 TString execCommand = fExecutableCommand;
627 SetAnalysisMacro(Form("%s.C", name));
628 SetExecutable(Form("%s.sh", name));
a8e31fb0 629// SetExecutableCommand("aliroot -b -q ");
139fe6ba 630 SetValidationScript(Form("%s_validation.sh", name));
631 StartAnalysis();
632 SetProductionMode(productionMode);
633 fAnalysisMacro = macro;
634 fExecutable = executable;
635 fExecutableCommand = execCommand;
636 fValidationScript = validation;
637 return kTRUE;
638}
639
77f34eae 640//______________________________________________________________________________
21ca8e59 641Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 642{
643// Generate test macros for a single module or for the full train.
5e8c7c22 644 fAdditionalLibs = "";
77f34eae 645 if (strlen(modname)) {
21ca8e59 646 if (!CheckDependencies()) return kFALSE;
647 AliAnalysisTaskCfg *mod = GetModule(modname);
648 if (!mod) {
649 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
650 return kFALSE;
651 }
652 if (!LoadModule(mod)) return kFALSE;
653 } else if (!LoadModules()) return kFALSE;
654 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
655 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 656 mgr->RunLocalInit();
21ca8e59 657 mgr->PrintStatus();
658 SetLocalTest(kTRUE);
659 Int_t productionMode = fProductionMode;
660 SetProductionMode();
661 TString macro = fAnalysisMacro;
662 TString executable = fExecutable;
663 TString validation = fValidationScript;
664 TString execCommand = fExecutableCommand;
665 SetAnalysisMacro(Form("%s.C", name));
666 SetExecutable(Form("%s.sh", name));
a8e31fb0 667// SetExecutableCommand("aliroot -b -q ");
21ca8e59 668 SetValidationScript(Form("%s_validation.sh", name));
669 WriteAnalysisFile();
670 WriteAnalysisMacro();
671 WriteExecutable();
672 WriteValidationScript();
673 SetLocalTest(kFALSE);
674 SetProductionMode(productionMode);
675 fAnalysisMacro = macro;
676 fExecutable = executable;
677 fExecutableCommand = execCommand;
678 fValidationScript = validation;
77f34eae 679 return kTRUE;
680}
681
682//______________________________________________________________________________
683Bool_t AliAnalysisAlien::LoadModules()
684{
685// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 686 fAdditionalLibs = "";
21ca8e59 687 Int_t nmodules = GetNmodules();
688 if (!nmodules) {
689 Warning("LoadModules", "No module to be loaded");
690 return kTRUE;
691 }
3e40fd4c 692 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
693 if (!mgr) {
694 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
695 return kFALSE;
696 }
77f34eae 697 if (!CheckDependencies()) return kFALSE;
21ca8e59 698 nmodules = GetNmodules();
77f34eae 699 AliAnalysisTaskCfg *mod;
21ca8e59 700 for (Int_t imod=0; imod<nmodules; imod++) {
701 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 702 if (!LoadModule(mod)) return kFALSE;
703 }
77f34eae 704 return kTRUE;
705}
706
74013535 707//______________________________________________________________________________
708void AliAnalysisAlien::SetRunPrefix(const char *prefix)
709{
710// Set the run number format. Can be a prefix or a format like "%09d"
711 fRunPrefix = prefix;
712 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
713}
714
f965131e 715//______________________________________________________________________________
716void AliAnalysisAlien::AddIncludePath(const char *path)
717{
718// Add include path in the remote analysis macro.
719 TString p(path);
720 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
721 else fIncludePath += Form("-I%s ", path);
722}
723
c57f56b7 724//______________________________________________________________________________
725void AliAnalysisAlien::AddRunNumber(Int_t run)
726{
727// Add a run number to the list of runs to be processed.
728 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 729 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 730}
731
98690895 732//______________________________________________________________________________
733void AliAnalysisAlien::AddRunList(const char* runList)
734{
735// Add several runs into the list of runs; they are expected to be separated by a blank character.
736 TString sList = runList;
737 TObjArray *list = sList.Tokenize(" ");
738 Int_t n = list->GetEntries();
739 for (Int_t i = 0; i < n; i++) {
740 TObjString *os = (TObjString*)list->At(i);
741 AddRunNumber(os->GetString().Atoi());
742 }
743 delete list;
744}
745
ee75cfc3 746//______________________________________________________________________________
747void AliAnalysisAlien::AddRunNumber(const char* run)
748{
749// Add a run number to the list of runs to be processed.
7aa5d539 750 TString runs = run;
751 TObjString *os;
752 TObjArray *arr = runs.Tokenize(" ");
753 TIter next(arr);
754 TString prefix;
755 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
756 while ((os=(TObjString*)next())){
757 if (fRunNumbers.Length()) fRunNumbers += " ";
758 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
759 }
760 delete arr;
ee75cfc3 761}
762
c57f56b7 763//______________________________________________________________________________
764void AliAnalysisAlien::AddDataFile(const char *lfn)
765{
766// Adds a data file to the input to be analysed. The file should be a valid LFN
767// or point to an existing file in the alien workdir.
768 if (!fInputFiles) fInputFiles = new TObjArray();
769 fInputFiles->Add(new TObjString(lfn));
770}
648174cf 771
772//______________________________________________________________________________
773void AliAnalysisAlien::AddExternalPackage(const char *package)
774{
775// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
776 if (fExternalPackages) fExternalPackages += " ";
777 fExternalPackages += package;
778}
779
c57f56b7 780//______________________________________________________________________________
781Bool_t AliAnalysisAlien::Connect()
782{
783// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
784 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 785 if (fProductionMode) return kTRUE;
c57f56b7 786 if (!gGrid) {
787 Info("Connect", "Trying to connect to AliEn ...");
788 TGrid::Connect("alien://");
789 }
790 if (!gGrid || !gGrid->IsConnected()) {
791 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
792 return kFALSE;
793 }
794 fUser = gGrid->GetUser();
795 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
796 return kTRUE;
797}
798
799//______________________________________________________________________________
800void AliAnalysisAlien::CdWork()
801{
802// Check validity of alien workspace. Create directory if possible.
803 if (!Connect()) {
804 Error("CdWork", "Alien connection required");
805 return;
806 }
807 TString homedir = gGrid->GetHomeDirectory();
808 TString workdir = homedir + fGridWorkingDir;
923e2ca5 809 if (DirectoryExists(workdir)) {
810 gGrid->Cd(workdir);
811 return;
812 }
813 // Work directory not existing - create it
814 gGrid->Cd(homedir);
b93f8109 815 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 816 gGrid->Cd(fGridWorkingDir);
d3b18c4c 817 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 818 } else {
d3b18c4c 819 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 820 workdir.Data(), homedir.Data());
821 fGridWorkingDir = "";
822 }
c57f56b7 823}
824
348be253 825//______________________________________________________________________________
826Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
827{
828// Check if file copying is possible.
d3b18c4c 829 if (fProductionMode) return kTRUE;
b5ae4fe0 830 TString salienpath(alienpath);
831 if (salienpath.Contains(" ")) {
832 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
833 return kFALSE;
834 }
348be253 835 if (!Connect()) {
836 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
837 return kFALSE;
838 }
d3339be3 839 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
840 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 841 // Check if alien_CLOSE_SE is defined
842 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
843 if (!closeSE.IsNull()) {
844 Info("CheckFileCopy", "Your current close storage is pointing to: \
845 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
846 } else {
847 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
848 }
849 // Check if grid directory exists.
850 if (!DirectoryExists(alienpath)) {
851 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
852 return kFALSE;
853 }
b5ae4fe0 854 TString stest = "plugin_test_copy";
855 TFile f(stest, "RECREATE");
348be253 856 // User may not have write permissions to current directory
857 if (f.IsZombie()) {
858 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
859 gSystem->WorkingDirectory());
860 return kFALSE;
861 }
862 f.Close();
b5ae4fe0 863 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
864 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 865 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 866 \n# 1. Make sure you have write permissions there. If this is the case: \
867 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
868 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
869 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
870 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 871 gSystem->Unlink(stest.Data());
348be253 872 return kFALSE;
873 }
b5ae4fe0 874 gSystem->Unlink(stest.Data());
875 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 876 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 877 return kTRUE;
878}
879
c57f56b7 880//______________________________________________________________________________
881Bool_t AliAnalysisAlien::CheckInputData()
882{
883// Check validity of input data. If necessary, create xml files.
d3b18c4c 884 if (fProductionMode) return kTRUE;
a8739e8a 885 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 886 if (!fGridDataDir.Length()) {
887 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
888 return kFALSE;
889 }
9f020952 890 if (fMergeViaJDL) {
891 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
892 return kFALSE;
893 }
d2a409b2 894 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 895 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
896 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 897 return kTRUE;
c57f56b7 898 }
899 // Process declared files
a2f5fc01 900 Bool_t isCollection = kFALSE;
901 Bool_t isXml = kFALSE;
902 Bool_t useTags = kFALSE;
c57f56b7 903 Bool_t checked = kFALSE;
d3b18c4c 904 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 905 TString file;
906 TString workdir = gGrid->GetHomeDirectory();
907 workdir += fGridWorkingDir;
908 if (fInputFiles) {
909 TObjString *objstr;
910 TIter next(fInputFiles);
911 while ((objstr=(TObjString*)next())) {
912 file = workdir;
913 file += "/";
914 file += objstr->GetString();
915 // Store full lfn path
916 if (FileExists(file)) objstr->SetString(file);
917 else {
918 file = objstr->GetName();
919 if (!FileExists(objstr->GetName())) {
920 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
921 objstr->GetName(), workdir.Data());
922 return kFALSE;
923 }
924 }
925 Bool_t iscoll, isxml, usetags;
926 CheckDataType(file, iscoll, isxml, usetags);
927 if (!checked) {
928 checked = kTRUE;
a2f5fc01 929 isCollection = iscoll;
930 isXml = isxml;
931 useTags = usetags;
932 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 933 } else {
a2f5fc01 934 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 935 Error("CheckInputData", "Some conflict was found in the types of inputs");
936 return kFALSE;
937 }
938 }
939 }
940 }
941 // Process requested run numbers
a8739e8a 942 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 943 // Check validity of alien data directory
944 if (!fGridDataDir.Length()) {
945 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
946 return kFALSE;
947 }
923e2ca5 948 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 949 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
950 return kFALSE;
951 }
a2f5fc01 952 if (isCollection) {
c57f56b7 953 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
954 return kFALSE;
955 }
956
a2f5fc01 957 if (checked && !isXml) {
c57f56b7 958 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
959 return kFALSE;
960 }
961 // Check validity of run number(s)
962 TObjArray *arr;
963 TObjString *os;
74013535 964 TString format;
319593fb 965 Int_t nruns = 0;
904f9f5f 966 TString schunk, schunk2;
c57f56b7 967 TString path;
968 if (!checked) {
969 checked = kTRUE;
a2f5fc01 970 useTags = fDataPattern.Contains("tag");
971 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 972 }
a2f5fc01 973 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 974 Error("CheckInputData", "Cannot mix input files using/not using tags");
975 return kFALSE;
976 }
977 if (fRunNumbers.Length()) {
a8739e8a 978 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 979 arr = fRunNumbers.Tokenize(" ");
980 TIter next(arr);
981 while ((os=(TObjString*)next())) {
982 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 983 if (!DirectoryExists(path)) {
984 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 985 continue;
c57f56b7 986 }
987 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
988 TString msg = "\n##### file: ";
989 msg += path;
990 msg += " type: xml_collection;";
a2f5fc01 991 if (useTags) msg += " using_tags: Yes";
c57f56b7 992 else msg += " using_tags: No";
23329835 993 Info("CheckDataType", "%s", msg.Data());
319593fb 994 if (fNrunsPerMaster<2) {
d2a409b2 995 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 996 } else {
997 nruns++;
998 if (((nruns-1)%fNrunsPerMaster) == 0) {
999 schunk = os->GetString();
1000 }
1001 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1002 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1003 AddDataFile(schunk);
319593fb 1004 }
c57f56b7 1005 }
1006 delete arr;
a8739e8a 1007 } else {
1008 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1009 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1010 format = Form("%%s/%s ", fRunPrefix.Data());
1011 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1012 if (!DirectoryExists(path)) {
a8739e8a 1013 continue;
1014 }
74013535 1015 format = Form("%%s/%s.xml", fRunPrefix.Data());
1016 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1017 TString msg = "\n##### file: ";
1018 msg += path;
1019 msg += " type: xml_collection;";
a2f5fc01 1020 if (useTags) msg += " using_tags: Yes";
a8739e8a 1021 else msg += " using_tags: No";
23329835 1022 Info("CheckDataType", "%s", msg.Data());
319593fb 1023 if (fNrunsPerMaster<2) {
74013535 1024 format = Form("%s.xml", fRunPrefix.Data());
1025 AddDataFile(Form(format.Data(),irun));
319593fb 1026 } else {
1027 nruns++;
1028 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1029 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1030 }
74013535 1031 format = Form("_%s.xml", fRunPrefix.Data());
1032 schunk2 = Form(format.Data(), irun);
319593fb 1033 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1034 schunk += schunk2;
d2a409b2 1035 AddDataFile(schunk);
319593fb 1036 }
a8739e8a 1037 }
904f9f5f 1038 if (!fInputFiles) {
1039 schunk += schunk2;
1040 AddDataFile(schunk);
1041 }
c57f56b7 1042 }
1043 return kTRUE;
1044}
1045
21ca8e59 1046//______________________________________________________________________________
43b807f6 1047Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1048{
1049// Copy data from the given grid directory according a pattern and make a local
1050// dataset.
2ba4c287 1051// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1052 if (!Connect()) {
1053 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
1054 return kFALSE;
1055 }
1056 if (!DirectoryExists(griddir)) {
1057 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
1058 return kFALSE;
1059 }
1060 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1061 printf("Running command: %s\n", command.Data());
1062 TGridResult *res = gGrid->Command(command);
1063 Int_t nfound = res->GetEntries();
1064 if (!nfound) {
1065 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
1066 return kFALSE;
1067 }
1068 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1069
1070 // archives
1071 TObjArray* additionalArchives = 0;
1072 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1073 additionalArchives = TString(archivefile).Tokenize(";");
1074 archivefile = additionalArchives->At(0)->GetName();
1075 additionalArchives->RemoveAt(0);
1076 additionalArchives->Compress();
1077 }
1078
21ca8e59 1079 // Copy files locally
1080 ofstream out;
1081 out.open(output, ios::out);
1082 TMap *map;
1083 TString turl, dirname, filename, temp;
1084 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1085 gSystem->MakeDirectory(outputdir);
1086 gSystem->ChangeDirectory(outputdir);
21ca8e59 1087 for (Int_t i=0; i<nfound; i++) {
1088 map = (TMap*)res->At(i);
1089 turl = map->GetValue("turl")->GetName();
1090 filename = gSystem->BaseName(turl.Data());
1091 dirname = gSystem->DirName(turl.Data());
1092 dirname = gSystem->BaseName(dirname.Data());
1093 gSystem->MakeDirectory(dirname);
43b807f6 1094
1095 TString source(turl);
1096 TString targetFileName(filename);
1097
1098 if (strlen(archivefile) > 0) {
1099 // TODO here the archive in which the file resides should be determined
1100 // however whereis returns only a guid, and guid2lfn does not work
1101 // Therefore we use the one provided as argument for now
1102 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1103 targetFileName = archivefile;
1104 }
1105 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
2ba4c287 1106 Bool_t success = kTRUE;
1107 if (additionalArchives)
1108 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++)
1109 {
1110 TString target;
1111 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1112 gSystem->MakeDirectory(gSystem->DirName(target));
1113 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1114 }
1115
1116 if (success) {
1117 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1118 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1119 }
21ca8e59 1120 }
1121 }
1122 gSystem->ChangeDirectory(cdir);
1123 delete res;
2ba4c287 1124 if (additionalArchives)
1125 delete additionalArchives;
21ca8e59 1126 return kTRUE;
1127}
1128
c57f56b7 1129//______________________________________________________________________________
1130Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1131{
1132// Create dataset for the grid data directory + run number.
3af861ff 1133 const Int_t gMaxEntries = 15000;
d3b18c4c 1134 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1135 if (!Connect()) {
1136 Error("CreateDataset", "Cannot create dataset with no grid connection");
1137 return kFALSE;
1138 }
1139
1140 // Cd workspace
d3b18c4c 1141 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1142 TString workdir = gGrid->GetHomeDirectory();
1143 workdir += fGridWorkingDir;
1144
1145 // Compose the 'find' command arguments
74013535 1146 TString format;
c57f56b7 1147 TString command;
1148 TString options = "-x collection ";
bb885a9e 1149 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1150 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1151 TString conditions = "";
3af861ff 1152 Int_t nstart = 0;
1153 Int_t ncount = 0;
1154 Int_t stage = 0;
c57f56b7 1155 TString file;
1156 TString path;
319593fb 1157 Int_t nruns = 0;
904f9f5f 1158 TString schunk, schunk2;
ab254fd1 1159 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1160 if (!fRunNumbers.Length() && !fRunRange[0]) {
1161 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1162 // Make a single data collection from data directory.
1163 path = fGridDataDir;
923e2ca5 1164 if (!DirectoryExists(path)) {
d2a409b2 1165 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1166 return kFALSE;
1167 }
923e2ca5 1168// CdWork();
d2a409b2 1169 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1170 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1171 while (1) {
1172 ncount = 0;
1173 stage++;
1174 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1175 command = "find ";
1176 command += Form("%s -o %d ",options.Data(), nstart);
1177 command += path;
1178 command += " ";
1179 command += pattern;
1180 command += conditions;
1181 printf("command: %s\n", command.Data());
1182 TGridResult *res = gGrid->Command(command);
1183 if (res) delete res;
1184 // Write standard output to file
1185 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1186 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1187 Bool_t nullFile = kFALSE;
1188 if (!hasGrep) {
1189 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1190 } else {
1191 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1192 if (nullFile) {
1193 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1194 gSystem->Exec("rm -f __tmp*");
1195 return kFALSE;
1196 }
1197 TString line;
1198 ifstream in;
1199 in.open("__tmp__");
1200 in >> line;
1201 in.close();
1202 gSystem->Exec("rm -f __tmp__");
1203 ncount = line.Atoi();
1204 }
1205 }
1206 if (ncount == gMaxEntries) {
1207 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1208 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1209 if (!cbase) cbase = cadd;
1210 else {
1211 cbase->Add(cadd);
1212 delete cadd;
defd7a3a 1213 }
3af861ff 1214 nstart += ncount;
1215 } else {
1216 if (cbase) {
1217 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1218 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1219 cbase->Add(cadd);
1220 delete cadd;
1221 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1222 delete cbase; cbase = 0;
1223 } else {
1224 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1225 }
1226 gSystem->Exec("rm -f __tmp*");
1227 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1228 break;
1229 }
0f389141 1230 }
1231 Bool_t fileExists = FileExists(file);
1232 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1233 // Copy xml file to alien space
0f389141 1234 if (fileExists) gGrid->Rm(file);
d2a409b2 1235 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1236 if (!FileExists(file)) {
1237 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1238 return kFALSE;
1239 }
1240 // Update list of files to be processed.
1241 }
1242 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1243 return kTRUE;
1244 }
c57f56b7 1245 // Several runs
a2f5fc01 1246 Bool_t nullResult = kTRUE;
a8739e8a 1247 if (fRunNumbers.Length()) {
1248 TObjArray *arr = fRunNumbers.Tokenize(" ");
1249 TObjString *os;
1250 TIter next(arr);
1251 while ((os=(TObjString*)next())) {
3af861ff 1252 nstart = 0;
1253 stage = 0;
dc030a24 1254 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1255 if (!DirectoryExists(path)) continue;
1256// CdWork();
a8739e8a 1257 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1258 else file = Form("%s.xml", os->GetString().Data());
319593fb 1259 // If local collection file does not exist, create it via 'find' command.
3af861ff 1260 while (1) {
1261 ncount = 0;
1262 stage++;
1263 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1264 command = "find ";
1265 command += Form("%s -o %d ",options.Data(), nstart);
1266 command += path;
1267 command += pattern;
1268 command += conditions;
1269 TGridResult *res = gGrid->Command(command);
1270 if (res) delete res;
1271 // Write standard output to file
1272 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1273 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1274 Bool_t nullFile = kFALSE;
1275 if (!hasGrep) {
1276 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1277 } else {
1278 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1279 if (nullFile) {
1280 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1281 gSystem->Exec("rm -f __tmp*");
1282 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1283 break;
3af861ff 1284 }
1285 TString line;
1286 ifstream in;
1287 in.open("__tmp__");
1288 in >> line;
1289 in.close();
1290 gSystem->Exec("rm -f __tmp__");
1291 ncount = line.Atoi();
1292 }
1293 nullResult = kFALSE;
1294 }
1295 if (ncount == gMaxEntries) {
1296 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1297 if (fNrunsPerMaster > 1) {
1298 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1299 file.Data(),gMaxEntries);
1300 return kFALSE;
1301 }
1302 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1303 if (!cbase) cbase = cadd;
1304 else {
1305 cbase->Add(cadd);
1306 delete cadd;
defd7a3a 1307 }
3af861ff 1308 nstart += ncount;
1309 } else {
c89a84bc 1310 if (cbase && fNrunsPerMaster<2) {
3af861ff 1311 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1312 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1313 cbase->Add(cadd);
1314 delete cadd;
1315 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1316 delete cbase; cbase = 0;
1317 } else {
1318 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1319 }
1320 gSystem->Exec("rm -f __tmp*");
1321 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1322 break;
d3339be3 1323 }
3af861ff 1324 }
a8739e8a 1325 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1326 // Check if there is one run per master job.
1327 if (fNrunsPerMaster<2) {
1328 if (FileExists(file)) {
0f389141 1329 if (fOverwriteMode) gGrid->Rm(file);
1330 else {
1331 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1332 continue;
1333 }
319593fb 1334 }
1335 // Copy xml file to alien space
1336 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1337 if (!FileExists(file)) {
1338 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1339 delete arr;
1340 return kFALSE;
1341 }
1342 } else {
1343 nruns++;
1344 if (((nruns-1)%fNrunsPerMaster) == 0) {
1345 schunk = os->GetString();
1346 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1347 } else {
1348 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1349 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1350 cbase->Add(cadd);
1351 delete cadd;
1352 }
1353 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1354 continue;
1355 }
1356 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1357 if (FileExists(schunk)) {
1358 if (fOverwriteMode) gGrid->Rm(file);
1359 else {
1360 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1361 continue;
1362 }
319593fb 1363 }
84fcd93f 1364 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1365 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1366 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1367 if (!FileExists(schunk)) {
1368 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1369 delete arr;
1370 return kFALSE;
1371 }
d3339be3 1372 }
a8739e8a 1373 }
1374 delete arr;
a2f5fc01 1375 if (nullResult) {
d3339be3 1376 Error("CreateDataset", "No valid dataset corresponding to the query!");
1377 return kFALSE;
1378 }
a8739e8a 1379 } else {
1380 // Process a full run range.
1381 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1382 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1383 nstart = 0;
1384 stage = 0;
74013535 1385 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1386 if (!DirectoryExists(path)) continue;
1387// CdWork();
74013535 1388 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1389 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1390 else file = Form(format.Data(), irun);
0f389141 1391 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1392 if (fOverwriteMode) gGrid->Rm(file);
1393 else {
1394 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1395 continue;
1396 }
a8739e8a 1397 }
319593fb 1398 // If local collection file does not exist, create it via 'find' command.
3af861ff 1399 while (1) {
1400 ncount = 0;
1401 stage++;
1402 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1403 command = "find ";
1404 command += Form("%s -o %d ",options.Data(), nstart);
1405 command += path;
1406 command += pattern;
1407 command += conditions;
1408 TGridResult *res = gGrid->Command(command);
1409 if (res) delete res;
1410 // Write standard output to file
1411 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1412 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1413 Bool_t nullFile = kFALSE;
1414 if (!hasGrep) {
1415 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1416 } else {
1417 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1418 if (nullFile) {
1419 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1420 gSystem->Exec("rm -f __tmp*");
1604c9ed 1421 break;
3af861ff 1422 }
1423 TString line;
1424 ifstream in;
1425 in.open("__tmp__");
1426 in >> line;
1427 in.close();
1428 gSystem->Exec("rm -f __tmp__");
1429 ncount = line.Atoi();
1430 }
1431 nullResult = kFALSE;
1432 }
1433 if (ncount == gMaxEntries) {
1434 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1435 if (fNrunsPerMaster > 1) {
1436 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1437 file.Data(),gMaxEntries);
1438 return kFALSE;
1439 }
1440 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1441 if (!cbase) cbase = cadd;
1442 else {
1443 cbase->Add(cadd);
1444 delete cadd;
defd7a3a 1445 }
3af861ff 1446 nstart += ncount;
1447 } else {
c89a84bc 1448 if (cbase && fNrunsPerMaster<2) {
3af861ff 1449 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1450 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1451 cbase->Add(cadd);
1452 delete cadd;
1453 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1454 delete cbase; cbase = 0;
1455 } else {
1456 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1457 }
1458 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1459 break;
d3339be3 1460 }
319593fb 1461 }
a8739e8a 1462 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1463 // Check if there is one run per master job.
1464 if (fNrunsPerMaster<2) {
1465 if (FileExists(file)) {
0f389141 1466 if (fOverwriteMode) gGrid->Rm(file);
1467 else {
1468 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1469 continue;
1470 }
319593fb 1471 }
1472 // Copy xml file to alien space
1473 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1474 if (!FileExists(file)) {
1475 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1476 return kFALSE;
1477 }
1478 } else {
1479 nruns++;
95e5b448 1480 // Check if the collection for the chunk exist locally.
1481 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1482 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1483 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1484 else continue;
1485 }
84fcd93f 1486 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1487 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1488 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1489 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1490 } else {
1491 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1492 cbase->Add(cadd);
1493 delete cadd;
1494 }
74013535 1495 format = Form("%%s_%s.xml", fRunPrefix.Data());
1496 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1497 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1498 continue;
1499 }
904f9f5f 1500 schunk = schunk2;
319593fb 1501 if (FileExists(schunk)) {
0f389141 1502 if (fOverwriteMode) gGrid->Rm(schunk);
1503 else {
1504 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1505 continue;
1506 }
319593fb 1507 }
84fcd93f 1508 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1509 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1510 if (FileExists(schunk)) {
0f389141 1511 if (fOverwriteMode) gGrid->Rm(schunk);
1512 else {
1513 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1514 continue;
1515 }
95e5b448 1516 }
319593fb 1517 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1518 if (!FileExists(schunk)) {
1519 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1520 return kFALSE;
1521 }
1522 }
c57f56b7 1523 }
a2f5fc01 1524 if (nullResult) {
d3339be3 1525 Error("CreateDataset", "No valid dataset corresponding to the query!");
1526 return kFALSE;
1527 }
a8739e8a 1528 }
c57f56b7 1529 return kTRUE;
1530}
1531
1532//______________________________________________________________________________
1533Bool_t AliAnalysisAlien::CreateJDL()
1534{
1535// Generate a JDL file according to current settings. The name of the file is
1536// specified by fJDLName.
1537 Bool_t error = kFALSE;
1538 TObjArray *arr = 0;
1539 Bool_t copy = kTRUE;
d3b18c4c 1540 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1541 Bool_t generate = kTRUE;
1542 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1543 if (!Connect()) {
1544 Error("CreateJDL", "Alien connection required");
1545 return kFALSE;
1546 }
1547 // Check validity of alien workspace
d3b18c4c 1548 TString workdir;
1549 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1550 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1551 workdir += fGridWorkingDir;
1552 if (generate) {
1553 TObjString *os;
1554 if (!fInputFiles) {
1555 Error("CreateJDL()", "Define some input files for your analysis.");
1556 error = kTRUE;
1557 }
1558 // Compose list of input files
1559 // Check if output files were defined
1560 if (!fOutputFiles.Length()) {
1561 Error("CreateJDL", "You must define at least one output file");
1562 error = kTRUE;
1563 }
1564 // Check if an output directory was defined and valid
1565 if (!fGridOutputDir.Length()) {
1566 Error("CreateJDL", "You must define AliEn output directory");
1567 error = kTRUE;
1568 } else {
d3b18c4c 1569 if (!fProductionMode) {
1570 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1571 if (!DirectoryExists(fGridOutputDir)) {
1572 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1573 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1574 } else {
1575 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1576 // error = kTRUE;
1577 }
67a3b254 1578 } else {
1579 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1580 }
1581 gGrid->Cd(workdir);
1582 }
c57f56b7 1583 }
1584 // Exit if any error up to now
1585 if (error) return kFALSE;
1586 // Set JDL fields
0f389141 1587 if (!fUser.IsNull()) {
1588 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1589 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1590 }
1591 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1592 TString mergeExec = fExecutable;
1593 mergeExec.ReplaceAll(".sh", "_merge.sh");
1594 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1595 mergeExec.ReplaceAll(".sh", ".C");
1596 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1597 if (!fArguments.IsNull())
1598 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1599 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1600 else {
1601 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1602 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1603 }
b3e07543 1604
1f0d1ca2 1605 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1606 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1607 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1608 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1609
0f389141 1610 if (fMaxInitFailed > 0) {
c57f56b7 1611 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1612 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1613 }
1614 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1615 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1616 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1617 }
1618 if (!IsOneStageMerging()) {
f47d5cb4 1619 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1620 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1621 }
1622 if (fSplitMode.Length()) {
c57f56b7 1623 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1624 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1625 }
1626 fMergingJDL->SetValue("Split", "\"se\"");
1627 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1628 if (!fAliROOTVersion.IsNull()) {
1629 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1630 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1631 }
1632 if (!fROOTVersion.IsNull()) {
c57f56b7 1633 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1634 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1635 }
1636 if (!fAPIVersion.IsNull()) {
c57f56b7 1637 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1638 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1639 }
648174cf 1640 if (!fExternalPackages.IsNull()) {
1641 arr = fExternalPackages.Tokenize(" ");
1642 TIter next(arr);
1643 while ((os=(TObjString*)next())) {
1644 TString pkgname = os->GetString();
1645 Int_t index = pkgname.Index("::");
1646 TString pkgversion = pkgname(index+2, pkgname.Length());
1647 pkgname.Remove(index);
1648 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1649 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1650 }
1651 delete arr;
1652 }
0f389141 1653 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1654 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1655 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1656 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1657 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1658 TString analysisFile = fExecutable;
1659 analysisFile.ReplaceAll(".sh", ".root");
1660 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1661 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1662 if (fAdditionalLibs.Length()) {
1663 arr = fAdditionalLibs.Tokenize(" ");
1664 TIter next(arr);
1665 while ((os=(TObjString*)next())) {
1666 if (os->GetString().Contains(".so")) continue;
1667 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1668 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1669 }
1670 delete arr;
1671 }
4e5c5506 1672 if (fPackages) {
1673 TIter next(fPackages);
1674 TObject *obj;
0f389141 1675 while ((obj=next())) {
4e5c5506 1676 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1677 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1678 }
4e5c5506 1679 }
c57f56b7 1680 if (fOutputArchive.Length()) {
e4d9a069 1681 TString outputArchive = fOutputArchive;
1682 if (!fRegisterExcludes.IsNull()) {
1683 arr = fRegisterExcludes.Tokenize(" ");
1684 TIter next1(arr);
1685 while ((os=(TObjString*)next1())) {
1686 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1687 outputArchive.ReplaceAll(os->GetString(),"");
1688 }
1689 delete arr;
1690 }
1691 arr = outputArchive.Tokenize(" ");
c57f56b7 1692 TIter next(arr);
0f389141 1693 Bool_t first = kTRUE;
1694 const char *comment = "Files to be archived";
1695 const char *comment1 = comment;
1696 while ((os=(TObjString*)next())) {
1697 if (!first) comment = NULL;
1698 if (!os->GetString().Contains("@") && fCloseSE.Length())
1699 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1700 else
1701 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1702 first = kFALSE;
1703 }
c57f56b7 1704 delete arr;
f790bc1b 1705 // Output archive for the merging jdl
f790bc1b 1706 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1707 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1708 // Add normal output files, extra files + terminate files
1709 TString files = GetListOfFiles("outextter");
37d21c01 1710 // Do not register files in fRegisterExcludes
1711 if (!fRegisterExcludes.IsNull()) {
1712 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1713 TIter next1(arr);
1714 while ((os=(TObjString*)next1())) {
1715 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1716 files.ReplaceAll(os->GetString(),"");
1717 }
66d71516 1718 delete arr;
0f389141 1719 }
f790bc1b 1720 files.ReplaceAll(".root", "*.root");
b83f84f0 1721 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1722 } else {
95f84779 1723 TString files = fOutputArchive;
1724 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1725 outputArchive = files;
f790bc1b 1726 }
0f389141 1727 arr = outputArchive.Tokenize(" ");
705adb3e 1728 TIter next2(arr);
0f389141 1729 comment = comment1;
1730 first = kTRUE;
705adb3e 1731 while ((os=(TObjString*)next2())) {
0f389141 1732 if (!first) comment = NULL;
0cdf65a8 1733 TString currentfile = os->GetString();
0cdf65a8 1734 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1735 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1736 else
0cdf65a8 1737 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1738 first = kFALSE;
1739 }
1740 delete arr;
c57f56b7 1741 }
149d288c 1742 arr = fOutputFiles.Tokenize(",");
c57f56b7 1743 TIter next(arr);
0f389141 1744 Bool_t first = kTRUE;
f790bc1b 1745 const char *comment = "Files to be saved";
43da816a 1746 while ((os=(TObjString*)next())) {
e1eaf596 1747 // Ignore ouputs in jdl that are also in outputarchive
1748 TString sout = os->GetString();
f790bc1b 1749 sout.ReplaceAll("*", "");
1750 sout.ReplaceAll(".root", "");
e1eaf596 1751 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1752 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1753 // Ignore fRegisterExcludes
1754 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1755 if (!first) comment = NULL;
43da816a 1756 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1757 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1758 else
0f389141 1759 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1760 first = kFALSE;
f790bc1b 1761 if (fMergeExcludes.Contains(sout)) continue;
1762 if (!os->GetString().Contains("@") && fCloseSE.Length())
1763 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1764 else
1765 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1766 }
c57f56b7 1767 delete arr;
0f389141 1768 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1769 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1770 TString validationScript = fValidationScript;
0f389141 1771 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1772 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1773 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1774 if (fMasterResubmitThreshold) {
1775 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1776 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1777 }
d2a409b2 1778 // Write a jdl with 2 input parameters: collection name and output dir name.
1779 WriteJDL(copy);
c57f56b7 1780 }
1781 // Copy jdl to grid workspace
a8739e8a 1782 if (copy) {
b5e4aaa7 1783 // Check if an output directory was defined and valid
1784 if (!fGridOutputDir.Length()) {
1785 Error("CreateJDL", "You must define AliEn output directory");
1786 return kFALSE;
1787 } else {
1788 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1789 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1790 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1791 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1792 } else {
1793 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1794 return kFALSE;
1795 }
1796 }
1797 gGrid->Cd(workdir);
1798 }
648174cf 1799 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1800 TString mergeJDLName = fExecutable;
1801 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1802 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1803 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1804 if (fProductionMode) {
648174cf 1805 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1806 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1807 }
648174cf 1808 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1809 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1810 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1811 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1812 Fatal("","Terminating");
1813// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1814 if (fMergeViaJDL) {
1815 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1816// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1817 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1818 Fatal("","Terminating");
0f389141 1819 }
648174cf 1820 }
c57f56b7 1821 if (fAdditionalLibs.Length()) {
e7c71df0 1822 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1823 TObjString *os;
1824 TIter next(arr);
1825 while ((os=(TObjString*)next())) {
c57f56b7 1826 if (os->GetString().Contains(".so")) continue;
4e5c5506 1827 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1828 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1829// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1830 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1831 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1832 Fatal("","Terminating");
c57f56b7 1833 }
1834 delete arr;
1835 }
4e5c5506 1836 if (fPackages) {
1837 TIter next(fPackages);
1838 TObject *obj;
1839 while ((obj=next())) {
fdbbc7be 1840 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1841 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1842// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1843 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1844 Form("%s/%s", workdir.Data(), obj->GetName())))
1845 Fatal("","Terminating");
4e5c5506 1846 }
1847 }
c57f56b7 1848 }
1849 return kTRUE;
1850}
1851
a8739e8a 1852//______________________________________________________________________________
d2a409b2 1853Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1854{
1855// Writes one or more JDL's corresponding to findex. If findex is negative,
1856// all run numbers are considered in one go (jdl). For non-negative indices
1857// they correspond to the indices in the array fInputFiles.
1858 if (!fInputFiles) return kFALSE;
d3b18c4c 1859 TObject *os;
1860 TString workdir;
1861 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1862 workdir += fGridWorkingDir;
f47d5cb4 1863 TString stageName = "$2";
1864 if (fProductionMode) stageName = "$4";
1865 if (!fMergeDirName.IsNull()) {
1866 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1867 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1868 } else {
1869 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1870 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1871 }
d3b18c4c 1872 if (fProductionMode) {
a8739e8a 1873 TIter next(fInputFiles);
b3e07543 1874 while ((os=next())) {
d3b18c4c 1875 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1876 }
56754bf6 1877 if (!fOutputToRunNo)
1878 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1879 else
1880 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1881 } else {
1882 if (!fRunNumbers.Length() && !fRunRange[0]) {
1883 // One jdl with no parameters in case input data is specified by name.
1884 TIter next(fInputFiles);
1885 while ((os=next()))
1886 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1887 if (!fOutputSingle.IsNull())
1888 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1889 else {
1890 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1891 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1892 }
1893 } else {
1894 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1895 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1896 if (!fOutputSingle.IsNull()) {
1897 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1898 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1899 } else {
1900 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1901 }
1902 }
a8739e8a 1903 }
1904
a8739e8a 1905 // Generate the JDL as a string
1906 TString sjdl = fGridJDL->Generate();
0f389141 1907 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1908 // Final merge jdl
1909 if (!fMergeDirName.IsNull()) {
1910 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1911 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1912 } else {
1913 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1914 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1915 }
b3e07543 1916 TString sjdl2 = fMergingJDL->Generate();
1917 Int_t index, index1;
a8739e8a 1918 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1919 sjdl.ReplaceAll("(member", "\n (member");
1920 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1921 sjdl.ReplaceAll("{", "{\n ");
1922 sjdl.ReplaceAll("};", "\n};");
1923 sjdl.ReplaceAll("{\n \n", "{\n");
1924 sjdl.ReplaceAll("\n\n", "\n");
1925 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1926 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1927 sjdl1.ReplaceAll("(member", "\n (member");
1928 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1929 sjdl1.ReplaceAll("{", "{\n ");
1930 sjdl1.ReplaceAll("};", "\n};");
1931 sjdl1.ReplaceAll("{\n \n", "{\n");
1932 sjdl1.ReplaceAll("\n\n", "\n");
1933 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1934 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1935 sjdl2.ReplaceAll("(member", "\n (member");
1936 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1937 sjdl2.ReplaceAll("{", "{\n ");
1938 sjdl2.ReplaceAll("};", "\n};");
1939 sjdl2.ReplaceAll("{\n \n", "{\n");
1940 sjdl2.ReplaceAll("\n\n", "\n");
1941 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1942 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1943 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1944 index = sjdl.Index("JDLVariables");
1945 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1946 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1947 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1948 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1949 index = fJobTag.Index(":");
1950 if (index < 0) index = fJobTag.Length();
1951 TString jobTag = fJobTag;
f47d5cb4 1952 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1953 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1954 if (fProductionMode) {
1955 sjdl1.Prepend("# Generated merging jdl (production mode) \
1956 \n# $1 = full alien path to output directory to be merged \
1957 \n# $2 = train number \
1958 \n# $3 = production (like LHC10b) \
1959 \n# $4 = merging stage \
1960 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1961 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1962 sjdl2.Prepend("# Generated merging jdl \
1963 \n# $1 = full alien path to output directory to be merged \
1964 \n# $2 = train number \
1965 \n# $3 = production (like LHC10b) \
1966 \n# $4 = merging stage \
1967 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1968 } else {
1969 sjdl1.Prepend("# Generated merging jdl \
1970 \n# $1 = full alien path to output directory to be merged \
1971 \n# $2 = merging stage \
03335324 1972 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1973 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1974 sjdl2.Prepend("# Generated merging jdl \
1975 \n# $1 = full alien path to output directory to be merged \
1976 \n# $2 = merging stage \
03335324 1977 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1978 }
0f389141 1979 index = sjdl1.Index("JDLVariables");
1980 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1981 index = sjdl2.Index("JDLVariables");
1982 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1983 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1984 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1985 index = sjdl2.Index("Split =");
1986 if (index>=0) {
1987 index1 = sjdl2.Index("\n", index);
1988 sjdl2.Remove(index, index1-index+1);
1989 }
1990 index = sjdl2.Index("SplitMaxInputFileNumber");
1991 if (index>=0) {
1992 index1 = sjdl2.Index("\n", index);
1993 sjdl2.Remove(index, index1-index+1);
1994 }
1995 index = sjdl2.Index("InputDataCollection");
1996 if (index>=0) {
1997 index1 = sjdl2.Index(";", index);
1998 sjdl2.Remove(index, index1-index+1);
1999 }
2000 index = sjdl2.Index("InputDataListFormat");
2001 if (index>=0) {
2002 index1 = sjdl2.Index("\n", index);
2003 sjdl2.Remove(index, index1-index+1);
2004 }
2005 index = sjdl2.Index("InputDataList");
2006 if (index>=0) {
2007 index1 = sjdl2.Index("\n", index);
2008 sjdl2.Remove(index, index1-index+1);
2009 }
f47d5cb4 2010 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2011 // Write jdl to file
a8739e8a 2012 ofstream out;
d2a409b2 2013 out.open(fJDLName.Data(), ios::out);
a8739e8a 2014 if (out.bad()) {
d3b18c4c 2015 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2016 return kFALSE;
2017 }
2018 out << sjdl << endl;
b3e07543 2019 out.close();
0f389141 2020 TString mergeJDLName = fExecutable;
2021 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2022 if (fMergeViaJDL) {
2023 ofstream out1;
2024 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2025 if (out1.bad()) {
d3b18c4c 2026 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2027 return kFALSE;
2028 }
2029 out1 << sjdl1 << endl;
b3e07543 2030 out1.close();
2031 ofstream out2;
2032 TString finalJDL = mergeJDLName;
2033 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2034 out2.open(finalJDL.Data(), ios::out);
2035 if (out2.bad()) {
2036 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2037 return kFALSE;
2038 }
2039 out2 << sjdl2 << endl;
2040 out2.close();
0f389141 2041 }
a8739e8a 2042
2043 // Copy jdl to grid workspace
2044 if (!copy) {
d3b18c4c 2045 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2046 } else {
d2a409b2 2047 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2048 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2049 TString finalJDL = mergeJDLName;
2050 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2051 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2052 if (fProductionMode) {
b5fe9cba 2053 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2054 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2055 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2056 }
d2a409b2 2057 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2058 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2059 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2060 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2061// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2062 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2063 Fatal("","Terminating");
0f389141 2064 if (fMergeViaJDL) {
b3e07543 2065 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2066// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2067// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2068 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2069 Fatal("","Terminating");
2070 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2071 Fatal("","Terminating");
0f389141 2072 }
a8739e8a 2073 }
2074 return kTRUE;
2075}
2076
c57f56b7 2077//______________________________________________________________________________
5513444a 2078Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2079{
2080// Returns true if file exists.
5513444a 2081 if (!gGrid) return kFALSE;
2ea55496 2082 TString slfn = lfn;
2083 slfn.ReplaceAll("alien://","");
2084 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2085 if (!res) return kFALSE;
2086 TMap *map = dynamic_cast<TMap*>(res->At(0));
2087 if (!map) {
2088 delete res;
2089 return kFALSE;
2090 }
2091 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2092 if (!objs || !objs->GetString().Length()) {
2093 delete res;
2094 return kFALSE;
2095 }
2096 delete res;
2097 return kTRUE;
2098}
2099
923e2ca5 2100//______________________________________________________________________________
2101Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2102{
2103// Returns true if directory exists. Can be also a path.
2104 if (!gGrid) return kFALSE;
2105 // Check if dirname is a path
2106 TString dirstripped = dirname;
2107 dirstripped = dirstripped.Strip();
2108 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2109 TString dir = gSystem->BaseName(dirstripped);
2110 dir += "/";
2111 TString path = gSystem->DirName(dirstripped);
2112 TGridResult *res = gGrid->Ls(path, "-F");
2113 if (!res) return kFALSE;
2114 TIter next(res);
2115 TMap *map;
2116 TObject *obj;
2117 while ((map=dynamic_cast<TMap*>(next()))) {
2118 obj = map->GetValue("name");
2119 if (!obj) break;
2120 if (dir == obj->GetName()) {
2121 delete res;
2122 return kTRUE;
2123 }
2124 }
2125 delete res;
2126 return kFALSE;
2127}
2128
c57f56b7 2129//______________________________________________________________________________
a2f5fc01 2130void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2131{
2132// Check input data type.
a2f5fc01 2133 isCollection = kFALSE;
2134 isXml = kFALSE;
2135 useTags = kFALSE;
c57f56b7 2136 if (!gGrid) {
2137 Error("CheckDataType", "No connection to grid");
2138 return;
2139 }
a2f5fc01 2140 isCollection = IsCollection(lfn);
c57f56b7 2141 TString msg = "\n##### file: ";
2142 msg += lfn;
a2f5fc01 2143 if (isCollection) {
c57f56b7 2144 msg += " type: raw_collection;";
2145 // special treatment for collections
a2f5fc01 2146 isXml = kFALSE;
c57f56b7 2147 // check for tag files in the collection
2148 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2149 if (!res) {
2150 msg += " using_tags: No (unknown)";
23329835 2151 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2152 return;
2153 }
2154 const char* typeStr = res->GetKey(0, "origLFN");
2155 if (!typeStr || !strlen(typeStr)) {
2156 msg += " using_tags: No (unknown)";
23329835 2157 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2158 return;
2159 }
2160 TString file = typeStr;
a2f5fc01 2161 useTags = file.Contains(".tag");
2162 if (useTags) msg += " using_tags: Yes";
c57f56b7 2163 else msg += " using_tags: No";
23329835 2164 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2165 return;
2166 }
2167 TString slfn(lfn);
2168 slfn.ToLower();
a2f5fc01 2169 isXml = slfn.Contains(".xml");
2170 if (isXml) {
c57f56b7 2171 // Open xml collection and check if there are tag files inside
2172 msg += " type: xml_collection;";
2173 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2174 if (!coll) {
2175 msg += " using_tags: No (unknown)";
23329835 2176 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2177 return;
2178 }
2179 TMap *map = coll->Next();
2180 if (!map) {
2181 msg += " using_tags: No (unknown)";
23329835 2182 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2183 return;
2184 }
2185 map = (TMap*)map->GetValue("");
2186 TString file;
2187 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2188 useTags = file.Contains(".tag");
c57f56b7 2189 delete coll;
a2f5fc01 2190 if (useTags) msg += " using_tags: Yes";
c57f56b7 2191 else msg += " using_tags: No";
23329835 2192 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2193 return;
2194 }
a2f5fc01 2195 useTags = slfn.Contains(".tag");
c57f56b7 2196 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2197 else msg += " type: unknown file;";
a2f5fc01 2198 if (useTags) msg += " using_tags: Yes";
c57f56b7 2199 else msg += " using_tags: No";
23329835 2200 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2201}
2202
4e5c5506 2203//______________________________________________________________________________
2204void AliAnalysisAlien::EnablePackage(const char *package)
2205{
2206// Enables a par file supposed to exist in the current directory.
2207 TString pkg(package);
2208 pkg.ReplaceAll(".par", "");
2209 pkg += ".par";
2210 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2211 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2212 return;
2213 }
2214 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2215 Info("EnablePackage", "AliEn plugin will use .par packages");
2216 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2217 if (!fPackages) {
2218 fPackages = new TObjArray();
2219 fPackages->SetOwner();
2220 }
2221 fPackages->Add(new TObjString(pkg));
2222}
2223
3bdcb562 2224//______________________________________________________________________________
2225TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2226{
2227// Make a tree from files having the location specified in fFileForTestMode.
2228// Inspired from JF's CreateESDChain.
2229 if (fFileForTestMode.IsNull()) {
2230 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2231 return NULL;
2232 }
2233 if (gSystem->AccessPathName(fFileForTestMode)) {
2234 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2235 return NULL;
2236 }
2237 // Open the file
2238 ifstream in;
2239 in.open(fFileForTestMode);
2240 Int_t count = 0;
2241 // Read the input list of files and add them to the chain
b505f1f8 2242 TString line;
2243 TChain *chain = new TChain(treeName);
2244 TChain *chainFriend = 0;
2245 if (!fFriendChainName.IsNull()) chainFriend = new TChain(treeName);
2246 while (in.good())
2247 {
3bdcb562 2248 in >> line;
2249 if (line.IsNull()) continue;
2250 if (count++ == fNtestFiles) break;
2251 TString esdFile(line);
2252 TFile *file = TFile::Open(esdFile);
b505f1f8 2253 if (file && !file->IsZombie()) {
2254 chain->Add(esdFile);
3bdcb562 2255 file->Close();
b505f1f8 2256 if (!fFriendChainName.IsNull()) {
4b5d9e21 2257 if (esdFile.Index("#") > -1)
2258 esdFile.Remove(esdFile.Index("#"));
2259 esdFile = gSystem->DirName(esdFile);
2260 esdFile += "/" + fFriendChainName;
b505f1f8 2261 file = TFile::Open(esdFile);
2262 if (file && !file->IsZombie()) {
2263 file->Close();
2264 chainFriend->Add(esdFile);
2265 } else {
2266 Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data());
2267 return 0;
2268 }
4b5d9e21 2269 }
3bdcb562 2270 } else {
2271 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2272 }
b505f1f8 2273 }
2274 in.close();
2275 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2276 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2277 delete chain;
b505f1f8 2278 delete chainFriend;
3bdcb562 2279 return NULL;
b505f1f8 2280 }
3bdcb562 2281// chain->ls();
b505f1f8 2282 if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend);
2283 return chain;
3bdcb562 2284}
2285
d2a409b2 2286//______________________________________________________________________________
2287const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2288{
2289// Get job status for all jobs with jobid>jobidstart.
2290 static char mstatus[20];
2291 mstatus[0] = '\0';
2292 nrunning = 0;
2293 nwaiting = 0;
2294 nerror = 0;
2295 ndone = 0;
2296 TGridJobStatusList *list = gGrid->Ps("");
2297 if (!list) return mstatus;
2298 Int_t nentries = list->GetSize();
2299 TGridJobStatus *status;
2300 Int_t pid;
2301 for (Int_t ijob=0; ijob<nentries; ijob++) {
2302 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2303 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2304 if (pid<jobidstart) continue;
2305 if (pid == lastid) {
8ddad121 2306 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2307 }
2308 switch (status->GetStatus()) {
2309 case TGridJobStatus::kWAITING:
2310 nwaiting++; break;
2311 case TGridJobStatus::kRUNNING:
2312 nrunning++; break;
2313 case TGridJobStatus::kABORTED:
2314 case TGridJobStatus::kFAIL:
2315 case TGridJobStatus::kUNKNOWN:
2316 nerror++; break;
2317 case TGridJobStatus::kDONE:
2318 ndone++;
2319 }
2320 }
2321 list->Delete();
2322 delete list;
2323 return mstatus;
2324}
2325
c57f56b7 2326//______________________________________________________________________________
2327Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2328{
2329// Returns true if file is a collection. Functionality duplicated from
2330// TAlien::Type() because we don't want to directly depend on TAlien.
2331 if (!gGrid) {
2332 Error("IsCollection", "No connection to grid");
2333 return kFALSE;
2334 }
2335 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2336 if (!res) return kFALSE;
2337 const char* typeStr = res->GetKey(0, "type");
2338 if (!typeStr || !strlen(typeStr)) return kFALSE;
2339 if (!strcmp(typeStr, "collection")) return kTRUE;
2340 delete res;
2341 return kFALSE;
2342}
2343
fe2d7fc2 2344//______________________________________________________________________________
2345Bool_t AliAnalysisAlien::IsSingleOutput() const
2346{
2347// Check if single-ouput option is on.
2348 return (!fOutputSingle.IsNull());
2349}
2350
16a4353c 2351//______________________________________________________________________________
2352void AliAnalysisAlien::Print(Option_t *) const
2353{
2354// Print current plugin settings.
84fcd93f 2355 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2356 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2357 if (mgr && mgr->IsProofMode()) {
2358 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2359 if (TestBit(AliAnalysisGrid::kTest))
2360 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2361 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2362 if (!fProofDataSet.IsNull())
2363 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2364 if (fProofReset==1)
2365 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2366 if (fProofReset>1)
2367 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2368 if (!fRootVersionForProof.IsNull())
2369 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2370 else
2371 printf("= ROOT version requested________________________ default\n");
2372 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2373 if (!fAliRootMode.IsNull())
2374 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2375 if (fNproofWorkers)
2376 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2377 if (fNproofWorkersPerSlave)
2378 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2379 if (TestSpecialBit(kClearPackages))
2380 printf("= ClearPackages requested...\n");
2381 if (fIncludePath.Data())
2382 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2383 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2384 if (fPackages && fPackages->GetEntries()) {
2385 TIter next(fPackages);
2386 TObject *obj;
2387 TString list;
2388 while ((obj=next())) list += obj->GetName();
2389 printf("= Par files to be used: ________________________ %s\n", list.Data());
2390 }
2391 if (TestSpecialBit(kProofConnectGrid))
2392 printf("= Requested PROOF connection to grid\n");
2393 return;
2394 }
e1c22e21 2395 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2396 if (fOverwriteMode) {
2397 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2398 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2399 }
348be253 2400 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2401 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2402 printf("= Production mode:______________________________ %d\n", fProductionMode);
2403 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2404 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2405 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2406 if (fUser.Length())
84fcd93f 2407 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2408 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2409 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2410 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2411 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2412 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2413 if (fRunNumbers.Length())
84fcd93f 2414 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2415 if (fRunRange[0])
74013535 2416 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2417 if (!fRunRange[0] && !fRunNumbers.Length()) {
2418 TIter next(fInputFiles);
2419 TObject *obj;
2420 TString list;
2421 while ((obj=next())) list += obj->GetName();
84fcd93f 2422 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2423 }
2424 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2425 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2426 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2427 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2428 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2429 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2430 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2431 printf("=====================================================================\n");
2432 printf("= Job price: ___________________________________ %d\n", fPrice);
2433 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2434 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2435 if (fMaxInitFailed>0)
84fcd93f 2436 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2437 if (fMasterResubmitThreshold>0)
84fcd93f 2438 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2439 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2440 if (fNrunsPerMaster>0)
84fcd93f 2441 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2442 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2443 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2444 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2445 if (fArguments.Length())
84fcd93f 2446 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2447 if (fExecutableArgs.Length())
2448 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2449 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2450 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2451 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2452 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2453 if (fDatasetName)
84fcd93f 2454 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2455 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2456 if (fIncludePath.Data())
84fcd93f 2457 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2458 if (fCloseSE.Length())
84fcd93f 2459 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2460 if (fFriendChainName.Length())
84fcd93f 2461 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2462 if (fPackages && fPackages->GetEntries()) {
16a4353c 2463 TIter next(fPackages);
2464 TObject *obj;
2465 TString list;
2466 while ((obj=next())) list += obj->GetName();
84fcd93f 2467 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2468 }
2469}
2470
c57f56b7 2471//______________________________________________________________________________
2472void AliAnalysisAlien::SetDefaults()
2473{
2474// Set default values for everything. What cannot be filled will be left empty.
2475 if (fGridJDL) delete fGridJDL;
2476 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2477 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2478 fPrice = 1;
2479 fTTL = 30000;
2480 fSplitMaxInputFileNumber = 100;
2481 fMaxInitFailed = 0;
2482 fMasterResubmitThreshold = 0;
bb885a9e 2483 fNtestFiles = 10;
149d288c 2484 fNreplicas = 2;
a8739e8a 2485 fRunRange[0] = 0;
2486 fRunRange[1] = 0;
8e3bd06a 2487 fRunPrefix = "%d";
319593fb 2488 fNrunsPerMaster = 1;
16a4353c 2489 fMaxMergeFiles = 100;
c57f56b7 2490 fRunNumbers = "";
2491 fExecutable = "analysis.sh";
0a1c1f7f 2492 fExecutableCommand = "root -b -q";
c57f56b7 2493 fArguments = "";
631c0b05 2494 fExecutableArgs = "";
c57f56b7 2495 fAnalysisMacro = "myAnalysis.C";
2496 fAnalysisSource = "";
2497 fAdditionalLibs = "";
2498 fSplitMode = "se";
2499 fAPIVersion = "";
2500 fROOTVersion = "";
2501 fAliROOTVersion = "";
2502 fUser = ""; // Your alien user name
2503 fGridWorkingDir = "";
2504 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2505 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2506 fFriendChainName = "";
c57f56b7 2507 fGridOutputDir = "output";
b83f84f0 2508 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2509 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2510 fInputFormat = "xml-single";
2511 fJDLName = "analysis.jdl";
c6cb3634 2512 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2513 fMergeExcludes = "";
0f389141 2514 fMergeViaJDL = 0;
348be253 2515 SetUseCopy(kTRUE);
2516 SetCheckCopy(kTRUE);
149d288c 2517 SetDefaultOutputs(kTRUE);
e1c22e21 2518 fOverwriteMode = 1;
c57f56b7 2519}
2520
0f389141 2521//______________________________________________________________________________
b3e07543 2522Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2523{
b3e07543 2524// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2525 // First check if the result is already in the output directory.
2526 if (FileExists(Form("%s/%s",aliendir,filename))) {
2527 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2528 return kFALSE;
2529 }
b3e07543 2530 // Now check the last stage done.
2531 Int_t stage = 0;
2532 while (1) {
2533 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2534 stage++;
2535 }
2536 // Next stage of merging
2537 stage++;
2538 TString pattern = "*root_archive.zip";
2539 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2540 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2541 if (res) delete res;
2542 // Write standard output to file
2543 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2544 // Count the number of files inside
2545 ifstream ifile;
2546 ifile.open(Form("Stage_%d.xml",stage));
2547 if (!ifile.good()) {
2548 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2549 return kFALSE;
b3e07543 2550 }
2551 TString line;
2552 Int_t nfiles = 0;
2553 while (!ifile.eof()) {
2554 ifile >> line;
2555 if (line.Contains("/event")) nfiles++;
7c2cd90a 2556 }
b3e07543 2557 ifile.close();
2558 if (!nfiles) {
2559 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2560 return kFALSE;
2561 } else {
2562 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2563 }
b3e07543 2564 // Copy the file in the output directory
2565 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2566// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2567 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2568 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2569 // Check if this is the last stage to be done.
2570 Bool_t laststage = (nfiles<nperchunk);
2571 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2572 if (laststage) {
2573 printf("### Submiting final merging stage %d\n", stage);
2574 TString finalJDL = jdl;
2575 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2576 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2577 Int_t jobId = SubmitSingleJob(query);
2578 if (!jobId) return kFALSE;
2579 } else {
2580 printf("### Submiting merging stage %d\n", stage);
03335324 2581 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2582 Int_t jobId = SubmitSingleJob(query);
2583 if (!jobId) return kFALSE;
7c2cd90a 2584 }
b3e07543 2585 return kTRUE;
2586}
7c2cd90a 2587
105631b5 2588//______________________________________________________________________________
2589AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2590{
2591// Loat the analysis manager from a file.
2592 TFile *file = TFile::Open(fname);
2593 if (!file) {
2594 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2595 return 0;
2596 }
2597 TIter nextkey(file->GetListOfKeys());
2598 AliAnalysisManager *mgr = 0;
2599 TKey *key;
2600 while ((key=(TKey*)nextkey())) {
2601 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2602 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2603 }
2604 if (!mgr)
2605 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2606 return mgr;
2607}
2608
7c2cd90a 2609//______________________________________________________________________________
2610Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2611{
2612// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2613 if (!gGrid) return 0;
2614 printf("=> %s ------> ",query);
2615 TGridResult *res = gGrid->Command(query);
2616 if (!res) return 0;
2617 TString jobId = res->GetKey(0,"jobId");
2618 delete res;
2619 if (jobId.IsNull()) {
2620 printf("submission failed. Reason:\n");
2621 gGrid->Stdout();
2622 gGrid->Stderr();
2623 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2624 return 0;
2625 }
2626 printf(" Job id: %s\n", jobId.Data());
2627 return atoi(jobId);
2628}
2629
2630//______________________________________________________________________________
b3e07543 2631Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2632{
b3e07543 2633// Merge given output files from basedir. Basedir can be an alien output directory
2634// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2635// files in a group (ignored for xml input). Merging can be done in stages:
2636// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2637// stage=1 : works with an xml of all root_archive.zip in the output directory
2638// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2639 TString outputFile = output;
0f389141 2640 TString command;
a2f5fc01 2641 TString outputChunk;
2642 TString previousChunk = "";
b3e07543 2643 TObjArray *listoffiles = new TObjArray();
2644// listoffiles->SetOwner();
a2f5fc01 2645 Int_t countChunk = 0;
2646 Int_t countZero = nmaxmerge;
0f389141 2647 Bool_t merged = kTRUE;
a2f5fc01 2648 Int_t index = outputFile.Index("@");
2649 if (index > 0) outputFile.Remove(index);
7c2cd90a 2650 TString inputFile = outputFile;
b3e07543 2651 TString sbasedir = basedir;
2652 if (sbasedir.Contains(".xml")) {
2653 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2654 nmaxmerge = 9999999;
2655 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2656 if (!coll) {
2657 ::Error("MergeOutput", "Input XML collection empty.");
2658 return kFALSE;
2659 }
2660 // Iterate grid collection
2661 while (coll->Next()) {
2662 TString fname = gSystem->DirName(coll->GetTURL());
2663 fname += "/";
2664 fname += inputFile;
2665 listoffiles->Add(new TNamed(fname.Data(),""));
2666 }
2667 } else {
2668 command = Form("find %s/ *%s", basedir, inputFile.Data());
2669 printf("command: %s\n", command.Data());
2670 TGridResult *res = gGrid->Command(command);
2671 if (!res) {
2672 ::Error("MergeOutput","No result for the find command\n");
2673 delete listoffiles;
2674 return kFALSE;
2675 }
2676 TIter nextmap(res);
2677 TMap *map = 0;
2678 while ((map=(TMap*)nextmap())) {
2679 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2680 if (!objs || !objs->GetString().Length()) {
2681 // Nothing found - skip this output
2682 delete res;
2683 delete listoffiles;
2684 return kFALSE;
2685 }
2686 listoffiles->Add(new TNamed(objs->GetName(),""));
2687 }
2688 delete res;
2689 }
2690 if (!listoffiles->GetEntries()) {
7c2cd90a 2691 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2692 delete listoffiles;
0f389141 2693 return kFALSE;
2694 }
2695
2696 TFileMerger *fm = 0;
b3e07543 2697 TIter next0(listoffiles);
2698 TObjArray *listoffilestmp = new TObjArray();
2699 listoffilestmp->SetOwner();
2700 TObject *nextfile;
2701 TString snextfile;
2702 // Keep only the files at upper level
2703 Int_t countChar = 0;
2704 while ((nextfile=next0())) {
2705 snextfile = nextfile->GetName();
2706 Int_t crtCount = snextfile.CountChar('/');
2707 if (nextfile == listoffiles->First()) countChar = crtCount;
2708 if (crtCount < countChar) countChar = crtCount;
2709 }
2710 next0.Reset();
2711 while ((nextfile=next0())) {
2712 snextfile = nextfile->GetName();
2713 Int_t crtCount = snextfile.CountChar('/');
2714 if (crtCount > countChar) {
2715 delete nextfile;
2716 continue;
2717 }
2718 listoffilestmp->Add(nextfile);
2719 }
2720 delete listoffiles;
2721 listoffiles = listoffilestmp; // Now contains 'good' files
2722 listoffiles->Print();
2723 TIter next(listoffiles);
7c2cd90a 2724 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2725 outputChunk = outputFile;
2726 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2727 // Check for existent temporary merge files
e1c22e21 2728 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2729 // Preserve old merging functionality for stage 0.
2730 if (stage==0) {
2731 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2732 while (1) {
2733 // Skip as many input files as in a chunk
2ea55496 2734 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2735 nextfile = next();
2736 if (!nextfile) {
2ea55496 2737 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2738 delete listoffiles;
2ea55496 2739 return kFALSE;
2740 }
b3e07543 2741 snextfile = nextfile->GetName();
7c2cd90a 2742 }
2743 outputChunk = outputFile;
2744 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2745 countChunk++;
2746 if (gSystem->AccessPathName(outputChunk)) continue;
2747 // Merged file with chunks up to <countChunk> found
2748 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2749 previousChunk = outputChunk;
2750 break;
2751 }
2752 }
2753 countZero = nmaxmerge;
2754
b3e07543 2755 while ((nextfile=next())) {
2756 snextfile = nextfile->GetName();
2ea55496 2757 // Loop 'find' results and get next LFN
7c2cd90a 2758 if (countZero == nmaxmerge) {
2759 // First file in chunk - create file merger and add previous chunk if any.
2b895f34 2760 fm = new TFileMerger(kTRUE);
7c2cd90a 2761 fm->SetFastMethod(kTRUE);
2762 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2763 outputChunk = outputFile;
2764 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2765 }
2766 // If last file found, put merged results in the output file
b3e07543 2767 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2768 // Add file to be merged and decrement chunk counter.
b3e07543 2769 fm->AddFile(snextfile);
7c2cd90a 2770 countZero--;
b3e07543 2771 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2772 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2773 // Nothing found - skip this output
2774 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2775 merged = kFALSE;
2776 break;
7c2cd90a 2777 }
2778 fm->OutputFile(outputChunk);
2779 // Merge the outputs, then go to next chunk
2780 if (!fm->Merge()) {
2781 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2782 merged = kFALSE;
2783 break;
7c2cd90a 2784 } else {
2785 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2786 gSystem->Unlink(previousChunk);
2787 }
b3e07543 2788 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2789 countChunk++;
2790 countZero = nmaxmerge;
2791 previousChunk = outputChunk;
0f389141 2792 }
0f389141 2793 }
b3e07543 2794 delete listoffiles;
2795 delete fm;
7c2cd90a 2796 return merged;
2797 }
2798 // Merging stage different than 0.
2799 // Move to the begining of the requested chunk.
2b895f34 2800 fm = new TFileMerger(kTRUE);
7c2cd90a 2801 fm->SetFastMethod(kTRUE);
b3e07543 2802 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2803 delete listoffiles;
7c2cd90a 2804 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2805 // Nothing found - skip this output
2806 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2807 delete fm;
2808 return kFALSE;
2809 }
b3e07543 2810 fm->OutputFile(outputFile);
7c2cd90a 2811 // Merge the outputs
2812 if (!fm->Merge()) {
2813 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2814 delete fm;
2815 return kFALSE;
2816 } else {
b3e07543 2817 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2818 }
7c2cd90a 2819 delete fm;
2820 return kTRUE;
0f389141 2821}
2822
c57f56b7 2823//______________________________________________________________________________
2824Bool_t AliAnalysisAlien::MergeOutputs()
2825{
2826// Merge analysis outputs existing in the AliEn space.
2827 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2828 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2829 if (!Connect()) {
2830 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2831 return kFALSE;
0f389141 2832 }
c9e8f7fd 2833 if (fMergeViaJDL) {
2834 if (!TestBit(AliAnalysisGrid::kMerge)) {
2835 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2836 return kFALSE;
2837 }
2838 if (fProductionMode) {
2839 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2840 return kFALSE;
2841 }
0f389141 2842 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2843 if (!SubmitMerging()) return kFALSE;
0f389141 2844 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2845 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2846 return kFALSE;
c57f56b7 2847 }
2848 // Get the output path
dc030a24 2849 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2850 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2851 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2852 return kFALSE;
2853 }
2854 if (!fOutputFiles.Length()) {
2855 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2856 return kFALSE;
0f389141 2857 }
2858 // Check if fast read option was requested
e1c22e21 2859 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2860 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2861 if (fFastReadOption) {
d3339be3 2862 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2863 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2864 gEnv->SetValue("XNet.ConnectTimeout",50);
2865 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2866 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2867 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2868 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2869 }
e8b839ab 2870 // Make sure we change the temporary directory
2871 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2872 // Set temporary compilation directory to current one
2873 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2874 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2875 TIter next(list);
2876 TObjString *str;
a2f5fc01 2877 TString outputFile;
c57f56b7 2878 Bool_t merged = kTRUE;
2879 while((str=(TObjString*)next())) {
a2f5fc01 2880 outputFile = str->GetString();
2881 Int_t index = outputFile.Index("@");
2882 if (index > 0) outputFile.Remove(index);
2883 TString outputChunk = outputFile;
2884 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2885 // Skip already merged outputs
a2f5fc01 2886 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2887 if (fOverwriteMode) {
a2f5fc01 2888 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2889 gSystem->Unlink(outputFile);
2890 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2891 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2892 outputChunk.Data());
2893 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2894 }
2895 } else {
a2f5fc01 2896 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2897 continue;
2898 }
2899 } else {
a2f5fc01 2900 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2901 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2902 outputChunk.Data());
2903 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2904 }
2905 }
37d21c01 2906 if (fMergeExcludes.Contains(outputFile.Data()) ||
2907 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 2908 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2909 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2910 if (!merged) {
2911 Error("MergeOutputs", "Terminate() will NOT be executed");
2912 return kFALSE;
ff07ec61 2913 }
2914 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2915 if (fileOpened) fileOpened->Close();
c57f56b7 2916 }
0f389141 2917 return kTRUE;
c57f56b7 2918}
2919
bb885a9e 2920//______________________________________________________________________________
2921void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2922{
2923// Use the output files connected to output containers from the analysis manager
2924// rather than the files defined by SetOutputFiles
2925 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2926 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2927 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2928}
2929
149d288c 2930//______________________________________________________________________________
2931void AliAnalysisAlien::SetOutputFiles(const char *list)
2932{
2933// Manually set the output files list.
2934// Removes duplicates. Not allowed if default outputs are not disabled.
2935 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2936 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2937 return;
2938 }
2939 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2940 fOutputFiles = "";
2941 TString slist = list;
2942 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2943 TObjArray *arr = slist.Tokenize(" ");
2944 TObjString *os;
2945 TIter next(arr);
2946 TString sout;
2947 while ((os=(TObjString*)next())) {
2948 sout = os->GetString();
2949 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2950 if (fOutputFiles.Contains(sout)) continue;
2951 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2952 fOutputFiles += sout;
2953 }
2954 delete arr;
f790bc1b 2955}
149d288c 2956
2957//______________________________________________________________________________
2958void AliAnalysisAlien::SetOutputArchive(const char *list)
2959{
2960// Manually set the output archive list. Free text - you are on your own...
2961// Not allowed if default outputs are not disabled.
2962 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2963 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2964 return;
2965 }
2966 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2967 fOutputArchive = list;
2968}
2969
2970//______________________________________________________________________________
2971void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2972{
2973// Setting a prefered output SE is not allowed anymore.
2974 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2975}
2976
28ce6c9c 2977//______________________________________________________________________________
2978void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
2979{
2980// Set some PROOF special parameter.
2981 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2982 if (pair) {
2983 TObject *old = pair->Key();
2984 TObject *val = pair->Value();
2985 fProofParam.Remove(old);
2986 delete old;
2987 delete val;
2988 }
2989 fProofParam.Add(new TObjString(pname), new TObjString(value));
2990}
2991
2992//______________________________________________________________________________
2993const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
2994{
2995// Returns a special PROOF parameter.
2996 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2997 if (!pair) return 0;
2998 return pair->Value()->GetName();
2999}
3000
5513444a 3001//______________________________________________________________________________
3002Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 3003{
3004// Start remote grid analysis.
3bdcb562 3005 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3006 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
3007 if (!mgr || !mgr->IsInitialized()) {
3008 Error("StartAnalysis", "You need an initialized analysis manager for this");
3009 return kFALSE;
3010 }
3011 // Are we in PROOF mode ?
3012 if (mgr->IsProofMode()) {
baa34407 3013 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3014 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3015 if (fProofCluster.IsNull()) {
3016 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3017 return kFALSE;
3018 }
3019 if (fProofDataSet.IsNull() && !testMode) {
3020 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3021 return kFALSE;
3022 }
3023 // Set the needed environment
3024 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3025 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3026 if (fProofReset && !testMode) {
3027 if (fProofReset==1) {
3028 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3029 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3030 } else {
3031 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3032 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3033 }
3034 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3035 return kFALSE;
3036 }
baa34407 3037
3038 if (!testMode) {
3039 // Check if there is an old active session
3040 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3041 if (nsessions) {
3042 Error("StartAnalysis","You have to reset your old session first\n");
3043 return kFALSE;
3044 }
53911ca6 3045 }
3bdcb562 3046 // Do we need to change the ROOT version ? The success of this cannot be checked.
3047 if (!fRootVersionForProof.IsNull() && !testMode) {
3048 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
3049 fProofCluster.Data(), fRootVersionForProof.Data()));
3050 }
3051 // Connect to PROOF and check the status
3052 Long_t proof = 0;
27734f0e 3053 TString sworkers;
3054 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3055 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3056 if (!testMode) {
27734f0e 3057 if (!sworkers.IsNull())
3058 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3059 else
3060 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3061 } else {
3062 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3063 if (!proof) {
3064 Error("StartAnalysis", "Could not start PROOF in test mode");
3065 return kFALSE;
3066 }
3067 }
3068 if (!proof) {
3069 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3070 return kFALSE;
3071 }
27734f0e 3072 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3073 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3074 // Set proof special parameters if any
3075 TIter nextpp(&fProofParam);
3076 TObject *proofparam;
3077 while ((proofparam=nextpp())) {
3078 TString svalue = GetProofParameter(proofparam->GetName());
3079 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3080 }
3bdcb562 3081 // Is dataset existing ?
3082 if (!testMode) {
3083 TString dataset = fProofDataSet;
3084 Int_t index = dataset.Index("#");
3085 if (index>=0) dataset.Remove(index);
830acc4c 3086// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3087// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3088// return kFALSE;
3089// }
3090// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3091 }
3092 // Is ClearPackages() needed ?
3093 if (TestSpecialBit(kClearPackages)) {
3094 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3095 gROOT->ProcessLine("gProof->ClearPackages();");
3096 }
3097 // Is a given aliroot mode requested ?
3098 TList optionsList;
1ac846e9 3099 TString parLibs;
830acc4c 3100 if (!fAliRootMode.IsNull()) {
3bdcb562 3101 TString alirootMode = fAliRootMode;
3102 if (alirootMode == "default") alirootMode = "";
3103 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3104 optionsList.SetOwner();
3105 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3106 // Check the additional libs to be loaded
3107 TString extraLibs;
1ac846e9 3108 Bool_t parMode = kFALSE;
dc1ffbcb 3109 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3110 // Parse the extra libs for .so
3111 if (fAdditionalLibs.Length()) {
3112 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3113 TIter next(list);
3114 TObjString *str;
1ac846e9 3115 while((str=(TObjString*)next())) {
3116 if (str->GetString().Contains(".so")) {
3117 if (parMode) {
3118 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3119 break;
3120 }
3121 TString stmp = str->GetName();
3122 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3123 stmp.ReplaceAll(".so","");
3124 if (!extraLibs.IsNull()) extraLibs += ":";
3125 extraLibs += stmp;
3126 continue;
3127 }
3128 if (str->GetString().Contains(".par")) {
3129 // The first par file found in the list will not allow any further .so
3130 parMode = kTRUE;
3131 if (!parLibs.IsNull()) parLibs += ":";
3132 parLibs += str->GetName();
3133 continue;
3134 }
3bdcb562 3135 }
3136 if (list) delete list;
3137 }
dc030a24 3138 if (!extraLibs.IsNull()) {
3139 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3140 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3141 }
830acc4c 3142 // Check extra includes
3143 if (!fIncludePath.IsNull()) {
3144 TString includePath = fIncludePath;
3145 includePath.ReplaceAll(" ",":");
dc030a24 3146 includePath.ReplaceAll("$ALICE_ROOT/","");
3147 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3148 includePath.ReplaceAll("-I","");
dc030a24 3149 includePath.Remove(TString::kTrailing, ':');
830acc4c 3150 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3151 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3152 }
27734f0e 3153 // Check if connection to grid is requested
3154 if (TestSpecialBit(kProofConnectGrid))
3155 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3156 // Enable AliRoot par
3157 if (testMode) {
3158 // Enable proof lite package
3159 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3160 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3161 TNamed *obj = (TNamed*)optionsList.At(i);
3162 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3163 }
3164 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3165 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3166 Info("StartAnalysis", "AliRootProofLite enabled");
3167 } else {
3168 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3169 return kFALSE;
3170 }
3171 } else {
dc030a24 3172 if ( ! fAliROOTVersion.IsNull() ) {
3173 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3174 fAliROOTVersion.Data(), &optionsList))) {
3175 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3176 return kFALSE;
3177 }
3178 }
27734f0e 3179 }
1ac846e9 3180 // Enable first par files from fAdditionalLibs
3181 if (!parLibs.IsNull()) {
3182 TObjArray *list = parLibs.Tokenize(":");
3183 TIter next(list);
3184 TObjString *package;
3185 while((package=(TObjString*)next())) {
40501bb7 3186 TString spkg = package->GetName();
3187 spkg.ReplaceAll(".par", "");
3188 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3189 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3190 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3191 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3192 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3193 return kFALSE;
3194 }
3195 } else {
3196 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3197 return kFALSE;
3198 }
3199 }
3200 if (list) delete list;
3201 }
3bdcb562 3202 } else {
3203 if (fAdditionalLibs.Contains(".so") && !testMode) {
3204 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3205 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3206 return kFALSE;
3207 }
3208 }
3209 // Enable par files if requested
3210 if (fPackages && fPackages->GetEntries()) {
3211 TIter next(fPackages);
3212 TObject *package;
3213 while ((package=next())) {
1ac846e9 3214 // Skip packages already enabled
3215 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3216 TString spkg = package->GetName();
3217 spkg.ReplaceAll(".par", "");
3218 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3219 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3220 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3221 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3222 return kFALSE;
3223 }
3224 } else {
3225 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3226 return kFALSE;
3227 }
3228 }
3229 }
3230 // Do we need to load analysis source files ?
3231 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3232 if (fAnalysisSource.Length()) {
3233 TObjArray *list = fAnalysisSource.Tokenize(" ");
3234 TIter next(list);
3235 TObjString *str;
3236 while((str=(TObjString*)next())) {
3237 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3238 }
3bdcb562 3239 if (list) delete list;
3240 }
830acc4c 3241 if (testMode) {
3242 // Register dataset to proof lite.
3243 if (fFileForTestMode.IsNull()) {
3244 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3245 return kFALSE;
3246 }
3247 if (gSystem->AccessPathName(fFileForTestMode)) {
3248 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3249 return kFALSE;
3250 }
3251 TFileCollection *coll = new TFileCollection();
3252 coll->AddFromFile(fFileForTestMode);
8ddad121 3253 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3254 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3255 }
3bdcb562 3256 return kTRUE;
3257 }
c57f56b7 3258
43da816a 3259 // Check if output files have to be taken from the analysis manager
3260 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3261 // Add output files and AOD files
3262 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3263 // Add extra files registered to the analysis manager
f790bc1b 3264 TString extra = GetListOfFiles("ext");
3265 if (!extra.IsNull()) {
149d288c 3266 extra.ReplaceAll(".root", "*.root");
f790bc1b 3267 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3268 fOutputFiles += extra;
84fcd93f 3269 }
149d288c 3270 // Compose the output archive.
b83f84f0 3271 fOutputArchive = "log_archive.zip:std*@disk=1 ";
3272 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3273 }
f7b1cbc2 3274// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3275 if (TestBit(AliAnalysisGrid::kOffline)) {
3276 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3277 \n there nor any job run. You can revise the JDL and analysis \
3278 \n macro then run the same in \"submit\" mode.");
3279 } else if (TestBit(AliAnalysisGrid::kTest)) {
3280 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3281 \n dataset.");
3282 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3283 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3284 \n space and job submitted.");
3285 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3286 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3287 if (fMergeViaJDL) CheckInputData();
5513444a 3288 return kTRUE;
c57f56b7 3289 } else {
3290 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3291 }
3292
348be253 3293 Print();
c57f56b7 3294 if (!Connect()) {
3295 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3296 return kFALSE;
16a4353c 3297 }
d3b18c4c 3298 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3299 if (!CheckInputData()) {
3300 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3301 return kFALSE;
c57f56b7 3302 }
d3339be3 3303 if (!CreateDataset(fDataPattern)) {
3304 TString serror;
3305 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3306 if (fRunNumbers.Length()) serror = "run numbers";
3307 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3308 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3309 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3310 return kFALSE;
3311 }
21ca8e59 3312 WriteAnalysisFile();
c57f56b7 3313 WriteAnalysisMacro();
3314 WriteExecutable();
3315 WriteValidationScript();
0f389141 3316 if (fMergeViaJDL) {
3317 WriteMergingMacro();
3318 WriteMergeExecutable();
3319 WriteValidationScript(kTRUE);
3320 }
5513444a 3321 if (!CreateJDL()) return kFALSE;
3322 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3323 if (testMode) {
c57f56b7 3324 // Locally testing the analysis
3325 Info("StartAnalysis", "\n_______________________________________________________________________ \
3326 \n Running analysis script in a daughter shell as on a worker node \
3327 \n_______________________________________________________________________");
149d288c 3328 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3329 TIter next(list);
3330 TObjString *str;
a2f5fc01 3331 TString outputFile;
c57f56b7 3332 while((str=(TObjString*)next())) {
a2f5fc01 3333 outputFile = str->GetString();
3334 Int_t index = outputFile.Index("@");
3335 if (index > 0) outputFile.Remove(index);
3336 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3337 }
3338 delete list;
3339 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3340 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3341// gSystem->Exec("cat stdout");
5513444a 3342 return kFALSE;
c57f56b7 3343 }
5513444a 3344 // Check if submitting is managed by LPM manager
a3e84053 3345 if (fProductionMode) {
7ac61826 3346 //TString prodfile = fJDLName;
3347 //prodfile.ReplaceAll(".jdl", ".prod");
3348 //WriteProductionFile(prodfile);
5513444a 3349 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3350 return kFALSE;
3351 }
a8739e8a 3352 // Submit AliEn job(s)
d2a409b2 3353 gGrid->Cd(fGridOutputDir);
a8739e8a 3354 TGridResult *res;
c57f56b7 3355 TString jobID = "";
d2a409b2 3356 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3357 // Submit a given xml or a set of runs
a8739e8a 3358 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3359 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3360 if (res) {
3361 const char *cjobId = res->GetKey(0,"jobId");
3362 if (!cjobId) {
a03be957 3363 gGrid->Stdout();
3364 gGrid->Stderr();
a8739e8a 3365 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3366 return kFALSE;
a8739e8a 3367 } else {
3368 Info("StartAnalysis", "\n_______________________________________________________________________ \
3369 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3370 \n_______________________________________________________________________",
3371 fJDLName.Data(), cjobId);
3372 jobID = cjobId;
3373 }
3374 delete res;
a03be957 3375 } else {
3376 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3377 return kFALSE;
a8739e8a 3378 }
3379 } else {
d2a409b2 3380 // Submit for a range of enumeration of runs.
a03be957 3381 if (!Submit()) return kFALSE;
c57f56b7 3382 }
a8739e8a 3383
c57f56b7 3384 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3385 \n You may exit at any time and terminate the job later using the option <terminate> \
3386 \n ##################################################################################", jobID.Data());
bb885a9e 3387 gSystem->Exec("aliensh");
5513444a 3388 return kTRUE;
c57f56b7 3389}
3390
f790bc1b 3391//______________________________________________________________________________
3392const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3393{
3394// Get a comma-separated list of output files of the requested type.
3395// Type can be (case unsensitive):
3396// aod - list of aod files (std, extensions and filters)
3397// out - list of output files connected to containers (but not aod's or extras)
3398// ext - list of extra files registered to the manager
3399// ter - list of files produced in terminate
3400 static TString files;
3401 files = "";
3402 TString stype = type;
3403 stype.ToLower();
3404 TString aodfiles, extra;
3405 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3406 if (!mgr) {
3407 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3408 return files.Data();
3409 }
3410 if (mgr->GetOutputEventHandler()) {
3411 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3412 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3413 if (!extraaod.IsNull()) {
3414 aodfiles += ",";
3415 aodfiles += extraaod;
3416 }
3417 }
3418 if (stype.Contains("aod")) {
3419 files = aodfiles;
3420 if (stype == "aod") return files.Data();
3421 }
3422 // Add output files that are not in the list of AOD files
3423 TString outputfiles = "";
3424 TIter next(mgr->GetOutputs());
3425 AliAnalysisDataContainer *output;
3426 const char *filename = 0;
3427 while ((output=(AliAnalysisDataContainer*)next())) {
3428 filename = output->GetFileName();
3429 if (!(strcmp(filename, "default"))) continue;
3430 if (outputfiles.Contains(filename)) continue;
3431 if (aodfiles.Contains(filename)) continue;
3432 if (!outputfiles.IsNull()) outputfiles += ",";
3433 outputfiles += filename;
3434 }
3435 if (stype.Contains("out")) {
3436 if (!files.IsNull()) files += ",";
3437 files += outputfiles;
3438 if (stype == "out") return files.Data();
3439 }
3440 // Add extra files registered to the analysis manager
3441 TString sextra;
3442 extra = mgr->GetExtraFiles();
3443 if (!extra.IsNull()) {
3444 extra.Strip();
3445 extra.ReplaceAll(" ", ",");
3446 TObjArray *fextra = extra.Tokenize(",");
3447 TIter nextx(fextra);
3448 TObject *obj;
3449 while ((obj=nextx())) {
3450 if (aodfiles.Contains(obj->GetName())) continue;
3451 if (outputfiles.Contains(obj->GetName())) continue;
3452 if (sextra.Contains(obj->GetName())) continue;
3453 if (!sextra.IsNull()) sextra += ",";
3454 sextra += obj->GetName();
3455 }
3456 delete fextra;
3457 if (stype.Contains("ext")) {
3458 if (!files.IsNull()) files += ",";
3459 files += sextra;
3460 }
3461 }
3462 if (stype == "ext") return files.Data();
3463 TString termfiles;
3464 if (!fTerminateFiles.IsNull()) {
3465 fTerminateFiles.Strip();
3466 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3467 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3468 TIter nextx(fextra);
3469 TObject *obj;
3470 while ((obj=nextx())) {
3471 if (aodfiles.Contains(obj->GetName())) continue;
3472 if (outputfiles.Contains(obj->GetName())) continue;
3473 if (termfiles.Contains(obj->GetName())) continue;
3474 if (sextra.Contains(obj->GetName())) continue;
3475 if (!termfiles.IsNull()) termfiles += ",";
3476 termfiles += obj->GetName();
3477 }
3478 delete fextra;
3479 }
3480 if (stype.Contains("ter")) {
3481 if (!files.IsNull() && !termfiles.IsNull()) {
3482 files += ",";
3483 files += termfiles;
3484 }
3485 }
3486 return files.Data();
3487}
3488
d2a409b2 3489//______________________________________________________________________________
a03be957 3490Bool_t AliAnalysisAlien::Submit()
d2a409b2 3491{
3492// Submit all master jobs.
3493 Int_t nmasterjobs = fInputFiles->GetEntries();
3494 Long_t tshoot = gSystem->Now();
a03be957 3495 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3496 while (fNsubmitted < nmasterjobs) {
3497 Long_t now = gSystem->Now();
3498 if ((now-tshoot)>30000) {
3499 tshoot = now;
a03be957 3500 if (!SubmitNext()) return kFALSE;
d2a409b2 3501 }
3502 }
a03be957 3503 return kTRUE;
d2a409b2 3504}
3505
0f389141 3506//______________________________________________________________________________
a03be957 3507Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3508{
3509// Submit all merging jobs.
dc030a24 3510 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3511 gGrid->Cd(fGridOutputDir);
3512 TString mergeJDLName = fExecutable;
3513 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3514 if (!fInputFiles) {
3515 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3516 return kFALSE;
3517 }
0f389141 3518 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3519 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3520 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3521 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3522 if (fOutputToRunNo) {
3523 // The output directory is the run number
3524 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3525 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3526 } else {
17b4e9a4 3527 if (!fRunNumbers.Length() && !fRunRange[0]) {
3528 // The output directory is the grid outdir
3529 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3530 runOutDir = fGridOutputDir;
3531 } else {
3532 // The output directory is the master number in 3 digits format
3533 printf("### Submitting merging job for master <%03d>\n", i);
3534 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3535 }
7c2cd90a 3536 }
3537 // Check now the number of merging stages.
2b222f7c 3538 TObjArray *list = fOutputFiles.Tokenize(",");
3539 TIter next(list);
3540 TObjString *str;
3541 TString outputFile;
3542 while((str=(TObjString*)next())) {
3543 outputFile = str->GetString();
3544 Int_t index = outputFile.Index("@");
3545 if (index > 0) outputFile.Remove(index);
37d21c01 3546 if (!fMergeExcludes.Contains(outputFile) &&
3547 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3548 }
3549 delete list;
b3e07543 3550 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3551 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3552 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3553 }
a03be957 3554 if (!ntosubmit) return kTRUE;
9f020952 3555 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3556 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3557 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3558 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3559 \n ################################################################################################################");
0f389141 3560 gSystem->Exec("aliensh");
a03be957 3561 return kTRUE;
0f389141 3562}
3563
d2a409b2 3564//______________________________________________________________________________
a03be957 3565Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3566{
08d5b699 3567// Submit next bunch of master jobs if the queue is free. The first master job is
3568// submitted right away, while the next will not be unless the previous was split.
3569// The plugin will not submit new master jobs if there are more that 500 jobs in
3570// waiting phase.
d2a409b2 3571 static Bool_t iscalled = kFALSE;
3572 static Int_t firstmaster = 0;
3573 static Int_t lastmaster = 0;
3574 static Int_t npermaster = 0;
a03be957 3575 if (iscalled) return kTRUE;
d2a409b2 3576 iscalled = kTRUE;
3577 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3578 Int_t ntosubmit = 0;
3579 TGridResult *res;
3580 TString jobID = "";
3bdcb562 3581 Int_t nmasterjobs = fInputFiles->GetEntries();
3582 if (!fNsubmitted) {
3583 ntosubmit = 1;
3584 if (!IsUseSubmitPolicy()) {
90d50a8c 3585 if (nmasterjobs>5)
3bdcb562 3586 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3587 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3588 ntosubmit = nmasterjobs;
3589 }
3590 } else {
d2a409b2 3591 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3592 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3593 // If last master not split, just return
a03be957 3594 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3595 // No more than 100 waiting jobs
08d5b699 3596 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3597 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3598 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3599 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3600 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3601 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3602 }
d2a409b2 3603 for (Int_t i=0; i<ntosubmit; i++) {
3604 // Submit for a range of enumeration of runs.
a03be957 3605 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3606 TString query;
cd11251e 3607 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3608 runOutDir.ReplaceAll(".xml", "");
3609 if (fOutputToRunNo)
3610 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3611 else
3612 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3613 printf("********* %s\n",query.Data());
d2a409b2 3614 res = gGrid->Command(query);
3615 if (res) {
98ca124f 3616 TString cjobId1 = res->GetKey(0,"jobId");
3617 if (!cjobId1.Length()) {
d2a409b2 3618 iscalled = kFALSE;
a03be957 3619 gGrid->Stdout();
3620 gGrid->Stderr();
3621 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3622 return kFALSE;
d2a409b2 3623 } else {
3624 Info("StartAnalysis", "\n_______________________________________________________________________ \
3625 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3626 \n_______________________________________________________________________",
98ca124f 3627 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 3628 jobID += cjobId1;
3629 jobID += " ";
98ca124f 3630 lastmaster = cjobId1.Atoi();
d2a409b2 3631 if (!firstmaster) firstmaster = lastmaster;
3632 fNsubmitted++;
3633 }
3634 delete res;
a03be957 3635 } else {
3636 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3637 return kFALSE;
d2a409b2 3638 }
3639 }
3640 iscalled = kFALSE;
a03be957 3641 return kTRUE;
d2a409b2 3642}
3643
c57f56b7 3644//______________________________________________________________________________
3645void AliAnalysisAlien::WriteAnalysisFile()
3646{
f10e8481 3647// Write current analysis manager into the file <analysisFile>
3648 TString analysisFile = fExecutable;
3649 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3650 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3651 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3652 if (!mgr || !mgr->IsInitialized()) {
3653 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3654 return;
3655 }
3656 // Check analysis type
3657 TObject *handler;
3658 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3659 handler = (TObject*)mgr->GetInputEventHandler();
3660 if (handler) {
65eb22f8 3661 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3662 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3663 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3664 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3665 } else {
3666 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3667 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3668 }
c57f56b7 3669 }
3670 TDirectory *cdir = gDirectory;
f10e8481 3671 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3672 if (file) {
e16a394c 3673 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3674 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3675 // Unless merging makes no sense
3676 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3677 mgr->Write();
3678 delete file;
fe2d7fc2 3679 // Enable termination for local jobs
3680 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3681 }
3682 if (cdir) cdir->cd();
f10e8481 3683 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3684 }
3685 Bool_t copy = kTRUE;
d3b18c4c 3686 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3687 if (copy) {
3688 CdWork();
3689 TString workdir = gGrid->GetHomeDirectory();
3690 workdir += fGridWorkingDir;
d3b18c4c 3691 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3692 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3693 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3694 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3695 }
3696}
3697
3698//______________________________________________________________________________
3699void AliAnalysisAlien::WriteAnalysisMacro()
3700{
3701// Write the analysis macro that will steer the analysis in grid mode.
3702 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3703 ofstream out;
3704 out.open(fAnalysisMacro.Data(), ios::out);
3705 if (!out.good()) {
3706 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3707 return;
3708 }
5b273635 3709 Bool_t hasSTEERBase = kFALSE;
3710 Bool_t hasESD = kFALSE;
3711 Bool_t hasAOD = kFALSE;
3712 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3713 Bool_t hasOADB = kFALSE;
5b273635 3714 Bool_t hasANALYSISalice = kFALSE;
3715 Bool_t hasCORRFW = kFALSE;
c57f56b7 3716 TString func = fAnalysisMacro;
3717 TString type = "ESD";
3718 TString comment = "// Analysis using ";
c85cfc0f 3719 if (IsUseMCchain()) {
3720 type = "MC";
3721 comment += "MC";
3722 } else {
3723 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3724 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3725 type = "AOD";
3726 comment += "AOD";
3727 }
3728 }
0df6ccf2 3729 if (type!="AOD" && fFriendChainName!="") {
3730 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3731 return;
3732 }
c57f56b7 3733 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3734 else comment += " data";
3735 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3736 func.ReplaceAll(".C", "");
3737 out << "void " << func.Data() << "()" << endl;
3738 out << "{" << endl;
3739 out << comment.Data() << endl;
3740 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3741 out << " TStopwatch timer;" << endl;
3742 out << " timer.Start();" << endl << endl;
13f0c00b 3743 // Change temp directory to current one
fdee57f1 3744 if (!IsLocalTest()) {
3745 out << "// connect to AliEn and make the chain" << endl;
3746 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
3747 }
13f0c00b 3748 out << "// Set temporary merging directory to current one" << endl;
3749 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3750 out << "// Set temporary compilation directory to current one" << endl;
3751 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3752 // Reset existing include path
3753 out << "// Reset existing include path and add current directory first in the search" << endl;
3754 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3755 if (!fExecutableCommand.Contains("aliroot")) {
3756 out << "// load base root libraries" << endl;
3757 out << " gSystem->Load(\"libTree\");" << endl;
3758 out << " gSystem->Load(\"libGeom\");" << endl;
3759 out << " gSystem->Load(\"libVMC\");" << endl;
3760 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3761 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3762 }
d5c6455a 3763 if (fAdditionalRootLibs.Length()) {
47a4137d 3764 // in principle libtree /lib geom libvmc etc. can go into this list, too
3765 out << "// Add aditional libraries" << endl;
3766 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3767 TIter next(list);
3768 TObjString *str;
3769 while((str=(TObjString*)next())) {
3770 if (str->GetString().Contains(".so"))
3771 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3772 }
47a4137d 3773 if (list) delete list;
d5c6455a 3774 }
57377eb5 3775 out << "// Load analysis framework libraries" << endl;
652561d5 3776 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3777 if (!fPackages) {
f790bc1b 3778 if (!fExecutableCommand.Contains("aliroot")) {
3779 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3780 out << " gSystem->Load(\"libESD\");" << endl;
3781 out << " gSystem->Load(\"libAOD\");" << endl;
3782 }
4e5c5506 3783 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3784 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3785 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3786 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3787 } else {
4e5c5506 3788 TIter next(fPackages);
3789 TObject *obj;
57377eb5 3790 TString pkgname;
57377eb5 3791 while ((obj=next())) {
3792 pkgname = obj->GetName();
4478e6f1 3793 if (pkgname == "STEERBase" ||
3794 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3795 if (pkgname == "ESD" ||
3796 pkgname == "ESD.par") hasESD = kTRUE;
3797 if (pkgname == "AOD" ||
3798 pkgname == "AOD.par") hasAOD = kTRUE;
3799 if (pkgname == "ANALYSIS" ||
3800 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 3801 if (pkgname == "OADB" ||
3802 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 3803 if (pkgname == "ANALYSISalice" ||
3804 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3805 if (pkgname == "CORRFW" ||
3806 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3807 }
3808 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3809 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3810 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3811 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3812 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3813 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3814 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3815 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3816 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 3817 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
3818 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 3819 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3820 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3821 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3822 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3823 out << "// Compile other par packages" << endl;
3824 next.Reset();
fcc9bb6f 3825 while ((obj=next())) {
3826 pkgname = obj->GetName();
4478e6f1 3827 if (pkgname == "STEERBase" ||
3828 pkgname == "STEERBase.par" ||
3829 pkgname == "ESD" ||
3830 pkgname == "ESD.par" ||
3831 pkgname == "AOD" ||
3832 pkgname == "AOD.par" ||
3833 pkgname == "ANALYSIS" ||
3834 pkgname == "ANALYSIS.par" ||
7ea40ac8 3835 pkgname == "OADB" ||
3836 pkgname == "OADB.par" ||
4478e6f1 3837 pkgname == "ANALYSISalice" ||
3838 pkgname == "ANALYSISalice.par" ||
3839 pkgname == "CORRFW" ||
3840 pkgname == "CORRFW.par") continue;
5b273635 3841 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3842 }
4e5c5506 3843 }
b3e07543 3844 out << "// include path" << endl;
3845 // Get the include path from the interpreter and remove entries pointing to AliRoot
3846 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3847 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3848 out << " TIter nextpath(listpaths);" << endl;
3849 out << " TObjString *pname;" << endl;
3850 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3851 out << " TString current = pname->GetName();" << endl;
3852 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3853 out << " gSystem->AddIncludePath(current);" << endl;
3854 out << " }" << endl;
3855 out << " if (listpaths) delete listpaths;" << endl;
3856 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3857 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3858 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3859 if (fAdditionalLibs.Length()) {
3860 out << "// Add aditional AliRoot libraries" << endl;
3861 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3862 TIter next(list);
3863 TObjString *str;
3864 while((str=(TObjString*)next())) {
3865 if (str->GetString().Contains(".so"))
3866 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3867 if (str->GetString().Contains(".par"))
3868 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3869 }
3870 if (list) delete list;
3871 }
3872 out << endl;
c57f56b7 3873 out << "// analysis source to be compiled at runtime (if any)" << endl;
3874 if (fAnalysisSource.Length()) {
3875 TObjArray *list = fAnalysisSource.Tokenize(" ");
3876 TIter next(list);
3877 TObjString *str;
3878 while((str=(TObjString*)next())) {
3879 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3880 }
3881 if (list) delete list;
3882 }
3883 out << endl;
b3e07543 3884// out << " printf(\"Currently load libraries:\\n\");" << endl;
3885// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3886 if (fFastReadOption) {
d3339be3 3887 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3888 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3889 out << "// fast xrootd reading enabled" << endl;
3890 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3891 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3892 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3893 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3894 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3895 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 3896 }
c57f56b7 3897 out << "// read the analysis manager from file" << endl;
f10e8481 3898 TString analysisFile = fExecutable;
3899 analysisFile.ReplaceAll(".sh", ".root");
105631b5 3900 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
3901 << analysisFile << "\");" << endl;
3902 out << " if (!mgr) return;" << endl;
21ca8e59 3903 if (IsLocalTest()) {
3904 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
3905 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 3906 if (fFileForTestMode.IsNull())
3907 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
3908 else
3909 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 3910 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4b5d9e21 3911 if (!fFriendChainName.IsNull())
3912 out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\");" << endl;
21ca8e59 3913 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 3914 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 3915 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 3916 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
3917 } else {
7a366e0b 3918 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 3919 out << " mgr->SetNSysInfo(100);" << endl;
3920 }
21ca8e59 3921 }
c57f56b7 3922 out << " mgr->PrintStatus();" << endl;
52b6a92b 3923 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3924 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3925 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3926 } else {
9a8288ff 3927 if (TestBit(AliAnalysisGrid::kTest))
3928 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3929 else
3930 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3931 }
3932 }
21ca8e59 3933 if (!IsLocalTest()) {
242accb2 3934 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 3935 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
3936 } else {
3937 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 3938 }
f7498086 3939 out << " timer.Stop();" << endl;
3940 out << " timer.Print();" << endl;
c57f56b7 3941 out << "}" << endl << endl;
21ca8e59 3942 if (!IsLocalTest()) {
fcc9bb6f 3943 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3944 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3945 out << "{" << endl;
3946 out << "// Create a chain using url's from xml file" << endl;
242accb2 3947 out << " TString filename;" << endl;
3948 out << " Int_t run = 0;" << endl;
c85cfc0f 3949 if (IsUseMCchain()) {
3950 out << " TString treename = \"TE\";" << endl;
3951 } else {
3952 out << " TString treename = type;" << endl;
3953 out << " treename.ToLower();" << endl;
3954 out << " treename += \"Tree\";" << endl;
3955 }
e02fee64 3956 out << " printf(\"***************************************\\n\");" << endl;
3957 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3958 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3959 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3960 out << " if (!coll) {" << endl;
3961 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3962 out << " return NULL;" << endl;
3963 out << " }" << endl;
242accb2 3964 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3965 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3966 if(fFriendChainName!="") {
3967 out << " TChain *chainFriend = new TChain(treename);" << endl;
3968 }
c57f56b7 3969 out << " coll->Reset();" << endl;
0df6ccf2 3970 out << " while (coll->Next()) {" << endl;
242accb2 3971 out << " filename = coll->GetTURL("");" << endl;
3972 out << " if (mgr) {" << endl;
3973 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3974 out << " if (nrun && nrun != run) {" << endl;
3975 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3976 out << " mgr->SetRunFromPath(nrun);" << endl;
3977 out << " run = nrun;" << endl;
3978 out << " }" << endl;
3979 out << " }" << endl;
3980 out << " chain->Add(filename);" << endl;
0df6ccf2 3981 if(fFriendChainName!="") {
3982 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
4b5d9e21 3983 out << " if (fileFriend.Index(\"#\") > -1) fileFriend.Remove(fileFriend.Index(\"#\"));" << endl;
3984 out << " fileFriend = gSystem->DirName(fileFriend);" << endl;
3985 out << " fileFriend += \"/\";" << endl;
3986 out << " fileFriend += \"" << fFriendChainName << "\";";
3987 out << " TFile *file = TFile::Open(fileFriend);" << endl;
3988 out << " if (file) {" << endl;
3989 out << " file->Close();" << endl;
3990 out << " chainFriend->Add(fileFriend.Data());" << endl;
3991 out << " } else {" << endl;
3992 out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
3993 out << " return 0;" << endl;
3994 out << " }" << endl;
0df6ccf2 3995 }
3996 out << " }" << endl;
c57f56b7 3997 out << " if (!chain->GetNtrees()) {" << endl;
3998 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3999 out << " return NULL;" << endl;
4000 out << " }" << endl;
0df6ccf2 4001 if(fFriendChainName!="") {
4002 out << " chain->AddFriend(chainFriend);" << endl;
4003 }
c57f56b7 4004 out << " return chain;" << endl;
fcc9bb6f 4005 out << "}" << endl << endl;
c57f56b7 4006 }
5b273635 4007 if (hasANALYSISalice) {
4008 out <<"//________________________________________________________________________________" << endl;
4009 out << "Bool_t SetupPar(const char *package) {" << endl;
4010 out << "// Compile the package and set it up." << endl;
4011 out << " TString pkgdir = package;" << endl;
4012 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4013 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4014 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4015 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4016 out << " // Check for BUILD.sh and execute" << endl;
4017 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4018 out << " printf(\"*******************************\\n\");" << endl;
4019 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4020 out << " printf(\"*******************************\\n\");" << endl;
4021 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4022 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4023 out << " gSystem->ChangeDirectory(cdir);" << endl;
4024 out << " return kFALSE;" << endl;
4025 out << " }" << endl;
4026 out << " } else {" << endl;
4027 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4028 out << " gSystem->ChangeDirectory(cdir);" << endl;
4029 out << " return kFALSE;" << endl;
4030 out << " }" << endl;
4031 out << " // Check for SETUP.C and execute" << endl;
4032 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4033 out << " printf(\"*******************************\\n\");" << endl;
4034 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4035 out << " printf(\"*******************************\\n\");" << endl;
4036 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4037 out << " } else {" << endl;
4038 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4039 out << " gSystem->ChangeDirectory(cdir);" << endl;
4040 out << " return kFALSE;" << endl;
4041 out << " }" << endl;
4042 out << " // Restore original workdir" << endl;
4043 out << " gSystem->ChangeDirectory(cdir);" << endl;
4044 out << " return kTRUE;" << endl;
4045 out << "}" << endl;
4046 }
c57f56b7 4047 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4048 }
4049 Bool_t copy = kTRUE;
d3b18c4c 4050 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4051 if (copy) {
4052 CdWork();
4053 TString workdir = gGrid->GetHomeDirectory();
4054 workdir += fGridWorkingDir;
4055 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4056 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4057// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4058 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4059 Form("alien://%s/%s", workdir.Data(),
4060 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4061 }
4062}
4063
0f389141 4064//______________________________________________________________________________
4065void AliAnalysisAlien::WriteMergingMacro()
4066{
4067// Write a macro to merge the outputs per master job.
4068 if (!fMergeViaJDL) return;
4069 if (!fOutputFiles.Length()) {
4070 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4071 return;
4072 }
4073 TString mergingMacro = fExecutable;
4074 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4075 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4076 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4077 ofstream out;
4078 out.open(mergingMacro.Data(), ios::out);
4079 if (!out.good()) {
4080 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4081 return;
4082 }
5b273635 4083 Bool_t hasSTEERBase = kFALSE;
4084 Bool_t hasESD = kFALSE;
4085 Bool_t hasAOD = kFALSE;
4086 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4087 Bool_t hasOADB = kFALSE;
5b273635 4088 Bool_t hasANALYSISalice = kFALSE;
4089 Bool_t hasCORRFW = kFALSE;
0f389141 4090 TString func = mergingMacro;
4091 TString comment;
4092 func.ReplaceAll(".C", "");
f47d5cb4 4093 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4094 out << "{" << endl;
4095 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4096 out << " TStopwatch timer;" << endl;
4097 out << " timer.Start();" << endl << endl;
b3e07543 4098 // Reset existing include path
4099 out << "// Reset existing include path and add current directory first in the search" << endl;
4100 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4101 if (!fExecutableCommand.Contains("aliroot")) {
4102 out << "// load base root libraries" << endl;
4103 out << " gSystem->Load(\"libTree\");" << endl;
4104 out << " gSystem->Load(\"libGeom\");" << endl;
4105 out << " gSystem->Load(\"libVMC\");" << endl;
4106 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4107 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4108 }
0f389141 4109 if (fAdditionalRootLibs.Length()) {
4110 // in principle libtree /lib geom libvmc etc. can go into this list, too
4111 out << "// Add aditional libraries" << endl;
4112 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4113 TIter next(list);
4114 TObjString *str;
4115 while((str=(TObjString*)next())) {
4116 if (str->GetString().Contains(".so"))
4117 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4118 }
4119 if (list) delete list;
4120 }
0f389141 4121 out << "// Load analysis framework libraries" << endl;
4122 if (!fPackages) {
7c2cd90a 4123 if (!fExecutableCommand.Contains("aliroot")) {
4124 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4125 out << " gSystem->Load(\"libESD\");" << endl;
4126 out << " gSystem->Load(\"libAOD\");" << endl;
4127 }
0f389141 4128 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4129 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4130 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4131 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4132 } else {
4133 TIter next(fPackages);
4134 TObject *obj;
4135 TString pkgname;
5b273635 4136 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4137 while ((obj=next())) {
4138 pkgname = obj->GetName();
4139 if (pkgname == "STEERBase" ||
4140 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4141 if (pkgname == "ESD" ||
4142 pkgname == "ESD.par") hasESD = kTRUE;
4143 if (pkgname == "AOD" ||
4144 pkgname == "AOD.par") hasAOD = kTRUE;
4145 if (pkgname == "ANALYSIS" ||
4146 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4147 if (pkgname == "OADB" ||
4148 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4149 if (pkgname == "ANALYSISalice" ||
4150 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4151 if (pkgname == "CORRFW" ||
4152 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4153 }
5b273635 4154 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4155 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4156 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4157 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4158 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4159 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4160 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4161 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4162 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4163 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4164 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4165 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4166 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4167 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4168 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4169 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4170 out << "// Compile other par packages" << endl;
4171 next.Reset();
4172 while ((obj=next())) {
4173 pkgname = obj->GetName();
4174 if (pkgname == "STEERBase" ||
4175 pkgname == "STEERBase.par" ||
4176 pkgname == "ESD" ||
4177 pkgname == "ESD.par" ||
4178 pkgname == "AOD" ||
4179 pkgname == "AOD.par" ||
4180 pkgname == "ANALYSIS" ||
4181 pkgname == "ANALYSIS.par" ||
7ea40ac8 4182 pkgname == "OADB" ||
4183 pkgname == "OADB.par" ||
0f389141 4184 pkgname == "ANALYSISalice" ||
4185 pkgname == "ANALYSISalice.par" ||
4186 pkgname == "CORRFW" ||
4187 pkgname == "CORRFW.par") continue;
5b273635 4188 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4189 }
4190 }
b3e07543 4191 out << "// include path" << endl;
4192 // Get the include path from the interpreter and remove entries pointing to AliRoot
4193 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4194 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4195 out << " TIter nextpath(listpaths);" << endl;
4196 out << " TObjString *pname;" << endl;
4197 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4198 out << " TString current = pname->GetName();" << endl;
4199 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4200 out << " gSystem->AddIncludePath(current);" << endl;
4201 out << " }" << endl;
4202 out << " if (listpaths) delete listpaths;" << endl;
4203 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4204 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4205 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4206 if (fAdditionalLibs.Length()) {
4207 out << "// Add aditional AliRoot libraries" << endl;
4208 TObjArray *list = fAdditionalLibs.Tokenize(" ");
4209 TIter next(list);
4210 TObjString *str;
4211 while((str=(TObjString*)next())) {
4212 if (str->GetString().Contains(".so"))
4213 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4214 }
4215 if (list) delete list;
4216 }
4217 out << endl;
4218 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4219 if (fAnalysisSource.Length()) {
4220 TObjArray *list = fAnalysisSource.Tokenize(" ");
4221 TIter next(list);
4222 TObjString *str;
4223 while((str=(TObjString*)next())) {
4224 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4225 }
4226 if (list) delete list;
4227 }
149d288c 4228 out << endl;
4229
0f389141 4230 if (fFastReadOption) {
4231 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4232 out << "// fast xrootd reading enabled" << endl;
4233 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4234 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4235 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4236 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4237 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4238 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4239 }
4240 // Change temp directory to current one
fdee57f1 4241 out << "// Connect to AliEn" << endl;
4242 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
e8b839ab 4243 out << "// Set temporary merging directory to current one" << endl;
4244 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4245 out << "// Set temporary compilation directory to current one" << endl;
4246 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
7c2cd90a 4247 out << " TString outputDir = dir;" << endl;
f790bc1b 4248 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4249 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4250 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4251 out << " TIter *iter = new TIter(list);" << endl;
4252 out << " TObjString *str;" << endl;
a2f5fc01 4253 out << " TString outputFile;" << endl;
0f389141 4254 out << " Bool_t merged = kTRUE;" << endl;
4255 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4256 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4257 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4258 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4259 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4260 out << " // Skip already merged outputs" << endl;
a2f5fc01 4261 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
4262 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 4263 out << " continue;" << endl;
4264 out << " }" << endl;
a2f5fc01 4265 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4266 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4267 out << " if (!merged) {" << endl;
a2f5fc01 4268 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4269 out << " return;" << endl;
0f389141 4270 out << " }" << endl;
4271 out << " }" << endl;
7c2cd90a 4272 out << " // all outputs merged, validate" << endl;
4273 out << " ofstream out;" << endl;
4274 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4275 out << " out.close();" << endl;
4276 out << " // read the analysis manager from file" << endl;
f866cba5 4277 TString analysisFile = fExecutable;
4278 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 4279 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
105631b5 4280 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4281 << analysisFile << "\");" << endl;
4282 out << " if (!mgr) return;" << endl;
5c4250fc 4283 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4284 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4285 out << " mgr->PrintStatus();" << endl;
4286 if (AliAnalysisManager::GetAnalysisManager()) {
4287 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
4288 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4289 } else {
9a8288ff 4290 if (TestBit(AliAnalysisGrid::kTest))
4291 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4292 else
4293 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4294 }
4295 }
9a8288ff 4296 out << " TTree *tree = NULL;" << endl;
4297 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4298 out << "}" << endl << endl;
5b273635 4299 if (hasANALYSISalice) {
4300 out <<"//________________________________________________________________________________" << endl;
4301 out << "Bool_t SetupPar(const char *package) {" << endl;
4302 out << "// Compile the package and set it up." << endl;
4303 out << " TString pkgdir = package;" << endl;
4304 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4305 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4306 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4307 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4308 out << " // Check for BUILD.sh and execute" << endl;
4309 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4310 out << " printf(\"*******************************\\n\");" << endl;
4311 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4312 out << " printf(\"*******************************\\n\");" << endl;
4313 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4314 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4315 out << " gSystem->ChangeDirectory(cdir);" << endl;
4316 out << " return kFALSE;" << endl;
4317 out << " }" << endl;
4318 out << " } else {" << endl;
4319 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4320 out << " gSystem->ChangeDirectory(cdir);" << endl;
4321 out << " return kFALSE;" << endl;
4322 out << " }" << endl;
4323 out << " // Check for SETUP.C and execute" << endl;
4324 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4325 out << " printf(\"*******************************\\n\");" << endl;
4326 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4327 out << " printf(\"*******************************\\n\");" << endl;
4328 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4329 out << " } else {" << endl;
4330 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4331 out << " gSystem->ChangeDirectory(cdir);" << endl;
4332 out << " return kFALSE;" << endl;
4333 out << " }" << endl;
4334 out << " // Restore original workdir" << endl;
4335 out << " gSystem->ChangeDirectory(cdir);" << endl;
4336 out << " return kTRUE;" << endl;
4337 out << "}" << endl;
4338 }
0f389141 4339 }
4340 Bool_t copy = kTRUE;
d3b18c4c 4341 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4342 if (copy) {
4343 CdWork();
4344 TString workdir = gGrid->GetHomeDirectory();
4345 workdir += fGridWorkingDir;
4346 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4347 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4348// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4349 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4350 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4351 }
4352}
4353
4354//______________________________________________________________________________
4355Bool_t AliAnalysisAlien::SetupPar(const char *package)
4356{
205b201f 4357// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4358// Note that for loading the compiled library. The current directory should have precedence in
4359// LD_LIBRARY_PATH
4360 TString pkgdir = package;
4361 pkgdir.ReplaceAll(".par","");
b50a41c3 4362 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4363 TString cdir = gSystem->WorkingDirectory();
4364 gSystem->ChangeDirectory(pkgdir);
4365 // Check for BUILD.sh and execute
4366 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4367 printf("**************************************************\n");
4368 printf("*** Building PAR archive %s\n", package);
4369 printf("**************************************************\n");
4370 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4371 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4372 gSystem->ChangeDirectory(cdir);
4373 return kFALSE;
4374 }
4375 } else {
4376 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4377 gSystem->ChangeDirectory(cdir);
4378 return kFALSE;
4379 }
4380 // Check for SETUP.C and execute
4381 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4382 printf("**************************************************\n");
4383 printf("*** Setup PAR archive %s\n", package);
4384 printf("**************************************************\n");
4385 gROOT->Macro("PROOF-INF/SETUP.C");
4386 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4387 } else {
4388 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4389 gSystem->ChangeDirectory(cdir);
4390 return kFALSE;
4391 }
4392 // Restore original workdir
4393 gSystem->ChangeDirectory(cdir);
4394 return kTRUE;
4395}
4396
c57f56b7 4397//______________________________________________________________________________
4398void AliAnalysisAlien::WriteExecutable()
4399{
4400// Generate the alien executable script.
4401 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4402 ofstream out;
4403 out.open(fExecutable.Data(), ios::out);
4404 if (out.bad()) {
5513444a 4405 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4406 return;
4407 }
4408 out << "#!/bin/bash" << endl;
2a4d5166 4409 // Make sure we can properly compile par files
4410 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4411 out << "echo \"=========================================\"" << endl;
4412 out << "echo \"############## PATH : ##############\"" << endl;
4413 out << "echo $PATH" << endl;
4414 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4415 out << "echo $LD_LIBRARY_PATH" << endl;
4416 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4417 out << "echo $ROOTSYS" << endl;
4418 out << "echo \"############## which root : ##############\"" << endl;
4419 out << "which root" << endl;
4420 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4421 out << "echo $ALICE_ROOT" << endl;
4422 out << "echo \"############## which aliroot : ##############\"" << endl;
4423 out << "which aliroot" << endl;
9c5ddadc 4424 out << "echo \"############## system limits : ##############\"" << endl;
4425 out << "ulimit -a" << endl;
4426 out << "echo \"############## memory : ##############\"" << endl;
4427 out << "free -m" << endl;
c57f56b7 4428 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4429 out << fExecutableCommand << " ";
370c9880 4430 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4431 out << "RET=$?" << endl;
4432 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4433 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4434 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4435 out << " let sig=\"$RET - 128\""<<endl;
4436 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4437 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4438 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4439 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4440 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4441 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4442 out << " fi"<<endl;
4443 out << " exit $RET"<< endl;
4444 out << "fi" << endl << endl ;
4445 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4446 out << "echo \"############## memory after: ##############\"" << endl;
4447 out << "free -m" << endl;
c57f56b7 4448 }
4449 Bool_t copy = kTRUE;
d3b18c4c 4450 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4451 if (copy) {
4452 CdWork();
4453 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4454 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4455 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4456 workdir += fGridWorkingDir;
4457 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4458 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4459 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4460// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4461 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4462 executable.Data())) Fatal("","Terminating");
c57f56b7 4463 }
4464}
4465
0f389141 4466//______________________________________________________________________________
4467void AliAnalysisAlien::WriteMergeExecutable()
4468{
4469// Generate the alien executable script for the merging job.
4470 if (!fMergeViaJDL) return;
4471 TString mergeExec = fExecutable;
4472 mergeExec.ReplaceAll(".sh", "_merge.sh");
4473 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4474 ofstream out;
4475 out.open(mergeExec.Data(), ios::out);
4476 if (out.bad()) {
4477 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4478 return;
4479 }
4480 out << "#!/bin/bash" << endl;
2a4d5166 4481 // Make sure we can properly compile par files
4482 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4483 out << "echo \"=========================================\"" << endl;
4484 out << "echo \"############## PATH : ##############\"" << endl;
4485 out << "echo $PATH" << endl;
4486 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4487 out << "echo $LD_LIBRARY_PATH" << endl;
4488 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4489 out << "echo $ROOTSYS" << endl;
4490 out << "echo \"############## which root : ##############\"" << endl;
4491 out << "which root" << endl;
4492 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4493 out << "echo $ALICE_ROOT" << endl;
4494 out << "echo \"############## which aliroot : ##############\"" << endl;
4495 out << "which aliroot" << endl;
4496 out << "echo \"############## system limits : ##############\"" << endl;
4497 out << "ulimit -a" << endl;
4498 out << "echo \"############## memory : ##############\"" << endl;
4499 out << "free -m" << endl;
4500 out << "echo \"=========================================\"" << endl << endl;
0f389141 4501 TString mergeMacro = fExecutable;
4502 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4503 if (IsOneStageMerging())
4504 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4505 else
f47d5cb4 4506 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4507 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4508 out << "RET=$?" << endl;
4509 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4510 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4511 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4512 out << " let sig=\"$RET - 128\""<<endl;
4513 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4514 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4515 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4516 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4517 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4518 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4519 out << " fi"<<endl;
4520 out << " exit $RET"<< endl;
4521 out << "fi" << endl << endl ;
0f389141 4522 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4523 out << "echo \"############## memory after: ##############\"" << endl;
4524 out << "free -m" << endl;
0f389141 4525 }
4526 Bool_t copy = kTRUE;
d3b18c4c 4527 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4528 if (copy) {
4529 CdWork();
4530 TString workdir = gGrid->GetHomeDirectory();
4531 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4532 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4533 workdir += fGridWorkingDir;
4534 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4535 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4536 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4537// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4538 if (!copyLocal2Alien("WriteMergeExecutable",
4539 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4540 }
4541}
4542
c57f56b7 4543//______________________________________________________________________________
5513444a 4544void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4545{
4546// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4547// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4548// Next lines: full_path_to_dataset XXX (XXX is a string)
4549// To submit, one has to: submit jdl XXX for all lines
4550 ofstream out;
4551 out.open(filename, ios::out);
4552 if (out.bad()) {
4553 Error("WriteProductionFile", "Bad file name: %s", filename);
4554 return;
4555 }
d3b18c4c 4556 TString workdir;
4557 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4558 workdir = gGrid->GetHomeDirectory();
5513444a 4559 workdir += fGridWorkingDir;
f5e8c702 4560 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4561 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4562 out << locjdl << " " << njobspermaster << endl;
5513444a 4563 Int_t nmasterjobs = fInputFiles->GetEntries();
4564 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4565 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4566 runOutDir.ReplaceAll(".xml", "");
4567 if (fOutputToRunNo)
4568 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4569 else
4570 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4571 }
d3b18c4c 4572 if (gGrid) {
4573 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4574 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4575// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4576 if (!copyLocal2Alien("WriteProductionFile", filename,
4577 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4578 }
5513444a 4579}
4580
4581//______________________________________________________________________________
0f389141 4582void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4583{
4584// Generate the alien validation script.
4585 // Generate the validation script
4586 TObjString *os;
d3b18c4c 4587 if (fValidationScript.IsNull()) {
4588 fValidationScript = fExecutable;
4589 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4590 }
4591 TString validationScript = fValidationScript;
4592 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4593 if (!Connect()) {
4594 Error("WriteValidationScript", "Alien connection required");
4595 return;
4596 }
d3b18c4c 4597 if (!fTerminateFiles.IsNull()) {
4598 fTerminateFiles.Strip();
4599 fTerminateFiles.ReplaceAll(" ",",");
4600 }
a2f5fc01 4601 TString outStream = "";
4602 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4603 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4604 ofstream out;
0d5d317c 4605 out.open(validationScript, ios::out);
c57f56b7 4606 out << "#!/bin/bash" << endl;
4607 out << "##################################################" << endl;
4608 out << "validateout=`dirname $0`" << endl;
4609 out << "validatetime=`date`" << endl;
4610 out << "validated=\"0\";" << endl;
4611 out << "error=0" << endl;
4612 out << "if [ -z $validateout ]" << endl;
4613 out << "then" << endl;
4614 out << " validateout=\".\"" << endl;
4615 out << "fi" << endl << endl;
4616 out << "cd $validateout;" << endl;
4617 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4618 out << "echo \"*******************************************************\"" << outStream << endl;
4619 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4620 out << "" << endl;
a2f5fc01 4621 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4622 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4623 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4624 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4625 out << "ls -la ./" << outStream << endl;
4626 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4627 out << "##################################################" << endl;
ebec370a 4628 out << "" << endl;
4629
4630 out << "if [ ! -f stderr ] ; then" << endl;
4631 out << " error=1" << endl;
a2f5fc01 4632 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4633 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4634 out << "fi" << endl;
4635
b34c9f51 4636 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4637 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4638 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4639 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4640 out << "" << endl;
4641
ebec370a 4642 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4643 out << " error=1" << endl;
a2f5fc01 4644 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4645 out << " echo \"$parArch\" " << outStream << endl;
4646 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4647 out << "fi" << endl;
4648
4649 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4650 out << " error=1" << endl;
a2f5fc01 4651 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4652 out << " echo \"$segViol\" " << outStream << endl;
4653 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4654 out << "fi" << endl;
4655
4656 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4657 out << " error=1" << endl;
a2f5fc01 4658 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4659 out << " echo \"$segFault\" " << outStream << endl;
4660 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4661 out << "fi" << endl;
4662
b34c9f51 4663 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4664 out << " error=1" << endl;
a2f5fc01 4665 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4666 out << " echo \"$glibcErr\" " << outStream << endl;
4667 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4668 out << "fi" << endl;
4669
ebec370a 4670 // Part dedicated to the specific analyses running into the train
4671
d3b18c4c 4672 TString outputFiles = fOutputFiles;
4673 if (merge && !fTerminateFiles.IsNull()) {
4674 outputFiles += ",";
4675 outputFiles += fTerminateFiles;
4676 }
4677 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4678 TIter next1(arr);
a2f5fc01 4679 TString outputFile;
bbd88db8 4680 while (!merge && (os=(TObjString*)next1())) {
4681 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4682 outputFile = os->GetString();
4683 Int_t index = outputFile.Index("@");
4684 if (index > 0) outputFile.Remove(index);
bbd88db8 4685 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4686 if (outputFile.Contains("*")) continue;
4687 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4688 out << " error=1" << endl;
7c2cd90a 4689 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4690 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4691 out << "fi" << endl;
4692 }
4693 delete arr;
7c2cd90a 4694 out << "if ! [ -f outputs_valid ] ; then" << endl;
4695 out << " error=1" << endl;
4696 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4697 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4698 out << "fi" << endl;
923e2ca5 4699
c57f56b7 4700 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4701 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4702 if (!IsKeepLogs()) {
4703 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4704 outStream = "";
149d288c 4705 out << " rm -f std*" << endl;
4706 }
c57f56b7 4707 out << "fi" << endl;
4708
a2f5fc01 4709 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4710 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4711 out << "cd -" << endl;
4712 out << "exit $error" << endl;
4713 }
4714 Bool_t copy = kTRUE;
d3b18c4c 4715 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4716 if (copy) {
4717 CdWork();
4718 TString workdir = gGrid->GetHomeDirectory();
4719 workdir += fGridWorkingDir;
d3b18c4c 4720 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4721 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 4722// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4723 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
4724 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 4725 }
4726}