]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Added job information to fileinfo.log: total duration, CPU time reported by ROOT...
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
c82bb898 51using std::ofstream;
52using std::ifstream;
53using std::ios;
54using std::endl;
c57f56b7 55ClassImp(AliAnalysisAlien)
6c395669 56#if 0
57;
58#endif
c57f56b7 59
6c395669 60namespace {
61 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
62 {
63 TString sl(Form("file:%s", loc));
64 TString sr(Form("alien://%s", rem));
65 Bool_t ret = TFile::Cp(sl, sr);
66 if (!ret) {
67 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
68 }
69 return ret;
70 }
71}
72
c57f56b7 73//______________________________________________________________________________
74AliAnalysisAlien::AliAnalysisAlien()
75 :AliAnalysisGrid(),
76 fGridJDL(NULL),
0f389141 77 fMergingJDL(NULL),
c57f56b7 78 fPrice(0),
79 fTTL(0),
80 fSplitMaxInputFileNumber(0),
81 fMaxInitFailed(0),
82 fMasterResubmitThreshold(0),
bb885a9e 83 fNtestFiles(0),
319593fb 84 fNrunsPerMaster(0),
16a4353c 85 fMaxMergeFiles(0),
b3e07543 86 fMaxMergeStages(0),
d2a409b2 87 fNsubmitted(0),
a3e84053 88 fProductionMode(0),
cd11251e 89 fOutputToRunNo(0),
0f389141 90 fMergeViaJDL(0),
91 fFastReadOption(0),
e1c22e21 92 fOverwriteMode(1),
149d288c 93 fNreplicas(2),
3bdcb562 94 fNproofWorkers(0),
27734f0e 95 fNproofWorkersPerSlave(0),
3bdcb562 96 fProofReset(0),
c57f56b7 97 fRunNumbers(),
98 fExecutable(),
0a1c1f7f 99 fExecutableCommand(),
c57f56b7 100 fArguments(),
631c0b05 101 fExecutableArgs(),
c57f56b7 102 fAnalysisMacro(),
103 fAnalysisSource(),
d3b18c4c 104 fValidationScript(),
d5c6455a 105 fAdditionalRootLibs(),
c57f56b7 106 fAdditionalLibs(),
107 fSplitMode(),
108 fAPIVersion(),
109 fROOTVersion(),
110 fAliROOTVersion(),
648174cf 111 fExternalPackages(),
c57f56b7 112 fUser(),
113 fGridWorkingDir(),
114 fGridDataDir(),
115 fDataPattern(),
116 fGridOutputDir(),
117 fOutputArchive(),
118 fOutputFiles(),
119 fInputFormat(),
e7c71df0 120 fDatasetName(),
c57f56b7 121 fJDLName(),
d3b18c4c 122 fTerminateFiles(),
bb885a9e 123 fMergeExcludes(),
37d21c01 124 fRegisterExcludes(),
f965131e 125 fIncludePath(),
bb885a9e 126 fCloseSE(),
0df6ccf2 127 fFriendChainName(),
c6cb3634 128 fJobTag(),
648174cf 129 fOutputSingle(),
5fce53f4 130 fRunPrefix(),
3bdcb562 131 fProofCluster(),
132 fProofDataSet(),
133 fFileForTestMode(),
134 fRootVersionForProof(),
135 fAliRootMode(),
f47d5cb4 136 fMergeDirName(),
4e5c5506 137 fInputFiles(0),
28ce6c9c 138 fPackages(0),
77f34eae 139 fModules(0),
28ce6c9c 140 fProofParam()
c57f56b7 141{
142// Dummy ctor.
143 SetDefaults();
144}
145
146//______________________________________________________________________________
147AliAnalysisAlien::AliAnalysisAlien(const char *name)
148 :AliAnalysisGrid(name),
149 fGridJDL(NULL),
0f389141 150 fMergingJDL(NULL),
c57f56b7 151 fPrice(0),
152 fTTL(0),
153 fSplitMaxInputFileNumber(0),
154 fMaxInitFailed(0),
155 fMasterResubmitThreshold(0),
bb885a9e 156 fNtestFiles(0),
319593fb 157 fNrunsPerMaster(0),
16a4353c 158 fMaxMergeFiles(0),
b3e07543 159 fMaxMergeStages(0),
d2a409b2 160 fNsubmitted(0),
a3e84053 161 fProductionMode(0),
cd11251e 162 fOutputToRunNo(0),
0f389141 163 fMergeViaJDL(0),
164 fFastReadOption(0),
e1c22e21 165 fOverwriteMode(1),
149d288c 166 fNreplicas(2),
3bdcb562 167 fNproofWorkers(0),
27734f0e 168 fNproofWorkersPerSlave(0),
3bdcb562 169 fProofReset(0),
c57f56b7 170 fRunNumbers(),
171 fExecutable(),
0a1c1f7f 172 fExecutableCommand(),
c57f56b7 173 fArguments(),
631c0b05 174 fExecutableArgs(),
c57f56b7 175 fAnalysisMacro(),
176 fAnalysisSource(),
d3b18c4c 177 fValidationScript(),
d5c6455a 178 fAdditionalRootLibs(),
c57f56b7 179 fAdditionalLibs(),
180 fSplitMode(),
181 fAPIVersion(),
182 fROOTVersion(),
183 fAliROOTVersion(),
648174cf 184 fExternalPackages(),
c57f56b7 185 fUser(),
186 fGridWorkingDir(),
187 fGridDataDir(),
188 fDataPattern(),
189 fGridOutputDir(),
190 fOutputArchive(),
191 fOutputFiles(),
192 fInputFormat(),
e7c71df0 193 fDatasetName(),
c57f56b7 194 fJDLName(),
d3b18c4c 195 fTerminateFiles(),
bb885a9e 196 fMergeExcludes(),
37d21c01 197 fRegisterExcludes(),
f965131e 198 fIncludePath(),
bb885a9e 199 fCloseSE(),
0df6ccf2 200 fFriendChainName(),
c6cb3634 201 fJobTag(),
648174cf 202 fOutputSingle(),
5fce53f4 203 fRunPrefix(),
3bdcb562 204 fProofCluster(),
205 fProofDataSet(),
206 fFileForTestMode(),
207 fRootVersionForProof(),
208 fAliRootMode(),
f47d5cb4 209 fMergeDirName(),
4e5c5506 210 fInputFiles(0),
28ce6c9c 211 fPackages(0),
77f34eae 212 fModules(0),
28ce6c9c 213 fProofParam()
c57f56b7 214{
215// Default ctor.
216 SetDefaults();
217}
218
219//______________________________________________________________________________
220AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
221 :AliAnalysisGrid(other),
222 fGridJDL(NULL),
0f389141 223 fMergingJDL(NULL),
c57f56b7 224 fPrice(other.fPrice),
225 fTTL(other.fTTL),
226 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
227 fMaxInitFailed(other.fMaxInitFailed),
228 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 229 fNtestFiles(other.fNtestFiles),
319593fb 230 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 231 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 232 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 233 fNsubmitted(other.fNsubmitted),
a3e84053 234 fProductionMode(other.fProductionMode),
cd11251e 235 fOutputToRunNo(other.fOutputToRunNo),
0f389141 236 fMergeViaJDL(other.fMergeViaJDL),
237 fFastReadOption(other.fFastReadOption),
238 fOverwriteMode(other.fOverwriteMode),
149d288c 239 fNreplicas(other.fNreplicas),
3bdcb562 240 fNproofWorkers(other.fNproofWorkers),
27734f0e 241 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 242 fProofReset(other.fProofReset),
c57f56b7 243 fRunNumbers(other.fRunNumbers),
244 fExecutable(other.fExecutable),
0a1c1f7f 245 fExecutableCommand(other.fExecutableCommand),
c57f56b7 246 fArguments(other.fArguments),
631c0b05 247 fExecutableArgs(other.fExecutableArgs),
c57f56b7 248 fAnalysisMacro(other.fAnalysisMacro),
249 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 250 fValidationScript(other.fValidationScript),
d5c6455a 251 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 252 fAdditionalLibs(other.fAdditionalLibs),
253 fSplitMode(other.fSplitMode),
254 fAPIVersion(other.fAPIVersion),
255 fROOTVersion(other.fROOTVersion),
256 fAliROOTVersion(other.fAliROOTVersion),
648174cf 257 fExternalPackages(other.fExternalPackages),
c57f56b7 258 fUser(other.fUser),
259 fGridWorkingDir(other.fGridWorkingDir),
260 fGridDataDir(other.fGridDataDir),
261 fDataPattern(other.fDataPattern),
262 fGridOutputDir(other.fGridOutputDir),
263 fOutputArchive(other.fOutputArchive),
264 fOutputFiles(other.fOutputFiles),
265 fInputFormat(other.fInputFormat),
e7c71df0 266 fDatasetName(other.fDatasetName),
c57f56b7 267 fJDLName(other.fJDLName),
d3b18c4c 268 fTerminateFiles(other.fTerminateFiles),
bb885a9e 269 fMergeExcludes(other.fMergeExcludes),
37d21c01 270 fRegisterExcludes(other.fRegisterExcludes),
f965131e 271 fIncludePath(other.fIncludePath),
bb885a9e 272 fCloseSE(other.fCloseSE),
0df6ccf2 273 fFriendChainName(other.fFriendChainName),
c6cb3634 274 fJobTag(other.fJobTag),
648174cf 275 fOutputSingle(other.fOutputSingle),
5fce53f4 276 fRunPrefix(other.fRunPrefix),
3bdcb562 277 fProofCluster(other.fProofCluster),
278 fProofDataSet(other.fProofDataSet),
279 fFileForTestMode(other.fFileForTestMode),
280 fRootVersionForProof(other.fRootVersionForProof),
281 fAliRootMode(other.fAliRootMode),
f47d5cb4 282 fMergeDirName(other.fMergeDirName),
4e5c5506 283 fInputFiles(0),
28ce6c9c 284 fPackages(0),
77f34eae 285 fModules(0),
28ce6c9c 286 fProofParam()
c57f56b7 287{
288// Copy ctor.
289 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 290 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 291 fRunRange[0] = other.fRunRange[0];
292 fRunRange[1] = other.fRunRange[1];
c57f56b7 293 if (other.fInputFiles) {
294 fInputFiles = new TObjArray();
295 TIter next(other.fInputFiles);
296 TObject *obj;
297 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
298 fInputFiles->SetOwner();
299 }
4e5c5506 300 if (other.fPackages) {
301 fPackages = new TObjArray();
302 TIter next(other.fPackages);
303 TObject *obj;
304 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
305 fPackages->SetOwner();
306 }
77f34eae 307 if (other.fModules) {
308 fModules = new TObjArray();
309 fModules->SetOwner();
310 TIter next(other.fModules);
311 AliAnalysisTaskCfg *mod, *crt;
312 while ((crt=(AliAnalysisTaskCfg*)next())) {
313 mod = new AliAnalysisTaskCfg(*crt);
314 fModules->Add(mod);
315 }
316 }
c57f56b7 317}
318
319//______________________________________________________________________________
320AliAnalysisAlien::~AliAnalysisAlien()
321{
322// Destructor.
77f34eae 323 delete fGridJDL;
324 delete fMergingJDL;
325 delete fInputFiles;
326 delete fPackages;
327 delete fModules;
28ce6c9c 328 fProofParam.DeleteAll();
c57f56b7 329}
330
331//______________________________________________________________________________
332AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
333{
334// Assignment.
335 if (this != &other) {
336 AliAnalysisGrid::operator=(other);
337 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 338 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 339 fPrice = other.fPrice;
340 fTTL = other.fTTL;
341 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
342 fMaxInitFailed = other.fMaxInitFailed;
343 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 344 fNtestFiles = other.fNtestFiles;
a3e84053 345 fNrunsPerMaster = other.fNrunsPerMaster;
346 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 347 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 348 fNsubmitted = other.fNsubmitted;
349 fProductionMode = other.fProductionMode;
cd11251e 350 fOutputToRunNo = other.fOutputToRunNo;
0f389141 351 fMergeViaJDL = other.fMergeViaJDL;
352 fFastReadOption = other.fFastReadOption;
353 fOverwriteMode = other.fOverwriteMode;
149d288c 354 fNreplicas = other.fNreplicas;
3bdcb562 355 fNproofWorkers = other.fNproofWorkers;
27734f0e 356 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 357 fProofReset = other.fProofReset;
c57f56b7 358 fRunNumbers = other.fRunNumbers;
359 fExecutable = other.fExecutable;
0a1c1f7f 360 fExecutableCommand = other.fExecutableCommand;
c57f56b7 361 fArguments = other.fArguments;
631c0b05 362 fExecutableArgs = other.fExecutableArgs;
c57f56b7 363 fAnalysisMacro = other.fAnalysisMacro;
364 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 365 fValidationScript = other.fValidationScript;
d5c6455a 366 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 367 fAdditionalLibs = other.fAdditionalLibs;
368 fSplitMode = other.fSplitMode;
369 fAPIVersion = other.fAPIVersion;
370 fROOTVersion = other.fROOTVersion;
371 fAliROOTVersion = other.fAliROOTVersion;
648174cf 372 fExternalPackages = other.fExternalPackages;
c57f56b7 373 fUser = other.fUser;
374 fGridWorkingDir = other.fGridWorkingDir;
375 fGridDataDir = other.fGridDataDir;
376 fDataPattern = other.fDataPattern;
377 fGridOutputDir = other.fGridOutputDir;
378 fOutputArchive = other.fOutputArchive;
379 fOutputFiles = other.fOutputFiles;
380 fInputFormat = other.fInputFormat;
e7c71df0 381 fDatasetName = other.fDatasetName;
c57f56b7 382 fJDLName = other.fJDLName;
d3b18c4c 383 fTerminateFiles = other.fTerminateFiles;
bb885a9e 384 fMergeExcludes = other.fMergeExcludes;
37d21c01 385 fRegisterExcludes = other.fRegisterExcludes;
f965131e 386 fIncludePath = other.fIncludePath;
bb885a9e 387 fCloseSE = other.fCloseSE;
0df6ccf2 388 fFriendChainName = other.fFriendChainName;
c6cb3634 389 fJobTag = other.fJobTag;
648174cf 390 fOutputSingle = other.fOutputSingle;
5fce53f4 391 fRunPrefix = other.fRunPrefix;
3bdcb562 392 fProofCluster = other.fProofCluster;
393 fProofDataSet = other.fProofDataSet;
394 fFileForTestMode = other.fFileForTestMode;
395 fRootVersionForProof = other.fRootVersionForProof;
396 fAliRootMode = other.fAliRootMode;
f47d5cb4 397 fMergeDirName = other.fMergeDirName;
c57f56b7 398 if (other.fInputFiles) {
399 fInputFiles = new TObjArray();
400 TIter next(other.fInputFiles);
401 TObject *obj;
402 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
403 fInputFiles->SetOwner();
404 }
4e5c5506 405 if (other.fPackages) {
406 fPackages = new TObjArray();
407 TIter next(other.fPackages);
408 TObject *obj;
409 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
410 fPackages->SetOwner();
411 }
77f34eae 412 if (other.fModules) {
413 fModules = new TObjArray();
414 fModules->SetOwner();
415 TIter next(other.fModules);
416 AliAnalysisTaskCfg *mod, *crt;
417 while ((crt=(AliAnalysisTaskCfg*)next())) {
418 mod = new AliAnalysisTaskCfg(*crt);
419 fModules->Add(mod);
420 }
421 }
c57f56b7 422 }
423 return *this;
424}
425
37d21c01 426//______________________________________________________________________________
427void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
428{
429// Add a single additional library to be loaded. Extension must be present.
430 TString lib(name);
431 if (!lib.Contains(".")) {
432 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
433 return;
434 }
435 if (fAdditionalLibs.Contains(name)) {
436 Warning("AddAdditionalLibrary", "Library %s already added.", name);
437 return;
438 }
439 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
440 fAdditionalLibs += lib;
441}
442
77f34eae 443//______________________________________________________________________________
444void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
445{
446// Adding a module. Checks if already existing. Becomes owned by this.
447 if (!module) return;
448 if (GetModule(module->GetName())) {
449 Error("AddModule", "A module having the same name %s already added", module->GetName());
450 return;
451 }
452 if (!fModules) {
453 fModules = new TObjArray();
454 fModules->SetOwner();
455 }
456 fModules->Add(module);
457}
458
459//______________________________________________________________________________
460void AliAnalysisAlien::AddModules(TObjArray *list)
461{
462// Adding a list of modules. Checks if already existing. Becomes owned by this.
463 TIter next(list);
464 AliAnalysisTaskCfg *module;
465 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
466}
467
468//______________________________________________________________________________
469Bool_t AliAnalysisAlien::CheckDependencies()
470{
471// Check if all dependencies are satisfied. Reorder modules if needed.
472 Int_t nmodules = GetNmodules();
473 if (!nmodules) {
474 Warning("CheckDependencies", "No modules added yet to check their dependencies");
475 return kTRUE;
476 }
477 AliAnalysisTaskCfg *mod = 0;
478 AliAnalysisTaskCfg *dep = 0;
479 TString depname;
480 Int_t i, j, k;
481 for (i=0; i<nmodules; i++) {
482 mod = (AliAnalysisTaskCfg*) fModules->At(i);
483 Int_t ndeps = mod->GetNdeps();
484 Int_t istart = i;
485 for (j=0; j<ndeps; j++) {
486 depname = mod->GetDependency(j);
487 dep = GetModule(depname);
488 if (!dep) {
489 Error("CheckDependencies","Dependency %s not added for module %s",
490 depname.Data(), mod->GetName());
491 return kFALSE;
492 }
493 if (dep->NeedsDependency(mod->GetName())) {
494 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
495 mod->GetName(), dep->GetName());
496 return kFALSE;
497 }
498 Int_t idep = fModules->IndexOf(dep);
499 // The dependency task must come first
500 if (idep>i) {
501 // Remove at idep and move all objects below up one slot
502 // down to index i included.
503 fModules->RemoveAt(idep);
3e40fd4c 504 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 505 fModules->AddAt(dep, i++);
506 }
507 //Redo from istart if dependencies were inserted
508 if (i>istart) i=istart-1;
509 }
510 }
511 return kTRUE;
512}
513
3e40fd4c 514//______________________________________________________________________________
515AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
516{
517// Create the analysis manager and optionally execute the macro in filename.
518 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
519 if (mgr) return mgr;
520 mgr = new AliAnalysisManager(name);
521 mgr->SetGridHandler((AliAnalysisGrid*)this);
522 if (strlen(filename)) {
523 TString line = gSystem->ExpandPathName(filename);
524 line.Prepend(".x ");
525 gROOT->ProcessLine(line.Data());
526 }
527 return mgr;
528}
529
77f34eae 530//______________________________________________________________________________
531Int_t AliAnalysisAlien::GetNmodules() const
532{
533// Get number of modules.
534 if (!fModules) return 0;
535 return fModules->GetEntries();
536}
537
538//______________________________________________________________________________
539AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
540{
541// Get a module by name.
542 if (!fModules) return 0;
543 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
544}
545
546//______________________________________________________________________________
547Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
548{
549// Load a given module.
550 if (mod->IsLoaded()) return kTRUE;
192ddca2 551 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
552 if (!mgr) {
553 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
554 return kFALSE;
555 }
77f34eae 556 Int_t ndeps = mod->GetNdeps();
557 TString depname;
558 for (Int_t j=0; j<ndeps; j++) {
559 depname = mod->GetDependency(j);
560 AliAnalysisTaskCfg *dep = GetModule(depname);
561 if (!dep) {
562 Error("LoadModule","Dependency %s not existing for module %s",
563 depname.Data(), mod->GetName());
564 return kFALSE;
565 }
566 if (!LoadModule(dep)) {
567 Error("LoadModule","Dependency %s for module %s could not be loaded",
568 depname.Data(), mod->GetName());
569 return kFALSE;
570 }
571 }
572 // Load libraries for the module
573 if (!mod->CheckLoadLibraries()) {
574 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
575 return kFALSE;
576 }
192ddca2 577 // Check if a custom file name was requested
578 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
579
580 // Check if a custom terminate file name was requested
581 if (strlen(mod->GetTerminateFileName())) {
582 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
583 fTerminateFiles += mod->GetTerminateFileName();
584 }
585
77f34eae 586 // Execute the macro
587 if (mod->ExecuteMacro()<0) {
588 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
589 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
590 return kFALSE;
591 }
592 // Configure dependencies
593 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
594 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
595 mod->GetConfigMacro()->GetTitle(), mod->GetName());
596 return kFALSE;
597 }
5e8c7c22 598 // Adjust extra libraries
599 Int_t nlibs = mod->GetNlibs();
600 TString lib;
601 for (Int_t i=0; i<nlibs; i++) {
602 lib = mod->GetLibrary(i);
5e8c7c22 603 lib = Form("lib%s.so", lib.Data());
c9d6bd12 604 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 605 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
606 fAdditionalLibs += lib;
607 }
77f34eae 608 return kTRUE;
609}
610
139fe6ba 611//______________________________________________________________________________
612Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
613{
614// Generate the full train.
615 fAdditionalLibs = "";
616 if (!LoadModules()) return kFALSE;
617 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
618 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 619 mgr->RunLocalInit();
139fe6ba 620 mgr->PrintStatus();
621 Int_t productionMode = fProductionMode;
622 SetProductionMode();
623 TString macro = fAnalysisMacro;
624 TString executable = fExecutable;
625 TString validation = fValidationScript;
626 TString execCommand = fExecutableCommand;
627 SetAnalysisMacro(Form("%s.C", name));
628 SetExecutable(Form("%s.sh", name));
a8e31fb0 629// SetExecutableCommand("aliroot -b -q ");
139fe6ba 630 SetValidationScript(Form("%s_validation.sh", name));
631 StartAnalysis();
632 SetProductionMode(productionMode);
633 fAnalysisMacro = macro;
634 fExecutable = executable;
635 fExecutableCommand = execCommand;
636 fValidationScript = validation;
637 return kTRUE;
638}
639
77f34eae 640//______________________________________________________________________________
21ca8e59 641Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 642{
643// Generate test macros for a single module or for the full train.
5e8c7c22 644 fAdditionalLibs = "";
77f34eae 645 if (strlen(modname)) {
21ca8e59 646 if (!CheckDependencies()) return kFALSE;
647 AliAnalysisTaskCfg *mod = GetModule(modname);
648 if (!mod) {
649 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
650 return kFALSE;
651 }
652 if (!LoadModule(mod)) return kFALSE;
653 } else if (!LoadModules()) return kFALSE;
654 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
655 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 656 mgr->RunLocalInit();
21ca8e59 657 mgr->PrintStatus();
658 SetLocalTest(kTRUE);
659 Int_t productionMode = fProductionMode;
660 SetProductionMode();
661 TString macro = fAnalysisMacro;
662 TString executable = fExecutable;
663 TString validation = fValidationScript;
664 TString execCommand = fExecutableCommand;
665 SetAnalysisMacro(Form("%s.C", name));
666 SetExecutable(Form("%s.sh", name));
a8e31fb0 667// SetExecutableCommand("aliroot -b -q ");
21ca8e59 668 SetValidationScript(Form("%s_validation.sh", name));
669 WriteAnalysisFile();
670 WriteAnalysisMacro();
671 WriteExecutable();
672 WriteValidationScript();
673 SetLocalTest(kFALSE);
674 SetProductionMode(productionMode);
675 fAnalysisMacro = macro;
676 fExecutable = executable;
677 fExecutableCommand = execCommand;
678 fValidationScript = validation;
77f34eae 679 return kTRUE;
680}
681
682//______________________________________________________________________________
683Bool_t AliAnalysisAlien::LoadModules()
684{
685// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 686 fAdditionalLibs = "";
21ca8e59 687 Int_t nmodules = GetNmodules();
688 if (!nmodules) {
689 Warning("LoadModules", "No module to be loaded");
690 return kTRUE;
691 }
3e40fd4c 692 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
693 if (!mgr) {
694 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
695 return kFALSE;
696 }
77f34eae 697 if (!CheckDependencies()) return kFALSE;
21ca8e59 698 nmodules = GetNmodules();
77f34eae 699 AliAnalysisTaskCfg *mod;
21ca8e59 700 for (Int_t imod=0; imod<nmodules; imod++) {
701 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 702 if (!LoadModule(mod)) return kFALSE;
703 }
77f34eae 704 return kTRUE;
705}
706
74013535 707//______________________________________________________________________________
708void AliAnalysisAlien::SetRunPrefix(const char *prefix)
709{
710// Set the run number format. Can be a prefix or a format like "%09d"
711 fRunPrefix = prefix;
712 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
713}
714
f965131e 715//______________________________________________________________________________
716void AliAnalysisAlien::AddIncludePath(const char *path)
717{
718// Add include path in the remote analysis macro.
719 TString p(path);
720 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
721 else fIncludePath += Form("-I%s ", path);
722}
723
c57f56b7 724//______________________________________________________________________________
725void AliAnalysisAlien::AddRunNumber(Int_t run)
726{
727// Add a run number to the list of runs to be processed.
728 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 729 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 730}
731
98690895 732//______________________________________________________________________________
733void AliAnalysisAlien::AddRunList(const char* runList)
734{
735// Add several runs into the list of runs; they are expected to be separated by a blank character.
736 TString sList = runList;
737 TObjArray *list = sList.Tokenize(" ");
738 Int_t n = list->GetEntries();
739 for (Int_t i = 0; i < n; i++) {
740 TObjString *os = (TObjString*)list->At(i);
741 AddRunNumber(os->GetString().Atoi());
742 }
743 delete list;
744}
745
ee75cfc3 746//______________________________________________________________________________
747void AliAnalysisAlien::AddRunNumber(const char* run)
748{
749// Add a run number to the list of runs to be processed.
7aa5d539 750 TString runs = run;
751 TObjString *os;
752 TObjArray *arr = runs.Tokenize(" ");
753 TIter next(arr);
754 TString prefix;
755 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
756 while ((os=(TObjString*)next())){
757 if (fRunNumbers.Length()) fRunNumbers += " ";
758 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
759 }
760 delete arr;
ee75cfc3 761}
762
c57f56b7 763//______________________________________________________________________________
764void AliAnalysisAlien::AddDataFile(const char *lfn)
765{
766// Adds a data file to the input to be analysed. The file should be a valid LFN
767// or point to an existing file in the alien workdir.
768 if (!fInputFiles) fInputFiles = new TObjArray();
769 fInputFiles->Add(new TObjString(lfn));
770}
648174cf 771
772//______________________________________________________________________________
773void AliAnalysisAlien::AddExternalPackage(const char *package)
774{
775// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
776 if (fExternalPackages) fExternalPackages += " ";
777 fExternalPackages += package;
778}
779
c57f56b7 780//______________________________________________________________________________
781Bool_t AliAnalysisAlien::Connect()
782{
783// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
784 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 785 if (fProductionMode) return kTRUE;
c57f56b7 786 if (!gGrid) {
787 Info("Connect", "Trying to connect to AliEn ...");
788 TGrid::Connect("alien://");
789 }
790 if (!gGrid || !gGrid->IsConnected()) {
791 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
792 return kFALSE;
793 }
794 fUser = gGrid->GetUser();
795 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
796 return kTRUE;
797}
798
799//______________________________________________________________________________
800void AliAnalysisAlien::CdWork()
801{
802// Check validity of alien workspace. Create directory if possible.
803 if (!Connect()) {
804 Error("CdWork", "Alien connection required");
805 return;
806 }
807 TString homedir = gGrid->GetHomeDirectory();
808 TString workdir = homedir + fGridWorkingDir;
923e2ca5 809 if (DirectoryExists(workdir)) {
810 gGrid->Cd(workdir);
811 return;
812 }
813 // Work directory not existing - create it
814 gGrid->Cd(homedir);
b93f8109 815 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 816 gGrid->Cd(fGridWorkingDir);
d3b18c4c 817 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 818 } else {
d3b18c4c 819 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 820 workdir.Data(), homedir.Data());
821 fGridWorkingDir = "";
822 }
c57f56b7 823}
824
348be253 825//______________________________________________________________________________
826Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
827{
828// Check if file copying is possible.
d3b18c4c 829 if (fProductionMode) return kTRUE;
b5ae4fe0 830 TString salienpath(alienpath);
831 if (salienpath.Contains(" ")) {
832 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
833 return kFALSE;
834 }
348be253 835 if (!Connect()) {
836 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
837 return kFALSE;
838 }
d3339be3 839 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
840 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 841 // Check if alien_CLOSE_SE is defined
842 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
843 if (!closeSE.IsNull()) {
844 Info("CheckFileCopy", "Your current close storage is pointing to: \
845 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
846 } else {
847 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
848 }
849 // Check if grid directory exists.
850 if (!DirectoryExists(alienpath)) {
851 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
852 return kFALSE;
853 }
b5ae4fe0 854 TString stest = "plugin_test_copy";
855 TFile f(stest, "RECREATE");
348be253 856 // User may not have write permissions to current directory
857 if (f.IsZombie()) {
858 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
859 gSystem->WorkingDirectory());
860 return kFALSE;
861 }
862 f.Close();
b5ae4fe0 863 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
864 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 865 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 866 \n# 1. Make sure you have write permissions there. If this is the case: \
867 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
868 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
869 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
870 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 871 gSystem->Unlink(stest.Data());
348be253 872 return kFALSE;
873 }
b5ae4fe0 874 gSystem->Unlink(stest.Data());
875 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 876 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 877 return kTRUE;
878}
879
c57f56b7 880//______________________________________________________________________________
881Bool_t AliAnalysisAlien::CheckInputData()
882{
883// Check validity of input data. If necessary, create xml files.
d3b18c4c 884 if (fProductionMode) return kTRUE;
a8739e8a 885 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 886 if (!fGridDataDir.Length()) {
887 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
888 return kFALSE;
889 }
9f020952 890 if (fMergeViaJDL) {
891 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
892 return kFALSE;
893 }
d2a409b2 894 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 895 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
896 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 897 return kTRUE;
c57f56b7 898 }
899 // Process declared files
a2f5fc01 900 Bool_t isCollection = kFALSE;
901 Bool_t isXml = kFALSE;
902 Bool_t useTags = kFALSE;
c57f56b7 903 Bool_t checked = kFALSE;
d3b18c4c 904 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 905 TString file;
906 TString workdir = gGrid->GetHomeDirectory();
907 workdir += fGridWorkingDir;
908 if (fInputFiles) {
909 TObjString *objstr;
910 TIter next(fInputFiles);
911 while ((objstr=(TObjString*)next())) {
912 file = workdir;
913 file += "/";
914 file += objstr->GetString();
915 // Store full lfn path
916 if (FileExists(file)) objstr->SetString(file);
917 else {
918 file = objstr->GetName();
919 if (!FileExists(objstr->GetName())) {
920 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
921 objstr->GetName(), workdir.Data());
922 return kFALSE;
923 }
924 }
925 Bool_t iscoll, isxml, usetags;
926 CheckDataType(file, iscoll, isxml, usetags);
927 if (!checked) {
928 checked = kTRUE;
a2f5fc01 929 isCollection = iscoll;
930 isXml = isxml;
931 useTags = usetags;
932 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 933 } else {
a2f5fc01 934 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 935 Error("CheckInputData", "Some conflict was found in the types of inputs");
936 return kFALSE;
937 }
938 }
939 }
940 }
941 // Process requested run numbers
a8739e8a 942 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 943 // Check validity of alien data directory
944 if (!fGridDataDir.Length()) {
945 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
946 return kFALSE;
947 }
923e2ca5 948 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 949 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
950 return kFALSE;
951 }
a2f5fc01 952 if (isCollection) {
c57f56b7 953 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
954 return kFALSE;
955 }
956
a2f5fc01 957 if (checked && !isXml) {
c57f56b7 958 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
959 return kFALSE;
960 }
961 // Check validity of run number(s)
962 TObjArray *arr;
963 TObjString *os;
74013535 964 TString format;
319593fb 965 Int_t nruns = 0;
904f9f5f 966 TString schunk, schunk2;
c57f56b7 967 TString path;
968 if (!checked) {
969 checked = kTRUE;
a2f5fc01 970 useTags = fDataPattern.Contains("tag");
971 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 972 }
a2f5fc01 973 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 974 Error("CheckInputData", "Cannot mix input files using/not using tags");
975 return kFALSE;
976 }
977 if (fRunNumbers.Length()) {
a8739e8a 978 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 979 arr = fRunNumbers.Tokenize(" ");
980 TIter next(arr);
981 while ((os=(TObjString*)next())) {
982 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 983 if (!DirectoryExists(path)) {
984 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 985 continue;
c57f56b7 986 }
987 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
988 TString msg = "\n##### file: ";
989 msg += path;
990 msg += " type: xml_collection;";
a2f5fc01 991 if (useTags) msg += " using_tags: Yes";
c57f56b7 992 else msg += " using_tags: No";
23329835 993 Info("CheckDataType", "%s", msg.Data());
319593fb 994 if (fNrunsPerMaster<2) {
d2a409b2 995 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 996 } else {
997 nruns++;
998 if (((nruns-1)%fNrunsPerMaster) == 0) {
999 schunk = os->GetString();
1000 }
1001 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1002 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1003 AddDataFile(schunk);
319593fb 1004 }
c57f56b7 1005 }
1006 delete arr;
a8739e8a 1007 } else {
1008 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1009 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1010 format = Form("%%s/%s ", fRunPrefix.Data());
1011 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1012 if (!DirectoryExists(path)) {
a8739e8a 1013 continue;
1014 }
74013535 1015 format = Form("%%s/%s.xml", fRunPrefix.Data());
1016 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1017 TString msg = "\n##### file: ";
1018 msg += path;
1019 msg += " type: xml_collection;";
a2f5fc01 1020 if (useTags) msg += " using_tags: Yes";
a8739e8a 1021 else msg += " using_tags: No";
23329835 1022 Info("CheckDataType", "%s", msg.Data());
319593fb 1023 if (fNrunsPerMaster<2) {
74013535 1024 format = Form("%s.xml", fRunPrefix.Data());
1025 AddDataFile(Form(format.Data(),irun));
319593fb 1026 } else {
1027 nruns++;
1028 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1029 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1030 }
74013535 1031 format = Form("_%s.xml", fRunPrefix.Data());
1032 schunk2 = Form(format.Data(), irun);
319593fb 1033 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1034 schunk += schunk2;
d2a409b2 1035 AddDataFile(schunk);
319593fb 1036 }
a8739e8a 1037 }
904f9f5f 1038 if (!fInputFiles) {
1039 schunk += schunk2;
1040 AddDataFile(schunk);
1041 }
c57f56b7 1042 }
1043 return kTRUE;
1044}
1045
21ca8e59 1046//______________________________________________________________________________
4a8b667c 1047Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1048{
1049// Copy data from the given grid directory according a pattern and make a local
1050// dataset.
2ba4c287 1051// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1052 if (!Connect()) {
1053 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
4a8b667c 1054 return 0;
21ca8e59 1055 }
1056 if (!DirectoryExists(griddir)) {
1057 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
4a8b667c 1058 return 0;
21ca8e59 1059 }
1060 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1061 printf("Running command: %s\n", command.Data());
1062 TGridResult *res = gGrid->Command(command);
1063 Int_t nfound = res->GetEntries();
1064 if (!nfound) {
1065 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
4a8b667c 1066 return 0;
21ca8e59 1067 }
1068 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1069
1070 // archives
1071 TObjArray* additionalArchives = 0;
1072 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1073 additionalArchives = TString(archivefile).Tokenize(";");
1074 archivefile = additionalArchives->At(0)->GetName();
1075 additionalArchives->RemoveAt(0);
1076 additionalArchives->Compress();
1077 }
1078
21ca8e59 1079 // Copy files locally
1080 ofstream out;
1081 out.open(output, ios::out);
1082 TMap *map;
1083 TString turl, dirname, filename, temp;
1084 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1085 gSystem->MakeDirectory(outputdir);
1086 gSystem->ChangeDirectory(outputdir);
4a8b667c 1087 Int_t ncopied = 0;
21ca8e59 1088 for (Int_t i=0; i<nfound; i++) {
1089 map = (TMap*)res->At(i);
1090 turl = map->GetValue("turl")->GetName();
1091 filename = gSystem->BaseName(turl.Data());
1092 dirname = gSystem->DirName(turl.Data());
1093 dirname = gSystem->BaseName(dirname.Data());
1094 gSystem->MakeDirectory(dirname);
43b807f6 1095
1096 TString source(turl);
1097 TString targetFileName(filename);
1098
1099 if (strlen(archivefile) > 0) {
4a8b667c 1100// TODO here the archive in which the file resides should be determined
1101// however whereis returns only a guid, and guid2lfn does not work
1102// Therefore we use the one provided as argument for now
1103 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1104 targetFileName = archivefile;
43b807f6 1105 }
1106 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
4a8b667c 1107 Bool_t success = kTRUE;
1108 if (additionalArchives) {
1109 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1110 TString target;
1111 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1112 gSystem->MakeDirectory(gSystem->DirName(target));
1113 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1114 }
1115 }
2ba4c287 1116
4a8b667c 1117 if (success) {
1118 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1119 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1120 ncopied++;
1121 }
21ca8e59 1122 }
1123 }
1124 gSystem->ChangeDirectory(cdir);
1125 delete res;
4a8b667c 1126 delete additionalArchives;
1127 return ncopied;
21ca8e59 1128}
1129
c57f56b7 1130//______________________________________________________________________________
1131Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1132{
1133// Create dataset for the grid data directory + run number.
3af861ff 1134 const Int_t gMaxEntries = 15000;
d3b18c4c 1135 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1136 if (!Connect()) {
1137 Error("CreateDataset", "Cannot create dataset with no grid connection");
1138 return kFALSE;
1139 }
1140
1141 // Cd workspace
d3b18c4c 1142 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1143 TString workdir = gGrid->GetHomeDirectory();
1144 workdir += fGridWorkingDir;
1145
1146 // Compose the 'find' command arguments
74013535 1147 TString format;
c57f56b7 1148 TString command;
1149 TString options = "-x collection ";
bb885a9e 1150 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1151 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1152 TString conditions = "";
3af861ff 1153 Int_t nstart = 0;
1154 Int_t ncount = 0;
1155 Int_t stage = 0;
c57f56b7 1156 TString file;
1157 TString path;
319593fb 1158 Int_t nruns = 0;
904f9f5f 1159 TString schunk, schunk2;
ab254fd1 1160 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1161 if (!fRunNumbers.Length() && !fRunRange[0]) {
1162 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1163 // Make a single data collection from data directory.
1164 path = fGridDataDir;
923e2ca5 1165 if (!DirectoryExists(path)) {
d2a409b2 1166 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1167 return kFALSE;
1168 }
923e2ca5 1169// CdWork();
d2a409b2 1170 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1171 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1172 while (1) {
1173 ncount = 0;
1174 stage++;
1175 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1176 command = "find ";
1177 command += Form("%s -o %d ",options.Data(), nstart);
1178 command += path;
1179 command += " ";
1180 command += pattern;
1181 command += conditions;
1182 printf("command: %s\n", command.Data());
1183 TGridResult *res = gGrid->Command(command);
1184 if (res) delete res;
1185 // Write standard output to file
1186 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1187 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1188 Bool_t nullFile = kFALSE;
1189 if (!hasGrep) {
1190 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1191 } else {
1192 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1193 if (nullFile) {
1194 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1195 gSystem->Exec("rm -f __tmp*");
1196 return kFALSE;
1197 }
1198 TString line;
1199 ifstream in;
1200 in.open("__tmp__");
1201 in >> line;
1202 in.close();
1203 gSystem->Exec("rm -f __tmp__");
1204 ncount = line.Atoi();
1205 }
1206 }
1207 if (ncount == gMaxEntries) {
1208 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1209 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1210 if (!cbase) cbase = cadd;
1211 else {
1212 cbase->Add(cadd);
1213 delete cadd;
defd7a3a 1214 }
3af861ff 1215 nstart += ncount;
1216 } else {
1217 if (cbase) {
1218 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1219 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1220 cbase->Add(cadd);
1221 delete cadd;
1222 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1223 delete cbase; cbase = 0;
1224 } else {
1225 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1226 }
1227 gSystem->Exec("rm -f __tmp*");
1228 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1229 break;
1230 }
0f389141 1231 }
1232 Bool_t fileExists = FileExists(file);
1233 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1234 // Copy xml file to alien space
0f389141 1235 if (fileExists) gGrid->Rm(file);
d2a409b2 1236 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1237 if (!FileExists(file)) {
1238 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1239 return kFALSE;
1240 }
1241 // Update list of files to be processed.
1242 }
1243 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1244 return kTRUE;
1245 }
c57f56b7 1246 // Several runs
a2f5fc01 1247 Bool_t nullResult = kTRUE;
a8739e8a 1248 if (fRunNumbers.Length()) {
1249 TObjArray *arr = fRunNumbers.Tokenize(" ");
1250 TObjString *os;
1251 TIter next(arr);
1252 while ((os=(TObjString*)next())) {
3af861ff 1253 nstart = 0;
1254 stage = 0;
dc030a24 1255 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1256 if (!DirectoryExists(path)) continue;
1257// CdWork();
a8739e8a 1258 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1259 else file = Form("%s.xml", os->GetString().Data());
319593fb 1260 // If local collection file does not exist, create it via 'find' command.
3af861ff 1261 while (1) {
1262 ncount = 0;
1263 stage++;
1264 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1265 command = "find ";
1266 command += Form("%s -o %d ",options.Data(), nstart);
1267 command += path;
1268 command += pattern;
1269 command += conditions;
1270 TGridResult *res = gGrid->Command(command);
1271 if (res) delete res;
1272 // Write standard output to file
1273 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1274 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1275 Bool_t nullFile = kFALSE;
1276 if (!hasGrep) {
1277 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1278 } else {
1279 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1280 if (nullFile) {
1281 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1282 gSystem->Exec("rm -f __tmp*");
1283 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1284 break;
3af861ff 1285 }
1286 TString line;
1287 ifstream in;
1288 in.open("__tmp__");
1289 in >> line;
1290 in.close();
1291 gSystem->Exec("rm -f __tmp__");
1292 ncount = line.Atoi();
1293 }
1294 nullResult = kFALSE;
1295 }
1296 if (ncount == gMaxEntries) {
1297 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1298 if (fNrunsPerMaster > 1) {
1299 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1300 file.Data(),gMaxEntries);
1301 return kFALSE;
1302 }
1303 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1304 if (!cbase) cbase = cadd;
1305 else {
1306 cbase->Add(cadd);
1307 delete cadd;
defd7a3a 1308 }
3af861ff 1309 nstart += ncount;
1310 } else {
c89a84bc 1311 if (cbase && fNrunsPerMaster<2) {
3af861ff 1312 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1313 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1314 cbase->Add(cadd);
1315 delete cadd;
1316 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1317 delete cbase; cbase = 0;
1318 } else {
1319 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1320 }
1321 gSystem->Exec("rm -f __tmp*");
1322 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1323 break;
d3339be3 1324 }
3af861ff 1325 }
a8739e8a 1326 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1327 // Check if there is one run per master job.
1328 if (fNrunsPerMaster<2) {
1329 if (FileExists(file)) {
0f389141 1330 if (fOverwriteMode) gGrid->Rm(file);
1331 else {
1332 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1333 continue;
1334 }
319593fb 1335 }
1336 // Copy xml file to alien space
1337 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1338 if (!FileExists(file)) {
1339 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1340 delete arr;
1341 return kFALSE;
1342 }
1343 } else {
1344 nruns++;
1345 if (((nruns-1)%fNrunsPerMaster) == 0) {
1346 schunk = os->GetString();
1347 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1348 } else {
1349 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1350 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1351 cbase->Add(cadd);
1352 delete cadd;
1353 }
1354 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1355 continue;
1356 }
1357 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1358 if (FileExists(schunk)) {
1359 if (fOverwriteMode) gGrid->Rm(file);
1360 else {
1361 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1362 continue;
1363 }
319593fb 1364 }
84fcd93f 1365 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1366 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1367 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1368 if (!FileExists(schunk)) {
1369 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1370 delete arr;
1371 return kFALSE;
1372 }
d3339be3 1373 }
a8739e8a 1374 }
1375 delete arr;
a2f5fc01 1376 if (nullResult) {
d3339be3 1377 Error("CreateDataset", "No valid dataset corresponding to the query!");
1378 return kFALSE;
1379 }
a8739e8a 1380 } else {
1381 // Process a full run range.
1382 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1383 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1384 nstart = 0;
1385 stage = 0;
74013535 1386 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1387 if (!DirectoryExists(path)) continue;
1388// CdWork();
74013535 1389 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1390 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1391 else file = Form(format.Data(), irun);
0f389141 1392 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1393 if (fOverwriteMode) gGrid->Rm(file);
1394 else {
1395 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1396 continue;
1397 }
a8739e8a 1398 }
319593fb 1399 // If local collection file does not exist, create it via 'find' command.
3af861ff 1400 while (1) {
1401 ncount = 0;
1402 stage++;
1403 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1404 command = "find ";
1405 command += Form("%s -o %d ",options.Data(), nstart);
1406 command += path;
1407 command += pattern;
1408 command += conditions;
1409 TGridResult *res = gGrid->Command(command);
1410 if (res) delete res;
1411 // Write standard output to file
1412 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1413 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1414 Bool_t nullFile = kFALSE;
1415 if (!hasGrep) {
1416 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1417 } else {
1418 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1419 if (nullFile) {
1420 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1421 gSystem->Exec("rm -f __tmp*");
1604c9ed 1422 break;
3af861ff 1423 }
1424 TString line;
1425 ifstream in;
1426 in.open("__tmp__");
1427 in >> line;
1428 in.close();
1429 gSystem->Exec("rm -f __tmp__");
1430 ncount = line.Atoi();
1431 }
1432 nullResult = kFALSE;
1433 }
1434 if (ncount == gMaxEntries) {
1435 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1436 if (fNrunsPerMaster > 1) {
1437 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1438 file.Data(),gMaxEntries);
1439 return kFALSE;
1440 }
1441 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1442 if (!cbase) cbase = cadd;
1443 else {
1444 cbase->Add(cadd);
1445 delete cadd;
defd7a3a 1446 }
3af861ff 1447 nstart += ncount;
1448 } else {
c89a84bc 1449 if (cbase && fNrunsPerMaster<2) {
3af861ff 1450 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1451 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1452 cbase->Add(cadd);
1453 delete cadd;
1454 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1455 delete cbase; cbase = 0;
1456 } else {
1457 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1458 }
1459 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1460 break;
d3339be3 1461 }
319593fb 1462 }
a8739e8a 1463 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1464 // Check if there is one run per master job.
1465 if (fNrunsPerMaster<2) {
1466 if (FileExists(file)) {
0f389141 1467 if (fOverwriteMode) gGrid->Rm(file);
1468 else {
1469 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1470 continue;
1471 }
319593fb 1472 }
1473 // Copy xml file to alien space
1474 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1475 if (!FileExists(file)) {
1476 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1477 return kFALSE;
1478 }
1479 } else {
1480 nruns++;
95e5b448 1481 // Check if the collection for the chunk exist locally.
1482 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1483 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1484 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1485 else continue;
1486 }
84fcd93f 1487 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1488 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1489 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1490 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1491 } else {
1492 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1493 cbase->Add(cadd);
1494 delete cadd;
1495 }
74013535 1496 format = Form("%%s_%s.xml", fRunPrefix.Data());
1497 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1498 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1499 continue;
1500 }
904f9f5f 1501 schunk = schunk2;
319593fb 1502 if (FileExists(schunk)) {
0f389141 1503 if (fOverwriteMode) gGrid->Rm(schunk);
1504 else {
1505 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1506 continue;
1507 }
319593fb 1508 }
84fcd93f 1509 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1510 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1511 if (FileExists(schunk)) {
0f389141 1512 if (fOverwriteMode) gGrid->Rm(schunk);
1513 else {
1514 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1515 continue;
1516 }
95e5b448 1517 }
319593fb 1518 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1519 if (!FileExists(schunk)) {
1520 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1521 return kFALSE;
1522 }
1523 }
c57f56b7 1524 }
a2f5fc01 1525 if (nullResult) {
d3339be3 1526 Error("CreateDataset", "No valid dataset corresponding to the query!");
1527 return kFALSE;
1528 }
a8739e8a 1529 }
c57f56b7 1530 return kTRUE;
1531}
1532
1533//______________________________________________________________________________
1534Bool_t AliAnalysisAlien::CreateJDL()
1535{
1536// Generate a JDL file according to current settings. The name of the file is
1537// specified by fJDLName.
ec5acdce 1538 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
c57f56b7 1539 Bool_t error = kFALSE;
1540 TObjArray *arr = 0;
1541 Bool_t copy = kTRUE;
d3b18c4c 1542 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1543 Bool_t generate = kTRUE;
1544 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1545 if (!Connect()) {
1546 Error("CreateJDL", "Alien connection required");
1547 return kFALSE;
1548 }
1549 // Check validity of alien workspace
d3b18c4c 1550 TString workdir;
1551 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1552 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1553 workdir += fGridWorkingDir;
1554 if (generate) {
1555 TObjString *os;
1556 if (!fInputFiles) {
1557 Error("CreateJDL()", "Define some input files for your analysis.");
1558 error = kTRUE;
1559 }
1560 // Compose list of input files
1561 // Check if output files were defined
1562 if (!fOutputFiles.Length()) {
1563 Error("CreateJDL", "You must define at least one output file");
1564 error = kTRUE;
1565 }
1566 // Check if an output directory was defined and valid
1567 if (!fGridOutputDir.Length()) {
1568 Error("CreateJDL", "You must define AliEn output directory");
1569 error = kTRUE;
1570 } else {
d3b18c4c 1571 if (!fProductionMode) {
1572 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1573 if (!DirectoryExists(fGridOutputDir)) {
1574 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1575 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1576 } else {
1577 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1578 // error = kTRUE;
1579 }
67a3b254 1580 } else {
1581 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1582 }
1583 gGrid->Cd(workdir);
1584 }
c57f56b7 1585 }
1586 // Exit if any error up to now
1587 if (error) return kFALSE;
1588 // Set JDL fields
0f389141 1589 if (!fUser.IsNull()) {
1590 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1591 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1592 }
1593 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1594 TString mergeExec = fExecutable;
1595 mergeExec.ReplaceAll(".sh", "_merge.sh");
1596 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1597 mergeExec.ReplaceAll(".sh", ".C");
1598 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1599 if (!fArguments.IsNull())
1600 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1601 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1602 else {
1603 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1604 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1605 }
b3e07543 1606
1f0d1ca2 1607 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1608 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1609 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1610 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1611
0f389141 1612 if (fMaxInitFailed > 0) {
c57f56b7 1613 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1614 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1615 }
1616 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1617 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1618 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1619 }
1620 if (!IsOneStageMerging()) {
f47d5cb4 1621 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1622 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1623 }
1624 if (fSplitMode.Length()) {
c57f56b7 1625 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1626 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1627 }
1628 fMergingJDL->SetValue("Split", "\"se\"");
1629 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1630 if (!fAliROOTVersion.IsNull()) {
1631 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1632 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1633 }
1634 if (!fROOTVersion.IsNull()) {
c57f56b7 1635 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1636 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1637 }
1638 if (!fAPIVersion.IsNull()) {
c57f56b7 1639 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1640 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1641 }
648174cf 1642 if (!fExternalPackages.IsNull()) {
1643 arr = fExternalPackages.Tokenize(" ");
1644 TIter next(arr);
1645 while ((os=(TObjString*)next())) {
1646 TString pkgname = os->GetString();
1647 Int_t index = pkgname.Index("::");
1648 TString pkgversion = pkgname(index+2, pkgname.Length());
1649 pkgname.Remove(index);
1650 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1651 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1652 }
1653 delete arr;
1654 }
0f389141 1655 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1656 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1657 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1658 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1659 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1660 TString analysisFile = fExecutable;
1661 analysisFile.ReplaceAll(".sh", ".root");
1662 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1663 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1664 if (fAdditionalLibs.Length()) {
1665 arr = fAdditionalLibs.Tokenize(" ");
1666 TIter next(arr);
1667 while ((os=(TObjString*)next())) {
1668 if (os->GetString().Contains(".so")) continue;
1669 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1670 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1671 }
1672 delete arr;
1673 }
4e5c5506 1674 if (fPackages) {
1675 TIter next(fPackages);
1676 TObject *obj;
0f389141 1677 while ((obj=next())) {
4e5c5506 1678 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1679 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1680 }
4e5c5506 1681 }
c57f56b7 1682 if (fOutputArchive.Length()) {
e4d9a069 1683 TString outputArchive = fOutputArchive;
1684 if (!fRegisterExcludes.IsNull()) {
1685 arr = fRegisterExcludes.Tokenize(" ");
1686 TIter next1(arr);
1687 while ((os=(TObjString*)next1())) {
1688 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1689 outputArchive.ReplaceAll(os->GetString(),"");
1690 }
1691 delete arr;
1692 }
1693 arr = outputArchive.Tokenize(" ");
c57f56b7 1694 TIter next(arr);
0f389141 1695 Bool_t first = kTRUE;
1696 const char *comment = "Files to be archived";
1697 const char *comment1 = comment;
1698 while ((os=(TObjString*)next())) {
1699 if (!first) comment = NULL;
1700 if (!os->GetString().Contains("@") && fCloseSE.Length())
1701 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1702 else
1703 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1704 first = kFALSE;
1705 }
c57f56b7 1706 delete arr;
f790bc1b 1707 // Output archive for the merging jdl
f790bc1b 1708 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1709 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1710 // Add normal output files, extra files + terminate files
1711 TString files = GetListOfFiles("outextter");
37d21c01 1712 // Do not register files in fRegisterExcludes
1713 if (!fRegisterExcludes.IsNull()) {
1714 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1715 TIter next1(arr);
1716 while ((os=(TObjString*)next1())) {
1717 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1718 files.ReplaceAll(os->GetString(),"");
1719 }
66d71516 1720 delete arr;
0f389141 1721 }
f790bc1b 1722 files.ReplaceAll(".root", "*.root");
ec5acdce 1723
1724 if (mgr->IsCollectThroughput())
1725 outputArchive += Form("root_archive.zip:%s,*.stat,*%s@disk=%d",files.Data(),mgr->GetFileInfoLog(),fNreplicas);
1726 else
1727 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1728 } else {
95f84779 1729 TString files = fOutputArchive;
1730 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1731 outputArchive = files;
f790bc1b 1732 }
0f389141 1733 arr = outputArchive.Tokenize(" ");
705adb3e 1734 TIter next2(arr);
0f389141 1735 comment = comment1;
1736 first = kTRUE;
705adb3e 1737 while ((os=(TObjString*)next2())) {
0f389141 1738 if (!first) comment = NULL;
0cdf65a8 1739 TString currentfile = os->GetString();
0cdf65a8 1740 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1741 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1742 else
0cdf65a8 1743 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1744 first = kFALSE;
1745 }
1746 delete arr;
c57f56b7 1747 }
149d288c 1748 arr = fOutputFiles.Tokenize(",");
c57f56b7 1749 TIter next(arr);
0f389141 1750 Bool_t first = kTRUE;
f790bc1b 1751 const char *comment = "Files to be saved";
43da816a 1752 while ((os=(TObjString*)next())) {
e1eaf596 1753 // Ignore ouputs in jdl that are also in outputarchive
1754 TString sout = os->GetString();
f790bc1b 1755 sout.ReplaceAll("*", "");
1756 sout.ReplaceAll(".root", "");
e1eaf596 1757 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1758 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1759 // Ignore fRegisterExcludes
1760 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1761 if (!first) comment = NULL;
43da816a 1762 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1763 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1764 else
0f389141 1765 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1766 first = kFALSE;
f790bc1b 1767 if (fMergeExcludes.Contains(sout)) continue;
1768 if (!os->GetString().Contains("@") && fCloseSE.Length())
1769 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1770 else
1771 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1772 }
c57f56b7 1773 delete arr;
0f389141 1774 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1775 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1776 TString validationScript = fValidationScript;
0f389141 1777 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1778 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1779 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1780 if (fMasterResubmitThreshold) {
1781 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1782 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1783 }
d2a409b2 1784 // Write a jdl with 2 input parameters: collection name and output dir name.
1785 WriteJDL(copy);
c57f56b7 1786 }
1787 // Copy jdl to grid workspace
a8739e8a 1788 if (copy) {
b5e4aaa7 1789 // Check if an output directory was defined and valid
1790 if (!fGridOutputDir.Length()) {
1791 Error("CreateJDL", "You must define AliEn output directory");
1792 return kFALSE;
1793 } else {
1794 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1795 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1796 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1797 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1798 } else {
1799 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1800 return kFALSE;
1801 }
1802 }
1803 gGrid->Cd(workdir);
1804 }
648174cf 1805 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1806 TString mergeJDLName = fExecutable;
1807 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1808 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1809 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1810 if (fProductionMode) {
648174cf 1811 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1812 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1813 }
648174cf 1814 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1815 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1816 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1817 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1818 Fatal("","Terminating");
1819// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1820 if (fMergeViaJDL) {
1821 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1822// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1823 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1824 Fatal("","Terminating");
0f389141 1825 }
648174cf 1826 }
c57f56b7 1827 if (fAdditionalLibs.Length()) {
e7c71df0 1828 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1829 TObjString *os;
1830 TIter next(arr);
1831 while ((os=(TObjString*)next())) {
c57f56b7 1832 if (os->GetString().Contains(".so")) continue;
4e5c5506 1833 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1834 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1835// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1836 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1837 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1838 Fatal("","Terminating");
c57f56b7 1839 }
1840 delete arr;
1841 }
4e5c5506 1842 if (fPackages) {
1843 TIter next(fPackages);
1844 TObject *obj;
1845 while ((obj=next())) {
fdbbc7be 1846 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1847 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1848// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1849 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1850 Form("%s/%s", workdir.Data(), obj->GetName())))
1851 Fatal("","Terminating");
4e5c5506 1852 }
1853 }
c57f56b7 1854 }
1855 return kTRUE;
1856}
1857
a8739e8a 1858//______________________________________________________________________________
d2a409b2 1859Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1860{
1861// Writes one or more JDL's corresponding to findex. If findex is negative,
1862// all run numbers are considered in one go (jdl). For non-negative indices
1863// they correspond to the indices in the array fInputFiles.
1864 if (!fInputFiles) return kFALSE;
d3b18c4c 1865 TObject *os;
1866 TString workdir;
1867 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1868 workdir += fGridWorkingDir;
f47d5cb4 1869 TString stageName = "$2";
1870 if (fProductionMode) stageName = "$4";
1871 if (!fMergeDirName.IsNull()) {
1872 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1873 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1874 } else {
1875 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1876 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1877 }
d3b18c4c 1878 if (fProductionMode) {
a8739e8a 1879 TIter next(fInputFiles);
b3e07543 1880 while ((os=next())) {
d3b18c4c 1881 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1882 }
56754bf6 1883 if (!fOutputToRunNo)
1884 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1885 else
1886 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1887 } else {
1888 if (!fRunNumbers.Length() && !fRunRange[0]) {
1889 // One jdl with no parameters in case input data is specified by name.
1890 TIter next(fInputFiles);
1891 while ((os=next()))
1892 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1893 if (!fOutputSingle.IsNull())
1894 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1895 else {
1896 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1897 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1898 }
1899 } else {
1900 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1901 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1902 if (!fOutputSingle.IsNull()) {
1903 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1904 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1905 } else {
1906 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1907 }
1908 }
a8739e8a 1909 }
1910
a8739e8a 1911 // Generate the JDL as a string
1912 TString sjdl = fGridJDL->Generate();
0f389141 1913 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1914 // Final merge jdl
1915 if (!fMergeDirName.IsNull()) {
1916 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1917 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1918 } else {
1919 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1920 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1921 }
b3e07543 1922 TString sjdl2 = fMergingJDL->Generate();
1923 Int_t index, index1;
a8739e8a 1924 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1925 sjdl.ReplaceAll("(member", "\n (member");
1926 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1927 sjdl.ReplaceAll("{", "{\n ");
1928 sjdl.ReplaceAll("};", "\n};");
1929 sjdl.ReplaceAll("{\n \n", "{\n");
1930 sjdl.ReplaceAll("\n\n", "\n");
1931 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1932 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1933 sjdl1.ReplaceAll("(member", "\n (member");
1934 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1935 sjdl1.ReplaceAll("{", "{\n ");
1936 sjdl1.ReplaceAll("};", "\n};");
1937 sjdl1.ReplaceAll("{\n \n", "{\n");
1938 sjdl1.ReplaceAll("\n\n", "\n");
1939 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1940 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1941 sjdl2.ReplaceAll("(member", "\n (member");
1942 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1943 sjdl2.ReplaceAll("{", "{\n ");
1944 sjdl2.ReplaceAll("};", "\n};");
1945 sjdl2.ReplaceAll("{\n \n", "{\n");
1946 sjdl2.ReplaceAll("\n\n", "\n");
1947 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1948 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1949 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1950 index = sjdl.Index("JDLVariables");
1951 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1952 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1953 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1954 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1955 index = fJobTag.Index(":");
1956 if (index < 0) index = fJobTag.Length();
1957 TString jobTag = fJobTag;
f47d5cb4 1958 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1959 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1960 if (fProductionMode) {
1961 sjdl1.Prepend("# Generated merging jdl (production mode) \
1962 \n# $1 = full alien path to output directory to be merged \
1963 \n# $2 = train number \
1964 \n# $3 = production (like LHC10b) \
1965 \n# $4 = merging stage \
1966 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1967 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1968 sjdl2.Prepend("# Generated merging jdl \
1969 \n# $1 = full alien path to output directory to be merged \
1970 \n# $2 = train number \
1971 \n# $3 = production (like LHC10b) \
1972 \n# $4 = merging stage \
1973 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1974 } else {
1975 sjdl1.Prepend("# Generated merging jdl \
1976 \n# $1 = full alien path to output directory to be merged \
1977 \n# $2 = merging stage \
03335324 1978 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1979 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1980 sjdl2.Prepend("# Generated merging jdl \
1981 \n# $1 = full alien path to output directory to be merged \
1982 \n# $2 = merging stage \
03335324 1983 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1984 }
0f389141 1985 index = sjdl1.Index("JDLVariables");
1986 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1987 index = sjdl2.Index("JDLVariables");
1988 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1989 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1990 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1991 index = sjdl2.Index("Split =");
1992 if (index>=0) {
1993 index1 = sjdl2.Index("\n", index);
1994 sjdl2.Remove(index, index1-index+1);
1995 }
1996 index = sjdl2.Index("SplitMaxInputFileNumber");
1997 if (index>=0) {
1998 index1 = sjdl2.Index("\n", index);
1999 sjdl2.Remove(index, index1-index+1);
2000 }
2001 index = sjdl2.Index("InputDataCollection");
2002 if (index>=0) {
2003 index1 = sjdl2.Index(";", index);
2004 sjdl2.Remove(index, index1-index+1);
2005 }
2006 index = sjdl2.Index("InputDataListFormat");
2007 if (index>=0) {
2008 index1 = sjdl2.Index("\n", index);
2009 sjdl2.Remove(index, index1-index+1);
2010 }
2011 index = sjdl2.Index("InputDataList");
2012 if (index>=0) {
2013 index1 = sjdl2.Index("\n", index);
2014 sjdl2.Remove(index, index1-index+1);
2015 }
f47d5cb4 2016 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2017 // Write jdl to file
a8739e8a 2018 ofstream out;
d2a409b2 2019 out.open(fJDLName.Data(), ios::out);
a8739e8a 2020 if (out.bad()) {
d3b18c4c 2021 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2022 return kFALSE;
2023 }
2024 out << sjdl << endl;
b3e07543 2025 out.close();
0f389141 2026 TString mergeJDLName = fExecutable;
2027 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2028 if (fMergeViaJDL) {
2029 ofstream out1;
2030 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2031 if (out1.bad()) {
d3b18c4c 2032 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2033 return kFALSE;
2034 }
2035 out1 << sjdl1 << endl;
b3e07543 2036 out1.close();
2037 ofstream out2;
2038 TString finalJDL = mergeJDLName;
2039 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2040 out2.open(finalJDL.Data(), ios::out);
2041 if (out2.bad()) {
2042 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2043 return kFALSE;
2044 }
2045 out2 << sjdl2 << endl;
2046 out2.close();
0f389141 2047 }
a8739e8a 2048
2049 // Copy jdl to grid workspace
2050 if (!copy) {
d3b18c4c 2051 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2052 } else {
d2a409b2 2053 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2054 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2055 TString finalJDL = mergeJDLName;
2056 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2057 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2058 if (fProductionMode) {
b5fe9cba 2059 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2060 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2061 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2062 }
d2a409b2 2063 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2064 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2065 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2066 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2067// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2068 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2069 Fatal("","Terminating");
0f389141 2070 if (fMergeViaJDL) {
b3e07543 2071 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2072// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2073// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2074 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2075 Fatal("","Terminating");
2076 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2077 Fatal("","Terminating");
0f389141 2078 }
a8739e8a 2079 }
2080 return kTRUE;
2081}
2082
c57f56b7 2083//______________________________________________________________________________
5513444a 2084Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2085{
2086// Returns true if file exists.
5513444a 2087 if (!gGrid) return kFALSE;
2ea55496 2088 TString slfn = lfn;
2089 slfn.ReplaceAll("alien://","");
2090 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2091 if (!res) return kFALSE;
2092 TMap *map = dynamic_cast<TMap*>(res->At(0));
2093 if (!map) {
2094 delete res;
2095 return kFALSE;
2096 }
2097 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2098 if (!objs || !objs->GetString().Length()) {
2099 delete res;
2100 return kFALSE;
2101 }
2102 delete res;
2103 return kTRUE;
2104}
2105
923e2ca5 2106//______________________________________________________________________________
2107Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2108{
2109// Returns true if directory exists. Can be also a path.
2110 if (!gGrid) return kFALSE;
2111 // Check if dirname is a path
2112 TString dirstripped = dirname;
2113 dirstripped = dirstripped.Strip();
2114 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2115 TString dir = gSystem->BaseName(dirstripped);
2116 dir += "/";
2117 TString path = gSystem->DirName(dirstripped);
2118 TGridResult *res = gGrid->Ls(path, "-F");
2119 if (!res) return kFALSE;
2120 TIter next(res);
2121 TMap *map;
2122 TObject *obj;
2123 while ((map=dynamic_cast<TMap*>(next()))) {
2124 obj = map->GetValue("name");
2125 if (!obj) break;
2126 if (dir == obj->GetName()) {
2127 delete res;
2128 return kTRUE;
2129 }
2130 }
2131 delete res;
2132 return kFALSE;
2133}
2134
c57f56b7 2135//______________________________________________________________________________
a2f5fc01 2136void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2137{
2138// Check input data type.
a2f5fc01 2139 isCollection = kFALSE;
2140 isXml = kFALSE;
2141 useTags = kFALSE;
c57f56b7 2142 if (!gGrid) {
2143 Error("CheckDataType", "No connection to grid");
2144 return;
2145 }
a2f5fc01 2146 isCollection = IsCollection(lfn);
c57f56b7 2147 TString msg = "\n##### file: ";
2148 msg += lfn;
a2f5fc01 2149 if (isCollection) {
c57f56b7 2150 msg += " type: raw_collection;";
2151 // special treatment for collections
a2f5fc01 2152 isXml = kFALSE;
c57f56b7 2153 // check for tag files in the collection
2154 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2155 if (!res) {
2156 msg += " using_tags: No (unknown)";
23329835 2157 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2158 return;
2159 }
2160 const char* typeStr = res->GetKey(0, "origLFN");
2161 if (!typeStr || !strlen(typeStr)) {
2162 msg += " using_tags: No (unknown)";
23329835 2163 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2164 return;
2165 }
2166 TString file = typeStr;
a2f5fc01 2167 useTags = file.Contains(".tag");
2168 if (useTags) msg += " using_tags: Yes";
c57f56b7 2169 else msg += " using_tags: No";
23329835 2170 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2171 return;
2172 }
2173 TString slfn(lfn);
2174 slfn.ToLower();
a2f5fc01 2175 isXml = slfn.Contains(".xml");
2176 if (isXml) {
c57f56b7 2177 // Open xml collection and check if there are tag files inside
2178 msg += " type: xml_collection;";
2179 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2180 if (!coll) {
2181 msg += " using_tags: No (unknown)";
23329835 2182 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2183 return;
2184 }
2185 TMap *map = coll->Next();
2186 if (!map) {
2187 msg += " using_tags: No (unknown)";
23329835 2188 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2189 return;
2190 }
2191 map = (TMap*)map->GetValue("");
2192 TString file;
2193 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2194 useTags = file.Contains(".tag");
c57f56b7 2195 delete coll;
a2f5fc01 2196 if (useTags) msg += " using_tags: Yes";
c57f56b7 2197 else msg += " using_tags: No";
23329835 2198 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2199 return;
2200 }
a2f5fc01 2201 useTags = slfn.Contains(".tag");
c57f56b7 2202 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2203 else msg += " type: unknown file;";
a2f5fc01 2204 if (useTags) msg += " using_tags: Yes";
c57f56b7 2205 else msg += " using_tags: No";
23329835 2206 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2207}
2208
4e5c5506 2209//______________________________________________________________________________
2210void AliAnalysisAlien::EnablePackage(const char *package)
2211{
2212// Enables a par file supposed to exist in the current directory.
2213 TString pkg(package);
2214 pkg.ReplaceAll(".par", "");
2215 pkg += ".par";
2216 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2217 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2218 return;
2219 }
2220 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2221 Info("EnablePackage", "AliEn plugin will use .par packages");
2222 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2223 if (!fPackages) {
2224 fPackages = new TObjArray();
2225 fPackages->SetOwner();
2226 }
2227 fPackages->Add(new TObjString(pkg));
2228}
2229
3bdcb562 2230//______________________________________________________________________________
2231TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2232{
2233// Make a tree from files having the location specified in fFileForTestMode.
2234// Inspired from JF's CreateESDChain.
2235 if (fFileForTestMode.IsNull()) {
2236 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2237 return NULL;
2238 }
2239 if (gSystem->AccessPathName(fFileForTestMode)) {
2240 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2241 return NULL;
2242 }
2243 // Open the file
2244 ifstream in;
2245 in.open(fFileForTestMode);
2246 Int_t count = 0;
2247 // Read the input list of files and add them to the chain
b505f1f8 2248 TString line;
d6c60197 2249 TString streeName(treeName);
2250 if (IsUseMCchain()) streeName = "TE";
2251 TChain *chain = new TChain(streeName);
b505f1f8 2252 TChain *chainFriend = 0;
d6c60197 2253 if (!fFriendChainName.IsNull()) chainFriend = new TChain(streeName);
b505f1f8 2254 while (in.good())
2255 {
3bdcb562 2256 in >> line;
2257 if (line.IsNull()) continue;
2258 if (count++ == fNtestFiles) break;
2259 TString esdFile(line);
2260 TFile *file = TFile::Open(esdFile);
b505f1f8 2261 if (file && !file->IsZombie()) {
2262 chain->Add(esdFile);
3bdcb562 2263 file->Close();
b505f1f8 2264 if (!fFriendChainName.IsNull()) {
4b5d9e21 2265 if (esdFile.Index("#") > -1)
2266 esdFile.Remove(esdFile.Index("#"));
2267 esdFile = gSystem->DirName(esdFile);
2268 esdFile += "/" + fFriendChainName;
b505f1f8 2269 file = TFile::Open(esdFile);
2270 if (file && !file->IsZombie()) {
2271 file->Close();
2272 chainFriend->Add(esdFile);
2273 } else {
2274 Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data());
2275 return 0;
2276 }
4b5d9e21 2277 }
3bdcb562 2278 } else {
2279 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2280 }
b505f1f8 2281 }
2282 in.close();
2283 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2284 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2285 delete chain;
b505f1f8 2286 delete chainFriend;
3bdcb562 2287 return NULL;
b505f1f8 2288 }
3bdcb562 2289// chain->ls();
b505f1f8 2290 if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend);
2291 return chain;
3bdcb562 2292}
2293
d2a409b2 2294//______________________________________________________________________________
2295const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2296{
2297// Get job status for all jobs with jobid>jobidstart.
2298 static char mstatus[20];
2299 mstatus[0] = '\0';
2300 nrunning = 0;
2301 nwaiting = 0;
2302 nerror = 0;
2303 ndone = 0;
2304 TGridJobStatusList *list = gGrid->Ps("");
2305 if (!list) return mstatus;
2306 Int_t nentries = list->GetSize();
2307 TGridJobStatus *status;
2308 Int_t pid;
2309 for (Int_t ijob=0; ijob<nentries; ijob++) {
2310 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2311 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2312 if (pid<jobidstart) continue;
2313 if (pid == lastid) {
8ddad121 2314 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2315 }
2316 switch (status->GetStatus()) {
2317 case TGridJobStatus::kWAITING:
2318 nwaiting++; break;
2319 case TGridJobStatus::kRUNNING:
2320 nrunning++; break;
2321 case TGridJobStatus::kABORTED:
2322 case TGridJobStatus::kFAIL:
2323 case TGridJobStatus::kUNKNOWN:
2324 nerror++; break;
2325 case TGridJobStatus::kDONE:
2326 ndone++;
2327 }
2328 }
2329 list->Delete();
2330 delete list;
2331 return mstatus;
2332}
2333
c57f56b7 2334//______________________________________________________________________________
2335Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2336{
2337// Returns true if file is a collection. Functionality duplicated from
2338// TAlien::Type() because we don't want to directly depend on TAlien.
2339 if (!gGrid) {
2340 Error("IsCollection", "No connection to grid");
2341 return kFALSE;
2342 }
2343 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2344 if (!res) return kFALSE;
2345 const char* typeStr = res->GetKey(0, "type");
2346 if (!typeStr || !strlen(typeStr)) return kFALSE;
2347 if (!strcmp(typeStr, "collection")) return kTRUE;
2348 delete res;
2349 return kFALSE;
2350}
2351
fe2d7fc2 2352//______________________________________________________________________________
2353Bool_t AliAnalysisAlien::IsSingleOutput() const
2354{
2355// Check if single-ouput option is on.
2356 return (!fOutputSingle.IsNull());
2357}
2358
16a4353c 2359//______________________________________________________________________________
2360void AliAnalysisAlien::Print(Option_t *) const
2361{
2362// Print current plugin settings.
84fcd93f 2363 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2364 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2365 if (mgr && mgr->IsProofMode()) {
2366 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2367 if (TestBit(AliAnalysisGrid::kTest))
2368 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2369 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2370 if (!fProofDataSet.IsNull())
2371 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2372 if (fProofReset==1)
2373 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2374 if (fProofReset>1)
2375 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2376 if (!fRootVersionForProof.IsNull())
2377 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2378 else
2379 printf("= ROOT version requested________________________ default\n");
2380 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2381 if (!fAliRootMode.IsNull())
2382 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2383 if (fNproofWorkers)
2384 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2385 if (fNproofWorkersPerSlave)
2386 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2387 if (TestSpecialBit(kClearPackages))
2388 printf("= ClearPackages requested...\n");
2389 if (fIncludePath.Data())
2390 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2391 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2392 if (fPackages && fPackages->GetEntries()) {
2393 TIter next(fPackages);
2394 TObject *obj;
2395 TString list;
2396 while ((obj=next())) list += obj->GetName();
2397 printf("= Par files to be used: ________________________ %s\n", list.Data());
2398 }
2399 if (TestSpecialBit(kProofConnectGrid))
2400 printf("= Requested PROOF connection to grid\n");
2401 return;
2402 }
e1c22e21 2403 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2404 if (fOverwriteMode) {
2405 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2406 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2407 }
348be253 2408 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2409 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2410 printf("= Production mode:______________________________ %d\n", fProductionMode);
2411 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2412 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2413 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2414 if (fUser.Length())
84fcd93f 2415 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2416 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2417 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2418 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2419 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2420 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2421 if (fRunNumbers.Length())
84fcd93f 2422 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2423 if (fRunRange[0])
74013535 2424 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2425 if (!fRunRange[0] && !fRunNumbers.Length()) {
2426 TIter next(fInputFiles);
2427 TObject *obj;
2428 TString list;
2429 while ((obj=next())) list += obj->GetName();
84fcd93f 2430 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2431 }
2432 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2433 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2434 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2435 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2436 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2437 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2438 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2439 printf("=====================================================================\n");
2440 printf("= Job price: ___________________________________ %d\n", fPrice);
2441 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2442 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2443 if (fMaxInitFailed>0)
84fcd93f 2444 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2445 if (fMasterResubmitThreshold>0)
84fcd93f 2446 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2447 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2448 if (fNrunsPerMaster>0)
84fcd93f 2449 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2450 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2451 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2452 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2453 if (fArguments.Length())
84fcd93f 2454 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2455 if (fExecutableArgs.Length())
2456 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2457 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2458 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2459 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2460 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2461 if (fDatasetName)
84fcd93f 2462 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2463 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2464 if (fIncludePath.Data())
84fcd93f 2465 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2466 if (fCloseSE.Length())
84fcd93f 2467 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2468 if (fFriendChainName.Length())
84fcd93f 2469 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2470 if (fPackages && fPackages->GetEntries()) {
16a4353c 2471 TIter next(fPackages);
2472 TObject *obj;
2473 TString list;
2474 while ((obj=next())) list += obj->GetName();
84fcd93f 2475 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2476 }
2477}
2478
c57f56b7 2479//______________________________________________________________________________
2480void AliAnalysisAlien::SetDefaults()
2481{
2482// Set default values for everything. What cannot be filled will be left empty.
2483 if (fGridJDL) delete fGridJDL;
2484 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2485 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2486 fPrice = 1;
2487 fTTL = 30000;
2488 fSplitMaxInputFileNumber = 100;
2489 fMaxInitFailed = 0;
2490 fMasterResubmitThreshold = 0;
bb885a9e 2491 fNtestFiles = 10;
149d288c 2492 fNreplicas = 2;
a8739e8a 2493 fRunRange[0] = 0;
2494 fRunRange[1] = 0;
8e3bd06a 2495 fRunPrefix = "%d";
319593fb 2496 fNrunsPerMaster = 1;
16a4353c 2497 fMaxMergeFiles = 100;
c57f56b7 2498 fRunNumbers = "";
2499 fExecutable = "analysis.sh";
0a1c1f7f 2500 fExecutableCommand = "root -b -q";
c57f56b7 2501 fArguments = "";
631c0b05 2502 fExecutableArgs = "";
c57f56b7 2503 fAnalysisMacro = "myAnalysis.C";
2504 fAnalysisSource = "";
2505 fAdditionalLibs = "";
2506 fSplitMode = "se";
2507 fAPIVersion = "";
2508 fROOTVersion = "";
2509 fAliROOTVersion = "";
2510 fUser = ""; // Your alien user name
2511 fGridWorkingDir = "";
2512 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2513 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2514 fFriendChainName = "";
c57f56b7 2515 fGridOutputDir = "output";
b83f84f0 2516 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2517 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2518 fInputFormat = "xml-single";
2519 fJDLName = "analysis.jdl";
c6cb3634 2520 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2521 fMergeExcludes = "";
0f389141 2522 fMergeViaJDL = 0;
348be253 2523 SetUseCopy(kTRUE);
2524 SetCheckCopy(kTRUE);
149d288c 2525 SetDefaultOutputs(kTRUE);
e1c22e21 2526 fOverwriteMode = 1;
c57f56b7 2527}
2528
0f389141 2529//______________________________________________________________________________
b3e07543 2530Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2531{
b3e07543 2532// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2533 // First check if the result is already in the output directory.
2534 if (FileExists(Form("%s/%s",aliendir,filename))) {
2535 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2536 return kFALSE;
2537 }
b3e07543 2538 // Now check the last stage done.
2539 Int_t stage = 0;
2540 while (1) {
2541 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2542 stage++;
2543 }
2544 // Next stage of merging
2545 stage++;
2546 TString pattern = "*root_archive.zip";
2547 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2548 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2549 if (res) delete res;
2550 // Write standard output to file
2551 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2552 // Count the number of files inside
2553 ifstream ifile;
2554 ifile.open(Form("Stage_%d.xml",stage));
2555 if (!ifile.good()) {
2556 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2557 return kFALSE;
b3e07543 2558 }
2559 TString line;
2560 Int_t nfiles = 0;
2561 while (!ifile.eof()) {
2562 ifile >> line;
2563 if (line.Contains("/event")) nfiles++;
7c2cd90a 2564 }
b3e07543 2565 ifile.close();
2566 if (!nfiles) {
2567 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2568 return kFALSE;
2569 } else {
2570 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2571 }
b3e07543 2572 // Copy the file in the output directory
2573 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2574// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2575 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2576 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2577 // Check if this is the last stage to be done.
2578 Bool_t laststage = (nfiles<nperchunk);
2579 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2580 if (laststage) {
2581 printf("### Submiting final merging stage %d\n", stage);
2582 TString finalJDL = jdl;
2583 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2584 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2585 Int_t jobId = SubmitSingleJob(query);
2586 if (!jobId) return kFALSE;
2587 } else {
2588 printf("### Submiting merging stage %d\n", stage);
03335324 2589 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2590 Int_t jobId = SubmitSingleJob(query);
2591 if (!jobId) return kFALSE;
7c2cd90a 2592 }
b3e07543 2593 return kTRUE;
2594}
7c2cd90a 2595
105631b5 2596//______________________________________________________________________________
2597AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2598{
2599// Loat the analysis manager from a file.
2600 TFile *file = TFile::Open(fname);
2601 if (!file) {
2602 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2603 return 0;
2604 }
2605 TIter nextkey(file->GetListOfKeys());
2606 AliAnalysisManager *mgr = 0;
2607 TKey *key;
2608 while ((key=(TKey*)nextkey())) {
2609 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2610 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2611 }
2612 if (!mgr)
2613 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2614 return mgr;
2615}
2616
7c2cd90a 2617//______________________________________________________________________________
2618Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2619{
2620// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2621 if (!gGrid) return 0;
2622 printf("=> %s ------> ",query);
2623 TGridResult *res = gGrid->Command(query);
2624 if (!res) return 0;
2625 TString jobId = res->GetKey(0,"jobId");
2626 delete res;
2627 if (jobId.IsNull()) {
2628 printf("submission failed. Reason:\n");
2629 gGrid->Stdout();
2630 gGrid->Stderr();
2631 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2632 return 0;
2633 }
2634 printf(" Job id: %s\n", jobId.Data());
2635 return atoi(jobId);
2636}
2637
ec5acdce 2638//______________________________________________________________________________
2639Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2640{
2641// Merges a collection of output files using concatenation.
2642 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2643 if (!coll) {
2644 ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2645 return kFALSE;
2646 }
2647 // Iterate grid collection
2648 TString outtmp;
2649 Bool_t merged = kFALSE;
2650 Int_t ifile = 0;
2651 while (coll->Next()) {
2652 TString fname = gSystem->DirName(coll->GetTURL());
2653 fname += "/";
2654 fname += output;
2655 outtmp = Form("%d_%s", ifile, output);
2656 if (!TFile::Cp(fname, outtmp)) {
2657 ::Error("MergeInfo", "Could not copy %s", fname.Data());
2658 continue;
2659 }
2660 ifile++;
2661 if (ifile<2) {
2662 gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2663 continue;
2664 }
2665 gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2666 gSystem->Exec("cp tempmerged lastmerged");
2667 }
2668 if (ifile) {
2669 gSystem->Exec(Form("cp lastmerged %s", output));
2670 gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2671 merged = kTRUE;
2672 }
2673 return merged;
2674}
2675
7c2cd90a 2676//______________________________________________________________________________
b3e07543 2677Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2678{
b3e07543 2679// Merge given output files from basedir. Basedir can be an alien output directory
2680// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2681// files in a group (ignored for xml input). Merging can be done in stages:
2682// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2683// stage=1 : works with an xml of all root_archive.zip in the output directory
2684// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2685 TString outputFile = output;
0f389141 2686 TString command;
a2f5fc01 2687 TString outputChunk;
2688 TString previousChunk = "";
b3e07543 2689 TObjArray *listoffiles = new TObjArray();
2690// listoffiles->SetOwner();
a2f5fc01 2691 Int_t countChunk = 0;
2692 Int_t countZero = nmaxmerge;
0f389141 2693 Bool_t merged = kTRUE;
a2f5fc01 2694 Int_t index = outputFile.Index("@");
2695 if (index > 0) outputFile.Remove(index);
7c2cd90a 2696 TString inputFile = outputFile;
b3e07543 2697 TString sbasedir = basedir;
2698 if (sbasedir.Contains(".xml")) {
2699 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2700 nmaxmerge = 9999999;
2701 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2702 if (!coll) {
2703 ::Error("MergeOutput", "Input XML collection empty.");
2704 return kFALSE;
2705 }
2706 // Iterate grid collection
2707 while (coll->Next()) {
2708 TString fname = gSystem->DirName(coll->GetTURL());
2709 fname += "/";
2710 fname += inputFile;
2711 listoffiles->Add(new TNamed(fname.Data(),""));
2712 }
2713 } else {
2714 command = Form("find %s/ *%s", basedir, inputFile.Data());
2715 printf("command: %s\n", command.Data());
2716 TGridResult *res = gGrid->Command(command);
2717 if (!res) {
2718 ::Error("MergeOutput","No result for the find command\n");
2719 delete listoffiles;
2720 return kFALSE;
2721 }
2722 TIter nextmap(res);
2723 TMap *map = 0;
2724 while ((map=(TMap*)nextmap())) {
2725 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2726 if (!objs || !objs->GetString().Length()) {
2727 // Nothing found - skip this output
2728 delete res;
2729 delete listoffiles;
2730 return kFALSE;
2731 }
2732 listoffiles->Add(new TNamed(objs->GetName(),""));
2733 }
2734 delete res;
2735 }
2736 if (!listoffiles->GetEntries()) {
7c2cd90a 2737 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2738 delete listoffiles;
0f389141 2739 return kFALSE;
2740 }
2741
2742 TFileMerger *fm = 0;
b3e07543 2743 TIter next0(listoffiles);
2744 TObjArray *listoffilestmp = new TObjArray();
2745 listoffilestmp->SetOwner();
2746 TObject *nextfile;
2747 TString snextfile;
2748 // Keep only the files at upper level
2749 Int_t countChar = 0;
2750 while ((nextfile=next0())) {
2751 snextfile = nextfile->GetName();
2752 Int_t crtCount = snextfile.CountChar('/');
2753 if (nextfile == listoffiles->First()) countChar = crtCount;
2754 if (crtCount < countChar) countChar = crtCount;
2755 }
2756 next0.Reset();
2757 while ((nextfile=next0())) {
2758 snextfile = nextfile->GetName();
2759 Int_t crtCount = snextfile.CountChar('/');
2760 if (crtCount > countChar) {
2761 delete nextfile;
2762 continue;
2763 }
2764 listoffilestmp->Add(nextfile);
2765 }
2766 delete listoffiles;
2767 listoffiles = listoffilestmp; // Now contains 'good' files
2768 listoffiles->Print();
2769 TIter next(listoffiles);
7c2cd90a 2770 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2771 outputChunk = outputFile;
2772 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2773 // Check for existent temporary merge files
e1c22e21 2774 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2775 // Preserve old merging functionality for stage 0.
2776 if (stage==0) {
2777 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2778 while (1) {
2779 // Skip as many input files as in a chunk
2ea55496 2780 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2781 nextfile = next();
2782 if (!nextfile) {
2ea55496 2783 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2784 delete listoffiles;
2ea55496 2785 return kFALSE;
2786 }
b3e07543 2787 snextfile = nextfile->GetName();
7c2cd90a 2788 }
2789 outputChunk = outputFile;
2790 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2791 countChunk++;
2792 if (gSystem->AccessPathName(outputChunk)) continue;
2793 // Merged file with chunks up to <countChunk> found
2794 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2795 previousChunk = outputChunk;
2796 break;
2797 }
2798 }
2799 countZero = nmaxmerge;
2800
b3e07543 2801 while ((nextfile=next())) {
2802 snextfile = nextfile->GetName();
2ea55496 2803 // Loop 'find' results and get next LFN
7c2cd90a 2804 if (countZero == nmaxmerge) {
2805 // First file in chunk - create file merger and add previous chunk if any.
2b895f34 2806 fm = new TFileMerger(kTRUE);
7c2cd90a 2807 fm->SetFastMethod(kTRUE);
2808 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2809 outputChunk = outputFile;
2810 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2811 }
2812 // If last file found, put merged results in the output file
b3e07543 2813 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2814 // Add file to be merged and decrement chunk counter.
b3e07543 2815 fm->AddFile(snextfile);
7c2cd90a 2816 countZero--;
b3e07543 2817 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2818 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2819 // Nothing found - skip this output
2820 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2821 merged = kFALSE;
2822 break;
7c2cd90a 2823 }
2824 fm->OutputFile(outputChunk);
2825 // Merge the outputs, then go to next chunk
2826 if (!fm->Merge()) {
2827 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2828 merged = kFALSE;
2829 break;
7c2cd90a 2830 } else {
2831 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2832 gSystem->Unlink(previousChunk);
2833 }
b3e07543 2834 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2835 countChunk++;
2836 countZero = nmaxmerge;
2837 previousChunk = outputChunk;
0f389141 2838 }
0f389141 2839 }
b3e07543 2840 delete listoffiles;
2841 delete fm;
7c2cd90a 2842 return merged;
2843 }
2844 // Merging stage different than 0.
2845 // Move to the begining of the requested chunk.
2b895f34 2846 fm = new TFileMerger(kTRUE);
7c2cd90a 2847 fm->SetFastMethod(kTRUE);
b3e07543 2848 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2849 delete listoffiles;
7c2cd90a 2850 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2851 // Nothing found - skip this output
2852 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2853 delete fm;
2854 return kFALSE;
2855 }
b3e07543 2856 fm->OutputFile(outputFile);
7c2cd90a 2857 // Merge the outputs
2858 if (!fm->Merge()) {
2859 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2860 delete fm;
2861 return kFALSE;
2862 } else {
b3e07543 2863 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2864 }
7c2cd90a 2865 delete fm;
2866 return kTRUE;
0f389141 2867}
2868
c57f56b7 2869//______________________________________________________________________________
2870Bool_t AliAnalysisAlien::MergeOutputs()
2871{
2872// Merge analysis outputs existing in the AliEn space.
2873 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2874 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2875 if (!Connect()) {
2876 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2877 return kFALSE;
0f389141 2878 }
c9e8f7fd 2879 if (fMergeViaJDL) {
2880 if (!TestBit(AliAnalysisGrid::kMerge)) {
2881 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2882 return kFALSE;
2883 }
2884 if (fProductionMode) {
2885 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2886 return kFALSE;
2887 }
0f389141 2888 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2889 if (!SubmitMerging()) return kFALSE;
0f389141 2890 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2891 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2892 return kFALSE;
c57f56b7 2893 }
2894 // Get the output path
dc030a24 2895 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2896 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2897 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2898 return kFALSE;
2899 }
2900 if (!fOutputFiles.Length()) {
2901 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2902 return kFALSE;
0f389141 2903 }
2904 // Check if fast read option was requested
e1c22e21 2905 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2906 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2907 if (fFastReadOption) {
d3339be3 2908 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2909 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2910 gEnv->SetValue("XNet.ConnectTimeout",50);
2911 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2912 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2913 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2914 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2915 }
e8b839ab 2916 // Make sure we change the temporary directory
2917 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2918 // Set temporary compilation directory to current one
2919 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2920 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2921 TIter next(list);
2922 TObjString *str;
a2f5fc01 2923 TString outputFile;
c57f56b7 2924 Bool_t merged = kTRUE;
2925 while((str=(TObjString*)next())) {
a2f5fc01 2926 outputFile = str->GetString();
2927 Int_t index = outputFile.Index("@");
2928 if (index > 0) outputFile.Remove(index);
2929 TString outputChunk = outputFile;
2930 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2931 // Skip already merged outputs
a2f5fc01 2932 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2933 if (fOverwriteMode) {
a2f5fc01 2934 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2935 gSystem->Unlink(outputFile);
2936 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2937 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2938 outputChunk.Data());
2939 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2940 }
2941 } else {
a2f5fc01 2942 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2943 continue;
2944 }
2945 } else {
a2f5fc01 2946 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2947 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2948 outputChunk.Data());
2949 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2950 }
2951 }
37d21c01 2952 if (fMergeExcludes.Contains(outputFile.Data()) ||
2953 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 2954 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2955 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2956 if (!merged) {
2957 Error("MergeOutputs", "Terminate() will NOT be executed");
2958 return kFALSE;
ff07ec61 2959 }
2960 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2961 if (fileOpened) fileOpened->Close();
c57f56b7 2962 }
0f389141 2963 return kTRUE;
c57f56b7 2964}
2965
bb885a9e 2966//______________________________________________________________________________
2967void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2968{
2969// Use the output files connected to output containers from the analysis manager
2970// rather than the files defined by SetOutputFiles
2971 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2972 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2973 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2974}
2975
149d288c 2976//______________________________________________________________________________
2977void AliAnalysisAlien::SetOutputFiles(const char *list)
2978{
2979// Manually set the output files list.
2980// Removes duplicates. Not allowed if default outputs are not disabled.
2981 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2982 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2983 return;
2984 }
2985 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2986 fOutputFiles = "";
2987 TString slist = list;
2988 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2989 TObjArray *arr = slist.Tokenize(" ");
2990 TObjString *os;
2991 TIter next(arr);
2992 TString sout;
2993 while ((os=(TObjString*)next())) {
2994 sout = os->GetString();
2995 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2996 if (fOutputFiles.Contains(sout)) continue;
2997 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2998 fOutputFiles += sout;
2999 }
3000 delete arr;
f790bc1b 3001}
149d288c 3002
3003//______________________________________________________________________________
3004void AliAnalysisAlien::SetOutputArchive(const char *list)
3005{
3006// Manually set the output archive list. Free text - you are on your own...
3007// Not allowed if default outputs are not disabled.
3008 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3009 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
3010 return;
3011 }
3012 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
3013 fOutputArchive = list;
3014}
3015
3016//______________________________________________________________________________
3017void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
3018{
3019// Setting a prefered output SE is not allowed anymore.
3020 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
3021}
3022
28ce6c9c 3023//______________________________________________________________________________
3024void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
3025{
3026// Set some PROOF special parameter.
3027 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3028 if (pair) {
3029 TObject *old = pair->Key();
3030 TObject *val = pair->Value();
3031 fProofParam.Remove(old);
3032 delete old;
3033 delete val;
3034 }
3035 fProofParam.Add(new TObjString(pname), new TObjString(value));
3036}
3037
3038//______________________________________________________________________________
3039const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
3040{
3041// Returns a special PROOF parameter.
3042 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3043 if (!pair) return 0;
3044 return pair->Value()->GetName();
3045}
3046
5513444a 3047//______________________________________________________________________________
3048Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 3049{
3050// Start remote grid analysis.
3bdcb562 3051 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3052 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
3053 if (!mgr || !mgr->IsInitialized()) {
3054 Error("StartAnalysis", "You need an initialized analysis manager for this");
3055 return kFALSE;
3056 }
3057 // Are we in PROOF mode ?
3058 if (mgr->IsProofMode()) {
baa34407 3059 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3060 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3061 if (fProofCluster.IsNull()) {
3062 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3063 return kFALSE;
3064 }
3065 if (fProofDataSet.IsNull() && !testMode) {
3066 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3067 return kFALSE;
3068 }
3069 // Set the needed environment
3070 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3071 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3072 if (fProofReset && !testMode) {
3073 if (fProofReset==1) {
3074 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3075 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3076 } else {
3077 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3078 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3079 }
3080 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3081 return kFALSE;
3082 }
baa34407 3083
3084 if (!testMode) {
3085 // Check if there is an old active session
3086 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3087 if (nsessions) {
3088 Error("StartAnalysis","You have to reset your old session first\n");
3089 return kFALSE;
3090 }
53911ca6 3091 }
3bdcb562 3092 // Do we need to change the ROOT version ? The success of this cannot be checked.
3093 if (!fRootVersionForProof.IsNull() && !testMode) {
3094 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
3095 fProofCluster.Data(), fRootVersionForProof.Data()));
3096 }
3097 // Connect to PROOF and check the status
3098 Long_t proof = 0;
27734f0e 3099 TString sworkers;
3100 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3101 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3102 if (!testMode) {
27734f0e 3103 if (!sworkers.IsNull())
3104 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3105 else
3106 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3107 } else {
3108 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3109 if (!proof) {
3110 Error("StartAnalysis", "Could not start PROOF in test mode");
3111 return kFALSE;
3112 }
3113 }
3114 if (!proof) {
3115 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3116 return kFALSE;
3117 }
27734f0e 3118 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3119 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3120 // Set proof special parameters if any
3121 TIter nextpp(&fProofParam);
3122 TObject *proofparam;
3123 while ((proofparam=nextpp())) {
3124 TString svalue = GetProofParameter(proofparam->GetName());
3125 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3126 }
3bdcb562 3127 // Is dataset existing ?
3128 if (!testMode) {
3129 TString dataset = fProofDataSet;
3130 Int_t index = dataset.Index("#");
3131 if (index>=0) dataset.Remove(index);
830acc4c 3132// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3133// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3134// return kFALSE;
3135// }
3136// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3137 }
3138 // Is ClearPackages() needed ?
3139 if (TestSpecialBit(kClearPackages)) {
3140 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3141 gROOT->ProcessLine("gProof->ClearPackages();");
3142 }
3143 // Is a given aliroot mode requested ?
3144 TList optionsList;
1ac846e9 3145 TString parLibs;
830acc4c 3146 if (!fAliRootMode.IsNull()) {
3bdcb562 3147 TString alirootMode = fAliRootMode;
3148 if (alirootMode == "default") alirootMode = "";
3149 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3150 optionsList.SetOwner();
3151 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3152 // Check the additional libs to be loaded
3153 TString extraLibs;
1ac846e9 3154 Bool_t parMode = kFALSE;
dc1ffbcb 3155 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3156 // Parse the extra libs for .so
3157 if (fAdditionalLibs.Length()) {
3158 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3159 TIter next(list);
3160 TObjString *str;
1ac846e9 3161 while((str=(TObjString*)next())) {
3162 if (str->GetString().Contains(".so")) {
3163 if (parMode) {
3164 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3165 break;
3166 }
3167 TString stmp = str->GetName();
3168 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3169 stmp.ReplaceAll(".so","");
3170 if (!extraLibs.IsNull()) extraLibs += ":";
3171 extraLibs += stmp;
3172 continue;
3173 }
3174 if (str->GetString().Contains(".par")) {
3175 // The first par file found in the list will not allow any further .so
3176 parMode = kTRUE;
3177 if (!parLibs.IsNull()) parLibs += ":";
3178 parLibs += str->GetName();
3179 continue;
3180 }
3bdcb562 3181 }
3182 if (list) delete list;
3183 }
dc030a24 3184 if (!extraLibs.IsNull()) {
3185 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3186 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3187 }
830acc4c 3188 // Check extra includes
3189 if (!fIncludePath.IsNull()) {
3190 TString includePath = fIncludePath;
3191 includePath.ReplaceAll(" ",":");
dc030a24 3192 includePath.ReplaceAll("$ALICE_ROOT/","");
3193 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3194 includePath.ReplaceAll("-I","");
dc030a24 3195 includePath.Remove(TString::kTrailing, ':');
830acc4c 3196 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3197 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3198 }
27734f0e 3199 // Check if connection to grid is requested
3200 if (TestSpecialBit(kProofConnectGrid))
3201 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3202 // Enable AliRoot par
3203 if (testMode) {
3204 // Enable proof lite package
3205 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3206 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3207 TNamed *obj = (TNamed*)optionsList.At(i);
3208 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3209 }
3210 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3211 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3212 Info("StartAnalysis", "AliRootProofLite enabled");
3213 } else {
3214 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3215 return kFALSE;
3216 }
3217 } else {
dc030a24 3218 if ( ! fAliROOTVersion.IsNull() ) {
3219 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3220 fAliROOTVersion.Data(), &optionsList))) {
3221 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3222 return kFALSE;
3223 }
3224 }
27734f0e 3225 }
1ac846e9 3226 // Enable first par files from fAdditionalLibs
3227 if (!parLibs.IsNull()) {
3228 TObjArray *list = parLibs.Tokenize(":");
3229 TIter next(list);
3230 TObjString *package;
3231 while((package=(TObjString*)next())) {
40501bb7 3232 TString spkg = package->GetName();
3233 spkg.ReplaceAll(".par", "");
3234 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3235 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3236 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3237 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3238 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3239 return kFALSE;
3240 }
3241 } else {
3242 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3243 return kFALSE;
3244 }
3245 }
3246 if (list) delete list;
3247 }
3bdcb562 3248 } else {
3249 if (fAdditionalLibs.Contains(".so") && !testMode) {
3250 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3251 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3252 return kFALSE;
3253 }
3254 }
3255 // Enable par files if requested
3256 if (fPackages && fPackages->GetEntries()) {
3257 TIter next(fPackages);
3258 TObject *package;
3259 while ((package=next())) {
1ac846e9 3260 // Skip packages already enabled
3261 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3262 TString spkg = package->GetName();
3263 spkg.ReplaceAll(".par", "");
3264 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3265 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3266 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3267 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3268 return kFALSE;
3269 }
3270 } else {
3271 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3272 return kFALSE;
3273 }
3274 }
3275 }
3276 // Do we need to load analysis source files ?
3277 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3278 if (fAnalysisSource.Length()) {
3279 TObjArray *list = fAnalysisSource.Tokenize(" ");
3280 TIter next(list);
3281 TObjString *str;
3282 while((str=(TObjString*)next())) {
3283 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3284 }
3bdcb562 3285 if (list) delete list;
3286 }
830acc4c 3287 if (testMode) {
3288 // Register dataset to proof lite.
3289 if (fFileForTestMode.IsNull()) {
3290 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3291 return kFALSE;
3292 }
3293 if (gSystem->AccessPathName(fFileForTestMode)) {
3294 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3295 return kFALSE;
3296 }
3297 TFileCollection *coll = new TFileCollection();
3298 coll->AddFromFile(fFileForTestMode);
8ddad121 3299 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3300 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3301 }
3bdcb562 3302 return kTRUE;
3303 }
c57f56b7 3304
43da816a 3305 // Check if output files have to be taken from the analysis manager
3306 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3307 // Add output files and AOD files
3308 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3309 // Add extra files registered to the analysis manager
f790bc1b 3310 TString extra = GetListOfFiles("ext");
3311 if (!extra.IsNull()) {
149d288c 3312 extra.ReplaceAll(".root", "*.root");
f790bc1b 3313 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3314 fOutputFiles += extra;
84fcd93f 3315 }
149d288c 3316 // Compose the output archive.
b83f84f0 3317 fOutputArchive = "log_archive.zip:std*@disk=1 ";
ec5acdce 3318 if (mgr->IsCollectThroughput())
3319 fOutputArchive += Form("root_archive.zip:%s,*.stat,*%s@disk=%d",fOutputFiles.Data(),mgr->GetFileInfoLog(),fNreplicas);
3320 else
3321 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3322 }
f7b1cbc2 3323// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3324 if (TestBit(AliAnalysisGrid::kOffline)) {
3325 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3326 \n there nor any job run. You can revise the JDL and analysis \
3327 \n macro then run the same in \"submit\" mode.");
3328 } else if (TestBit(AliAnalysisGrid::kTest)) {
3329 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3330 \n dataset.");
3331 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3332 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3333 \n space and job submitted.");
3334 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3335 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3336 if (fMergeViaJDL) CheckInputData();
5513444a 3337 return kTRUE;
c57f56b7 3338 } else {
3339 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3340 }
3341
348be253 3342 Print();
c57f56b7 3343 if (!Connect()) {
3344 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3345 return kFALSE;
16a4353c 3346 }
d3b18c4c 3347 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3348 if (!CheckInputData()) {
3349 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3350 return kFALSE;
c57f56b7 3351 }
d3339be3 3352 if (!CreateDataset(fDataPattern)) {
3353 TString serror;
3354 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3355 if (fRunNumbers.Length()) serror = "run numbers";
3356 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3357 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3358 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3359 return kFALSE;
3360 }
21ca8e59 3361 WriteAnalysisFile();
c57f56b7 3362 WriteAnalysisMacro();
3363 WriteExecutable();
3364 WriteValidationScript();
0f389141 3365 if (fMergeViaJDL) {
3366 WriteMergingMacro();
3367 WriteMergeExecutable();
3368 WriteValidationScript(kTRUE);
3369 }
5513444a 3370 if (!CreateJDL()) return kFALSE;
3371 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3372 if (testMode) {
c57f56b7 3373 // Locally testing the analysis
3374 Info("StartAnalysis", "\n_______________________________________________________________________ \
3375 \n Running analysis script in a daughter shell as on a worker node \
3376 \n_______________________________________________________________________");
149d288c 3377 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3378 TIter next(list);
3379 TObjString *str;
a2f5fc01 3380 TString outputFile;
c57f56b7 3381 while((str=(TObjString*)next())) {
a2f5fc01 3382 outputFile = str->GetString();
3383 Int_t index = outputFile.Index("@");
3384 if (index > 0) outputFile.Remove(index);
3385 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3386 }
3387 delete list;
3388 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3389 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3390// gSystem->Exec("cat stdout");
5513444a 3391 return kFALSE;
c57f56b7 3392 }
5513444a 3393 // Check if submitting is managed by LPM manager
a3e84053 3394 if (fProductionMode) {
7ac61826 3395 //TString prodfile = fJDLName;
3396 //prodfile.ReplaceAll(".jdl", ".prod");
3397 //WriteProductionFile(prodfile);
5513444a 3398 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3399 return kFALSE;
3400 }
a8739e8a 3401 // Submit AliEn job(s)
d2a409b2 3402 gGrid->Cd(fGridOutputDir);
a8739e8a 3403 TGridResult *res;
c57f56b7 3404 TString jobID = "";
d2a409b2 3405 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3406 // Submit a given xml or a set of runs
a8739e8a 3407 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3408 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3409 if (res) {
3410 const char *cjobId = res->GetKey(0,"jobId");
3411 if (!cjobId) {
a03be957 3412 gGrid->Stdout();
3413 gGrid->Stderr();
a8739e8a 3414 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3415 return kFALSE;
a8739e8a 3416 } else {
3417 Info("StartAnalysis", "\n_______________________________________________________________________ \
3418 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3419 \n_______________________________________________________________________",
3420 fJDLName.Data(), cjobId);
3421 jobID = cjobId;
3422 }
3423 delete res;
a03be957 3424 } else {
3425 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3426 return kFALSE;
a8739e8a 3427 }
3428 } else {
d2a409b2 3429 // Submit for a range of enumeration of runs.
a03be957 3430 if (!Submit()) return kFALSE;
c57f56b7 3431 }
a8739e8a 3432
c57f56b7 3433 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3434 \n You may exit at any time and terminate the job later using the option <terminate> \
3435 \n ##################################################################################", jobID.Data());
bb885a9e 3436 gSystem->Exec("aliensh");
5513444a 3437 return kTRUE;
c57f56b7 3438}
3439
f790bc1b 3440//______________________________________________________________________________
3441const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3442{
3443// Get a comma-separated list of output files of the requested type.
3444// Type can be (case unsensitive):
3445// aod - list of aod files (std, extensions and filters)
3446// out - list of output files connected to containers (but not aod's or extras)
3447// ext - list of extra files registered to the manager
3448// ter - list of files produced in terminate
3449 static TString files;
3450 files = "";
3451 TString stype = type;
3452 stype.ToLower();
3453 TString aodfiles, extra;
3454 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3455 if (!mgr) {
3456 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3457 return files.Data();
3458 }
3459 if (mgr->GetOutputEventHandler()) {
3460 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3461 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3462 if (!extraaod.IsNull()) {
3463 aodfiles += ",";
3464 aodfiles += extraaod;
3465 }
3466 }
3467 if (stype.Contains("aod")) {
3468 files = aodfiles;
3469 if (stype == "aod") return files.Data();
3470 }
3471 // Add output files that are not in the list of AOD files
3472 TString outputfiles = "";
3473 TIter next(mgr->GetOutputs());
3474 AliAnalysisDataContainer *output;
3475 const char *filename = 0;
3476 while ((output=(AliAnalysisDataContainer*)next())) {
3477 filename = output->GetFileName();
3478 if (!(strcmp(filename, "default"))) continue;
3479 if (outputfiles.Contains(filename)) continue;
3480 if (aodfiles.Contains(filename)) continue;
3481 if (!outputfiles.IsNull()) outputfiles += ",";
3482 outputfiles += filename;
3483 }
3484 if (stype.Contains("out")) {
3485 if (!files.IsNull()) files += ",";
3486 files += outputfiles;
3487 if (stype == "out") return files.Data();
3488 }
3489 // Add extra files registered to the analysis manager
3490 TString sextra;
3491 extra = mgr->GetExtraFiles();
3492 if (!extra.IsNull()) {
3493 extra.Strip();
3494 extra.ReplaceAll(" ", ",");
3495 TObjArray *fextra = extra.Tokenize(",");
3496 TIter nextx(fextra);
3497 TObject *obj;
3498 while ((obj=nextx())) {
3499 if (aodfiles.Contains(obj->GetName())) continue;
3500 if (outputfiles.Contains(obj->GetName())) continue;
3501 if (sextra.Contains(obj->GetName())) continue;
3502 if (!sextra.IsNull()) sextra += ",";
3503 sextra += obj->GetName();
3504 }
3505 delete fextra;
3506 if (stype.Contains("ext")) {
3507 if (!files.IsNull()) files += ",";
3508 files += sextra;
3509 }
3510 }
3511 if (stype == "ext") return files.Data();
3512 TString termfiles;
3513 if (!fTerminateFiles.IsNull()) {
3514 fTerminateFiles.Strip();
3515 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3516 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3517 TIter nextx(fextra);
3518 TObject *obj;
3519 while ((obj=nextx())) {
3520 if (aodfiles.Contains(obj->GetName())) continue;
3521 if (outputfiles.Contains(obj->GetName())) continue;
3522 if (termfiles.Contains(obj->GetName())) continue;
3523 if (sextra.Contains(obj->GetName())) continue;
3524 if (!termfiles.IsNull()) termfiles += ",";
3525 termfiles += obj->GetName();
3526 }
3527 delete fextra;
3528 }
3529 if (stype.Contains("ter")) {
3530 if (!files.IsNull() && !termfiles.IsNull()) {
3531 files += ",";
3532 files += termfiles;
3533 }
3534 }
3535 return files.Data();
3536}
3537
d2a409b2 3538//______________________________________________________________________________
a03be957 3539Bool_t AliAnalysisAlien::Submit()
d2a409b2 3540{
3541// Submit all master jobs.
3542 Int_t nmasterjobs = fInputFiles->GetEntries();
3543 Long_t tshoot = gSystem->Now();
a03be957 3544 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3545 while (fNsubmitted < nmasterjobs) {
3546 Long_t now = gSystem->Now();
3547 if ((now-tshoot)>30000) {
3548 tshoot = now;
a03be957 3549 if (!SubmitNext()) return kFALSE;
d2a409b2 3550 }
3551 }
a03be957 3552 return kTRUE;
d2a409b2 3553}
3554
0f389141 3555//______________________________________________________________________________
a03be957 3556Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3557{
3558// Submit all merging jobs.
dc030a24 3559 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3560 gGrid->Cd(fGridOutputDir);
3561 TString mergeJDLName = fExecutable;
3562 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3563 if (!fInputFiles) {
3564 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3565 return kFALSE;
3566 }
0f389141 3567 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3568 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3569 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3570 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3571 if (fOutputToRunNo) {
3572 // The output directory is the run number
3573 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3574 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3575 } else {
17b4e9a4 3576 if (!fRunNumbers.Length() && !fRunRange[0]) {
3577 // The output directory is the grid outdir
3578 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3579 runOutDir = fGridOutputDir;
3580 } else {
3581 // The output directory is the master number in 3 digits format
3582 printf("### Submitting merging job for master <%03d>\n", i);
3583 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3584 }
7c2cd90a 3585 }
3586 // Check now the number of merging stages.
2b222f7c 3587 TObjArray *list = fOutputFiles.Tokenize(",");
3588 TIter next(list);
3589 TObjString *str;
3590 TString outputFile;
3591 while((str=(TObjString*)next())) {
3592 outputFile = str->GetString();
3593 Int_t index = outputFile.Index("@");
3594 if (index > 0) outputFile.Remove(index);
37d21c01 3595 if (!fMergeExcludes.Contains(outputFile) &&
3596 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3597 }
3598 delete list;
b3e07543 3599 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3600 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3601 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3602 }
a03be957 3603 if (!ntosubmit) return kTRUE;
9f020952 3604 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3605 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3606 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3607 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3608 \n ################################################################################################################");
0f389141 3609 gSystem->Exec("aliensh");
a03be957 3610 return kTRUE;
0f389141 3611}
3612
d2a409b2 3613//______________________________________________________________________________
a03be957 3614Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3615{
08d5b699 3616// Submit next bunch of master jobs if the queue is free. The first master job is
3617// submitted right away, while the next will not be unless the previous was split.
3618// The plugin will not submit new master jobs if there are more that 500 jobs in
3619// waiting phase.
d2a409b2 3620 static Bool_t iscalled = kFALSE;
3621 static Int_t firstmaster = 0;
3622 static Int_t lastmaster = 0;
3623 static Int_t npermaster = 0;
a03be957 3624 if (iscalled) return kTRUE;
d2a409b2 3625 iscalled = kTRUE;
3626 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3627 Int_t ntosubmit = 0;
3628 TGridResult *res;
3629 TString jobID = "";
3bdcb562 3630 Int_t nmasterjobs = fInputFiles->GetEntries();
3631 if (!fNsubmitted) {
3632 ntosubmit = 1;
3633 if (!IsUseSubmitPolicy()) {
90d50a8c 3634 if (nmasterjobs>5)
3bdcb562 3635 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3636 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3637 ntosubmit = nmasterjobs;
3638 }
3639 } else {
d2a409b2 3640 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3641 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3642 // If last master not split, just return
a03be957 3643 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3644 // No more than 100 waiting jobs
08d5b699 3645 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3646 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3647 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3648 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3649 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3650 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3651 }
d2a409b2 3652 for (Int_t i=0; i<ntosubmit; i++) {
3653 // Submit for a range of enumeration of runs.
a03be957 3654 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3655 TString query;
cd11251e 3656 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3657 runOutDir.ReplaceAll(".xml", "");
3658 if (fOutputToRunNo)
3659 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3660 else
3661 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3662 printf("********* %s\n",query.Data());
d2a409b2 3663 res = gGrid->Command(query);
3664 if (res) {
98ca124f 3665 TString cjobId1 = res->GetKey(0,"jobId");
3666 if (!cjobId1.Length()) {
d2a409b2 3667 iscalled = kFALSE;
a03be957 3668 gGrid->Stdout();
3669 gGrid->Stderr();
3670 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3671 return kFALSE;
d2a409b2 3672 } else {
3673 Info("StartAnalysis", "\n_______________________________________________________________________ \
3674 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3675 \n_______________________________________________________________________",
98ca124f 3676 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 3677 jobID += cjobId1;
3678 jobID += " ";
98ca124f 3679 lastmaster = cjobId1.Atoi();
d2a409b2 3680 if (!firstmaster) firstmaster = lastmaster;
3681 fNsubmitted++;
3682 }
3683 delete res;
a03be957 3684 } else {
3685 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3686 return kFALSE;
d2a409b2 3687 }
3688 }
3689 iscalled = kFALSE;
a03be957 3690 return kTRUE;
d2a409b2 3691}
3692
c57f56b7 3693//______________________________________________________________________________
3694void AliAnalysisAlien::WriteAnalysisFile()
3695{
f10e8481 3696// Write current analysis manager into the file <analysisFile>
3697 TString analysisFile = fExecutable;
3698 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3699 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3700 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3701 if (!mgr || !mgr->IsInitialized()) {
3702 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3703 return;
3704 }
3705 // Check analysis type
3706 TObject *handler;
3707 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3708 handler = (TObject*)mgr->GetInputEventHandler();
3709 if (handler) {
65eb22f8 3710 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3711 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3712 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3713 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3714 } else {
3715 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3716 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3717 }
c57f56b7 3718 }
3719 TDirectory *cdir = gDirectory;
f10e8481 3720 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3721 if (file) {
e16a394c 3722 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3723 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3724 // Unless merging makes no sense
3725 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3726 mgr->Write();
3727 delete file;
fe2d7fc2 3728 // Enable termination for local jobs
3729 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3730 }
3731 if (cdir) cdir->cd();
f10e8481 3732 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3733 }
3734 Bool_t copy = kTRUE;
d3b18c4c 3735 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3736 if (copy) {
3737 CdWork();
3738 TString workdir = gGrid->GetHomeDirectory();
3739 workdir += fGridWorkingDir;
d3b18c4c 3740 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3741 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3742 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3743 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3744 }
3745}
3746
3747//______________________________________________________________________________
3748void AliAnalysisAlien::WriteAnalysisMacro()
3749{
3750// Write the analysis macro that will steer the analysis in grid mode.
3751 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3752 ofstream out;
3753 out.open(fAnalysisMacro.Data(), ios::out);
3754 if (!out.good()) {
3755 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3756 return;
3757 }
5b273635 3758 Bool_t hasSTEERBase = kFALSE;
3759 Bool_t hasESD = kFALSE;
3760 Bool_t hasAOD = kFALSE;
3761 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3762 Bool_t hasOADB = kFALSE;
5b273635 3763 Bool_t hasANALYSISalice = kFALSE;
3764 Bool_t hasCORRFW = kFALSE;
c57f56b7 3765 TString func = fAnalysisMacro;
3766 TString type = "ESD";
3767 TString comment = "// Analysis using ";
c85cfc0f 3768 if (IsUseMCchain()) {
3769 type = "MC";
3770 comment += "MC";
3771 } else {
3772 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3773 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3774 type = "AOD";
3775 comment += "AOD";
3776 }
3777 }
0df6ccf2 3778 if (type!="AOD" && fFriendChainName!="") {
3779 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3780 return;
3781 }
c57f56b7 3782 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3783 else comment += " data";
3784 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3785 func.ReplaceAll(".C", "");
3786 out << "void " << func.Data() << "()" << endl;
3787 out << "{" << endl;
3788 out << comment.Data() << endl;
3789 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3790 out << " TStopwatch timer;" << endl;
3791 out << " timer.Start();" << endl << endl;
13f0c00b 3792 // Change temp directory to current one
fdee57f1 3793 if (!IsLocalTest()) {
3794 out << "// connect to AliEn and make the chain" << endl;
3795 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
3796 }
13f0c00b 3797 out << "// Set temporary merging directory to current one" << endl;
3798 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3799 out << "// Set temporary compilation directory to current one" << endl;
3800 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3801 // Reset existing include path
3802 out << "// Reset existing include path and add current directory first in the search" << endl;
3803 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3804 if (!fExecutableCommand.Contains("aliroot")) {
3805 out << "// load base root libraries" << endl;
3806 out << " gSystem->Load(\"libTree\");" << endl;
3807 out << " gSystem->Load(\"libGeom\");" << endl;
3808 out << " gSystem->Load(\"libVMC\");" << endl;
3809 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3810 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3811 }
d5c6455a 3812 if (fAdditionalRootLibs.Length()) {
47a4137d 3813 // in principle libtree /lib geom libvmc etc. can go into this list, too
3814 out << "// Add aditional libraries" << endl;
3815 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3816 TIter next(list);
3817 TObjString *str;
3818 while((str=(TObjString*)next())) {
3819 if (str->GetString().Contains(".so"))
3820 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3821 }
47a4137d 3822 if (list) delete list;
d5c6455a 3823 }
57377eb5 3824 out << "// Load analysis framework libraries" << endl;
652561d5 3825 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3826 if (!fPackages) {
f790bc1b 3827 if (!fExecutableCommand.Contains("aliroot")) {
3828 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3829 out << " gSystem->Load(\"libESD\");" << endl;
3830 out << " gSystem->Load(\"libAOD\");" << endl;
3831 }
4e5c5506 3832 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3833 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3834 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3835 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3836 } else {
4e5c5506 3837 TIter next(fPackages);
3838 TObject *obj;
57377eb5 3839 TString pkgname;
57377eb5 3840 while ((obj=next())) {
3841 pkgname = obj->GetName();
4478e6f1 3842 if (pkgname == "STEERBase" ||
3843 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3844 if (pkgname == "ESD" ||
3845 pkgname == "ESD.par") hasESD = kTRUE;
3846 if (pkgname == "AOD" ||
3847 pkgname == "AOD.par") hasAOD = kTRUE;
3848 if (pkgname == "ANALYSIS" ||
3849 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 3850 if (pkgname == "OADB" ||
3851 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 3852 if (pkgname == "ANALYSISalice" ||
3853 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3854 if (pkgname == "CORRFW" ||
3855 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3856 }
3857 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3858 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3859 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3860 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3861 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3862 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3863 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3864 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3865 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 3866 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
3867 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 3868 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3869 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3870 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3871 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3872 out << "// Compile other par packages" << endl;
3873 next.Reset();
fcc9bb6f 3874 while ((obj=next())) {
3875 pkgname = obj->GetName();
4478e6f1 3876 if (pkgname == "STEERBase" ||
3877 pkgname == "STEERBase.par" ||
3878 pkgname == "ESD" ||
3879 pkgname == "ESD.par" ||
3880 pkgname == "AOD" ||
3881 pkgname == "AOD.par" ||
3882 pkgname == "ANALYSIS" ||
3883 pkgname == "ANALYSIS.par" ||
7ea40ac8 3884 pkgname == "OADB" ||
3885 pkgname == "OADB.par" ||
4478e6f1 3886 pkgname == "ANALYSISalice" ||
3887 pkgname == "ANALYSISalice.par" ||
3888 pkgname == "CORRFW" ||
3889 pkgname == "CORRFW.par") continue;
5b273635 3890 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3891 }
4e5c5506 3892 }
b3e07543 3893 out << "// include path" << endl;
3894 // Get the include path from the interpreter and remove entries pointing to AliRoot
3895 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3896 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3897 out << " TIter nextpath(listpaths);" << endl;
3898 out << " TObjString *pname;" << endl;
3899 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3900 out << " TString current = pname->GetName();" << endl;
3901 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3902 out << " gSystem->AddIncludePath(current);" << endl;
3903 out << " }" << endl;
3904 out << " if (listpaths) delete listpaths;" << endl;
3905 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3906 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3907 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3908 if (fAdditionalLibs.Length()) {
3909 out << "// Add aditional AliRoot libraries" << endl;
3910 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3911 TIter next(list);
3912 TObjString *str;
3913 while((str=(TObjString*)next())) {
3914 if (str->GetString().Contains(".so"))
3915 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3916 if (str->GetString().Contains(".par"))
3917 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3918 }
3919 if (list) delete list;
3920 }
3921 out << endl;
c57f56b7 3922 out << "// analysis source to be compiled at runtime (if any)" << endl;
3923 if (fAnalysisSource.Length()) {
3924 TObjArray *list = fAnalysisSource.Tokenize(" ");
3925 TIter next(list);
3926 TObjString *str;
3927 while((str=(TObjString*)next())) {
3928 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3929 }
3930 if (list) delete list;
3931 }
3932 out << endl;
b3e07543 3933// out << " printf(\"Currently load libraries:\\n\");" << endl;
3934// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3935 if (fFastReadOption) {
d3339be3 3936 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3937 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3938 out << "// fast xrootd reading enabled" << endl;
3939 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3940 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3941 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3942 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3943 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3944 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 3945 }
c57f56b7 3946 out << "// read the analysis manager from file" << endl;
f10e8481 3947 TString analysisFile = fExecutable;
3948 analysisFile.ReplaceAll(".sh", ".root");
105631b5 3949 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
3950 << analysisFile << "\");" << endl;
3951 out << " if (!mgr) return;" << endl;
21ca8e59 3952 if (IsLocalTest()) {
3953 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
3954 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 3955 if (fFileForTestMode.IsNull())
3956 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
3957 else
3958 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 3959 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4b5d9e21 3960 if (!fFriendChainName.IsNull())
3961 out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\");" << endl;
024dfa52 3962 if (IsUseMCchain())
3963 out << " plugin->SetUseMCchain();" << endl;
21ca8e59 3964 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 3965 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 3966 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 3967 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
3968 } else {
7a366e0b 3969 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 3970 out << " mgr->SetNSysInfo(100);" << endl;
3971 }
21ca8e59 3972 }
c57f56b7 3973 out << " mgr->PrintStatus();" << endl;
52b6a92b 3974 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3975 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3976 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3977 } else {
9a8288ff 3978 if (TestBit(AliAnalysisGrid::kTest))
3979 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3980 else
3981 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3982 }
3983 }
21ca8e59 3984 if (!IsLocalTest()) {
242accb2 3985 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 3986 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
3987 } else {
3988 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 3989 }
f7498086 3990 out << " timer.Stop();" << endl;
3991 out << " timer.Print();" << endl;
c57f56b7 3992 out << "}" << endl << endl;
21ca8e59 3993 if (!IsLocalTest()) {
fcc9bb6f 3994 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3995 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3996 out << "{" << endl;
3997 out << "// Create a chain using url's from xml file" << endl;
242accb2 3998 out << " TString filename;" << endl;
3999 out << " Int_t run = 0;" << endl;
c85cfc0f 4000 if (IsUseMCchain()) {
4001 out << " TString treename = \"TE\";" << endl;
4002 } else {
4003 out << " TString treename = type;" << endl;
4004 out << " treename.ToLower();" << endl;
4005 out << " treename += \"Tree\";" << endl;
4006 }
e02fee64 4007 out << " printf(\"***************************************\\n\");" << endl;
4008 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
4009 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 4010 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
4011 out << " if (!coll) {" << endl;
4012 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
4013 out << " return NULL;" << endl;
4014 out << " }" << endl;
242accb2 4015 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 4016 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 4017 if(fFriendChainName!="") {
4018 out << " TChain *chainFriend = new TChain(treename);" << endl;
4019 }
c57f56b7 4020 out << " coll->Reset();" << endl;
0df6ccf2 4021 out << " while (coll->Next()) {" << endl;
242accb2 4022 out << " filename = coll->GetTURL("");" << endl;
4023 out << " if (mgr) {" << endl;
4024 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
4025 out << " if (nrun && nrun != run) {" << endl;
4026 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
4027 out << " mgr->SetRunFromPath(nrun);" << endl;
4028 out << " run = nrun;" << endl;
4029 out << " }" << endl;
4030 out << " }" << endl;
4031 out << " chain->Add(filename);" << endl;
0df6ccf2 4032 if(fFriendChainName!="") {
4033 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
4b5d9e21 4034 out << " if (fileFriend.Index(\"#\") > -1) fileFriend.Remove(fileFriend.Index(\"#\"));" << endl;
4035 out << " fileFriend = gSystem->DirName(fileFriend);" << endl;
4036 out << " fileFriend += \"/\";" << endl;
4037 out << " fileFriend += \"" << fFriendChainName << "\";";
4038 out << " TFile *file = TFile::Open(fileFriend);" << endl;
4039 out << " if (file) {" << endl;
4040 out << " file->Close();" << endl;
4041 out << " chainFriend->Add(fileFriend.Data());" << endl;
4042 out << " } else {" << endl;
4043 out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
4044 out << " return 0;" << endl;
4045 out << " }" << endl;
0df6ccf2 4046 }
4047 out << " }" << endl;
c57f56b7 4048 out << " if (!chain->GetNtrees()) {" << endl;
4049 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
4050 out << " return NULL;" << endl;
4051 out << " }" << endl;
0df6ccf2 4052 if(fFriendChainName!="") {
4053 out << " chain->AddFriend(chainFriend);" << endl;
4054 }
c57f56b7 4055 out << " return chain;" << endl;
fcc9bb6f 4056 out << "}" << endl << endl;
c57f56b7 4057 }
5b273635 4058 if (hasANALYSISalice) {
4059 out <<"//________________________________________________________________________________" << endl;
4060 out << "Bool_t SetupPar(const char *package) {" << endl;
4061 out << "// Compile the package and set it up." << endl;
4062 out << " TString pkgdir = package;" << endl;
4063 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4064 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4065 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4066 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4067 out << " // Check for BUILD.sh and execute" << endl;
4068 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4069 out << " printf(\"*******************************\\n\");" << endl;
4070 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4071 out << " printf(\"*******************************\\n\");" << endl;
4072 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4073 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4074 out << " gSystem->ChangeDirectory(cdir);" << endl;
4075 out << " return kFALSE;" << endl;
4076 out << " }" << endl;
4077 out << " } else {" << endl;
4078 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4079 out << " gSystem->ChangeDirectory(cdir);" << endl;
4080 out << " return kFALSE;" << endl;
4081 out << " }" << endl;
4082 out << " // Check for SETUP.C and execute" << endl;
4083 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4084 out << " printf(\"*******************************\\n\");" << endl;
4085 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4086 out << " printf(\"*******************************\\n\");" << endl;
4087 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4088 out << " } else {" << endl;
4089 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4090 out << " gSystem->ChangeDirectory(cdir);" << endl;
4091 out << " return kFALSE;" << endl;
4092 out << " }" << endl;
4093 out << " // Restore original workdir" << endl;
4094 out << " gSystem->ChangeDirectory(cdir);" << endl;
4095 out << " return kTRUE;" << endl;
4096 out << "}" << endl;
4097 }
c57f56b7 4098 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4099 }
4100 Bool_t copy = kTRUE;
d3b18c4c 4101 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4102 if (copy) {
4103 CdWork();
4104 TString workdir = gGrid->GetHomeDirectory();
4105 workdir += fGridWorkingDir;
4106 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4107 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4108// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4109 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4110 Form("alien://%s/%s", workdir.Data(),
4111 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4112 }
4113}
4114
0f389141 4115//______________________________________________________________________________
4116void AliAnalysisAlien::WriteMergingMacro()
4117{
4118// Write a macro to merge the outputs per master job.
4119 if (!fMergeViaJDL) return;
4120 if (!fOutputFiles.Length()) {
4121 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4122 return;
4123 }
ec5acdce 4124 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
0f389141 4125 TString mergingMacro = fExecutable;
4126 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4127 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4128 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4129 ofstream out;
4130 out.open(mergingMacro.Data(), ios::out);
4131 if (!out.good()) {
4132 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4133 return;
4134 }
5b273635 4135 Bool_t hasSTEERBase = kFALSE;
4136 Bool_t hasESD = kFALSE;
4137 Bool_t hasAOD = kFALSE;
4138 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4139 Bool_t hasOADB = kFALSE;
5b273635 4140 Bool_t hasANALYSISalice = kFALSE;
4141 Bool_t hasCORRFW = kFALSE;
0f389141 4142 TString func = mergingMacro;
4143 TString comment;
4144 func.ReplaceAll(".C", "");
f47d5cb4 4145 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4146 out << "{" << endl;
4147 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4148 out << " TStopwatch timer;" << endl;
4149 out << " timer.Start();" << endl << endl;
b3e07543 4150 // Reset existing include path
4151 out << "// Reset existing include path and add current directory first in the search" << endl;
4152 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4153 if (!fExecutableCommand.Contains("aliroot")) {
4154 out << "// load base root libraries" << endl;
4155 out << " gSystem->Load(\"libTree\");" << endl;
4156 out << " gSystem->Load(\"libGeom\");" << endl;
4157 out << " gSystem->Load(\"libVMC\");" << endl;
4158 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4159 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4160 }
0f389141 4161 if (fAdditionalRootLibs.Length()) {
4162 // in principle libtree /lib geom libvmc etc. can go into this list, too
4163 out << "// Add aditional libraries" << endl;
4164 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4165 TIter next(list);
4166 TObjString *str;
4167 while((str=(TObjString*)next())) {
4168 if (str->GetString().Contains(".so"))
4169 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4170 }
4171 if (list) delete list;
4172 }
0f389141 4173 out << "// Load analysis framework libraries" << endl;
4174 if (!fPackages) {
7c2cd90a 4175 if (!fExecutableCommand.Contains("aliroot")) {
4176 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4177 out << " gSystem->Load(\"libESD\");" << endl;
4178 out << " gSystem->Load(\"libAOD\");" << endl;
4179 }
0f389141 4180 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4181 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4182 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4183 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4184 } else {
4185 TIter next(fPackages);
4186 TObject *obj;
4187 TString pkgname;
5b273635 4188 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4189 while ((obj=next())) {
4190 pkgname = obj->GetName();
4191 if (pkgname == "STEERBase" ||
4192 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4193 if (pkgname == "ESD" ||
4194 pkgname == "ESD.par") hasESD = kTRUE;
4195 if (pkgname == "AOD" ||
4196 pkgname == "AOD.par") hasAOD = kTRUE;
4197 if (pkgname == "ANALYSIS" ||
4198 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4199 if (pkgname == "OADB" ||
4200 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4201 if (pkgname == "ANALYSISalice" ||
4202 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4203 if (pkgname == "CORRFW" ||
4204 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4205 }
5b273635 4206 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4207 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4208 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4209 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4210 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4211 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4212 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4213 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4214 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4215 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4216 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4217 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4218 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4219 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4220 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4221 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4222 out << "// Compile other par packages" << endl;
4223 next.Reset();
4224 while ((obj=next())) {
4225 pkgname = obj->GetName();
4226 if (pkgname == "STEERBase" ||
4227 pkgname == "STEERBase.par" ||
4228 pkgname == "ESD" ||
4229 pkgname == "ESD.par" ||
4230 pkgname == "AOD" ||
4231 pkgname == "AOD.par" ||
4232 pkgname == "ANALYSIS" ||
4233 pkgname == "ANALYSIS.par" ||
7ea40ac8 4234 pkgname == "OADB" ||
4235 pkgname == "OADB.par" ||
0f389141 4236 pkgname == "ANALYSISalice" ||
4237 pkgname == "ANALYSISalice.par" ||
4238 pkgname == "CORRFW" ||
4239 pkgname == "CORRFW.par") continue;
5b273635 4240 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4241 }
4242 }
b3e07543 4243 out << "// include path" << endl;
4244 // Get the include path from the interpreter and remove entries pointing to AliRoot
4245 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4246 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4247 out << " TIter nextpath(listpaths);" << endl;
4248 out << " TObjString *pname;" << endl;
4249 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4250 out << " TString current = pname->GetName();" << endl;
4251 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4252 out << " gSystem->AddIncludePath(current);" << endl;
4253 out << " }" << endl;
4254 out << " if (listpaths) delete listpaths;" << endl;
4255 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4256 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4257 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4258 if (fAdditionalLibs.Length()) {
4259 out << "// Add aditional AliRoot libraries" << endl;
4260 TObjArray *list = fAdditionalLibs.Tokenize(" ");
4261 TIter next(list);
4262 TObjString *str;
4263 while((str=(TObjString*)next())) {
4264 if (str->GetString().Contains(".so"))
4265 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4266 }
4267 if (list) delete list;
4268 }
4269 out << endl;
4270 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4271 if (fAnalysisSource.Length()) {
4272 TObjArray *list = fAnalysisSource.Tokenize(" ");
4273 TIter next(list);
4274 TObjString *str;
4275 while((str=(TObjString*)next())) {
4276 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4277 }
4278 if (list) delete list;
4279 }
149d288c 4280 out << endl;
4281
0f389141 4282 if (fFastReadOption) {
4283 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4284 out << "// fast xrootd reading enabled" << endl;
4285 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4286 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4287 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4288 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4289 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4290 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4291 }
4292 // Change temp directory to current one
fdee57f1 4293 out << "// Connect to AliEn" << endl;
4294 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
e8b839ab 4295 out << "// Set temporary merging directory to current one" << endl;
4296 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4297 out << "// Set temporary compilation directory to current one" << endl;
4298 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
7c2cd90a 4299 out << " TString outputDir = dir;" << endl;
f790bc1b 4300 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4301 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4302 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4303 out << " TIter *iter = new TIter(list);" << endl;
4304 out << " TObjString *str;" << endl;
a2f5fc01 4305 out << " TString outputFile;" << endl;
0f389141 4306 out << " Bool_t merged = kTRUE;" << endl;
ec5acdce 4307 TString analysisFile = fExecutable;
4308 analysisFile.ReplaceAll(".sh", ".root");
4309 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4310 << analysisFile << "\");" << endl;
4311 out << " if (!mgr) {" << endl;
4312 out << " printf(\"ERROR: Analysis manager could not be extracted from file \");" << endl;
4313 out << " return;" << endl;
4314 out << " }" << endl;
0f389141 4315 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4316 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4317 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4318 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4319 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4320 out << " // Skip already merged outputs" << endl;
a2f5fc01 4321 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
4322 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 4323 out << " continue;" << endl;
4324 out << " }" << endl;
a2f5fc01 4325 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4326 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4327 out << " if (!merged) {" << endl;
a2f5fc01 4328 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4329 out << " return;" << endl;
0f389141 4330 out << " }" << endl;
4331 out << " }" << endl;
ec5acdce 4332 if (mgr && mgr->IsCollectThroughput()) {
4333 out << " TString infolog = \"" << mgr->GetFileInfoLog() << "\";" << endl;
4334 out << " AliAnalysisAlien::MergeInfo(infolog, outputDir);" << endl;
4335 }
7c2cd90a 4336 out << " // all outputs merged, validate" << endl;
4337 out << " ofstream out;" << endl;
4338 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4339 out << " out.close();" << endl;
4340 out << " // read the analysis manager from file" << endl;
f47d5cb4 4341 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
5c4250fc 4342 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4343 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4344 out << " mgr->PrintStatus();" << endl;
ec5acdce 4345 if (mgr) {
4346 if (mgr->GetDebugLevel()>3) {
f866cba5 4347 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4348 } else {
9a8288ff 4349 if (TestBit(AliAnalysisGrid::kTest))
4350 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4351 else
4352 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4353 }
4354 }
9a8288ff 4355 out << " TTree *tree = NULL;" << endl;
4356 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4357 out << "}" << endl << endl;
5b273635 4358 if (hasANALYSISalice) {
4359 out <<"//________________________________________________________________________________" << endl;
4360 out << "Bool_t SetupPar(const char *package) {" << endl;
4361 out << "// Compile the package and set it up." << endl;
4362 out << " TString pkgdir = package;" << endl;
4363 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4364 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4365 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4366 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4367 out << " // Check for BUILD.sh and execute" << endl;
4368 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4369 out << " printf(\"*******************************\\n\");" << endl;
4370 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4371 out << " printf(\"*******************************\\n\");" << endl;
4372 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4373 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4374 out << " gSystem->ChangeDirectory(cdir);" << endl;
4375 out << " return kFALSE;" << endl;
4376 out << " }" << endl;
4377 out << " } else {" << endl;
4378 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4379 out << " gSystem->ChangeDirectory(cdir);" << endl;
4380 out << " return kFALSE;" << endl;
4381 out << " }" << endl;
4382 out << " // Check for SETUP.C and execute" << endl;
4383 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4384 out << " printf(\"*******************************\\n\");" << endl;
4385 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4386 out << " printf(\"*******************************\\n\");" << endl;
4387 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4388 out << " } else {" << endl;
4389 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4390 out << " gSystem->ChangeDirectory(cdir);" << endl;
4391 out << " return kFALSE;" << endl;
4392 out << " }" << endl;
4393 out << " // Restore original workdir" << endl;
4394 out << " gSystem->ChangeDirectory(cdir);" << endl;
4395 out << " return kTRUE;" << endl;
4396 out << "}" << endl;
4397 }
0f389141 4398 }
4399 Bool_t copy = kTRUE;
d3b18c4c 4400 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4401 if (copy) {
4402 CdWork();
4403 TString workdir = gGrid->GetHomeDirectory();
4404 workdir += fGridWorkingDir;
4405 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4406 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4407// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4408 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4409 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4410 }
4411}
4412
4413//______________________________________________________________________________
4414Bool_t AliAnalysisAlien::SetupPar(const char *package)
4415{
205b201f 4416// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4417// Note that for loading the compiled library. The current directory should have precedence in
4418// LD_LIBRARY_PATH
4419 TString pkgdir = package;
4420 pkgdir.ReplaceAll(".par","");
b50a41c3 4421 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4422 TString cdir = gSystem->WorkingDirectory();
4423 gSystem->ChangeDirectory(pkgdir);
4424 // Check for BUILD.sh and execute
4425 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4426 printf("**************************************************\n");
4427 printf("*** Building PAR archive %s\n", package);
4428 printf("**************************************************\n");
4429 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4430 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4431 gSystem->ChangeDirectory(cdir);
4432 return kFALSE;
4433 }
4434 } else {
4435 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4436 gSystem->ChangeDirectory(cdir);
4437 return kFALSE;
4438 }
4439 // Check for SETUP.C and execute
4440 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4441 printf("**************************************************\n");
4442 printf("*** Setup PAR archive %s\n", package);
4443 printf("**************************************************\n");
4444 gROOT->Macro("PROOF-INF/SETUP.C");
4445 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4446 } else {
4447 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4448 gSystem->ChangeDirectory(cdir);
4449 return kFALSE;
4450 }
4451 // Restore original workdir
4452 gSystem->ChangeDirectory(cdir);
4453 return kTRUE;
4454}
4455
c57f56b7 4456//______________________________________________________________________________
4457void AliAnalysisAlien::WriteExecutable()
4458{
4459// Generate the alien executable script.
4460 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4461 ofstream out;
4462 out.open(fExecutable.Data(), ios::out);
4463 if (out.bad()) {
5513444a 4464 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4465 return;
4466 }
4467 out << "#!/bin/bash" << endl;
2a4d5166 4468 // Make sure we can properly compile par files
4469 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4470 out << "echo \"=========================================\"" << endl;
4471 out << "echo \"############## PATH : ##############\"" << endl;
4472 out << "echo $PATH" << endl;
4473 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4474 out << "echo $LD_LIBRARY_PATH" << endl;
4475 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4476 out << "echo $ROOTSYS" << endl;
4477 out << "echo \"############## which root : ##############\"" << endl;
4478 out << "which root" << endl;
4479 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4480 out << "echo $ALICE_ROOT" << endl;
4481 out << "echo \"############## which aliroot : ##############\"" << endl;
4482 out << "which aliroot" << endl;
9c5ddadc 4483 out << "echo \"############## system limits : ##############\"" << endl;
4484 out << "ulimit -a" << endl;
4485 out << "echo \"############## memory : ##############\"" << endl;
4486 out << "free -m" << endl;
c57f56b7 4487 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4488 out << fExecutableCommand << " ";
370c9880 4489 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4490 out << "RET=$?" << endl;
4491 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4492 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4493 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4494 out << " let sig=\"$RET - 128\""<<endl;
4495 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4496 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4497 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4498 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4499 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4500 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4501 out << " fi"<<endl;
4502 out << " exit $RET"<< endl;
4503 out << "fi" << endl << endl ;
4504 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4505 out << "echo \"############## memory after: ##############\"" << endl;
4506 out << "free -m" << endl;
c57f56b7 4507 }
4508 Bool_t copy = kTRUE;
d3b18c4c 4509 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4510 if (copy) {
4511 CdWork();
4512 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4513 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4514 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4515 workdir += fGridWorkingDir;
4516 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4517 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4518 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4519// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4520 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4521 executable.Data())) Fatal("","Terminating");
c57f56b7 4522 }
4523}
4524
0f389141 4525//______________________________________________________________________________
4526void AliAnalysisAlien::WriteMergeExecutable()
4527{
4528// Generate the alien executable script for the merging job.
4529 if (!fMergeViaJDL) return;
4530 TString mergeExec = fExecutable;
4531 mergeExec.ReplaceAll(".sh", "_merge.sh");
4532 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4533 ofstream out;
4534 out.open(mergeExec.Data(), ios::out);
4535 if (out.bad()) {
4536 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4537 return;
4538 }
4539 out << "#!/bin/bash" << endl;
2a4d5166 4540 // Make sure we can properly compile par files
4541 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4542 out << "echo \"=========================================\"" << endl;
4543 out << "echo \"############## PATH : ##############\"" << endl;
4544 out << "echo $PATH" << endl;
4545 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4546 out << "echo $LD_LIBRARY_PATH" << endl;
4547 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4548 out << "echo $ROOTSYS" << endl;
4549 out << "echo \"############## which root : ##############\"" << endl;
4550 out << "which root" << endl;
4551 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4552 out << "echo $ALICE_ROOT" << endl;
4553 out << "echo \"############## which aliroot : ##############\"" << endl;
4554 out << "which aliroot" << endl;
4555 out << "echo \"############## system limits : ##############\"" << endl;
4556 out << "ulimit -a" << endl;
4557 out << "echo \"############## memory : ##############\"" << endl;
4558 out << "free -m" << endl;
4559 out << "echo \"=========================================\"" << endl << endl;
0f389141 4560 TString mergeMacro = fExecutable;
4561 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4562 if (IsOneStageMerging())
4563 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4564 else
f47d5cb4 4565 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4566 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4567 out << "RET=$?" << endl;
4568 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4569 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4570 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4571 out << " let sig=\"$RET - 128\""<<endl;
4572 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4573 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4574 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4575 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4576 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4577 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4578 out << " fi"<<endl;
4579 out << " exit $RET"<< endl;
4580 out << "fi" << endl << endl ;
0f389141 4581 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4582 out << "echo \"############## memory after: ##############\"" << endl;
4583 out << "free -m" << endl;
0f389141 4584 }
4585 Bool_t copy = kTRUE;
d3b18c4c 4586 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4587 if (copy) {
4588 CdWork();
4589 TString workdir = gGrid->GetHomeDirectory();
4590 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4591 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4592 workdir += fGridWorkingDir;
4593 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4594 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4595 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4596// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4597 if (!copyLocal2Alien("WriteMergeExecutable",
4598 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4599 }
4600}
4601
c57f56b7 4602//______________________________________________________________________________
5513444a 4603void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4604{
4605// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4606// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4607// Next lines: full_path_to_dataset XXX (XXX is a string)
4608// To submit, one has to: submit jdl XXX for all lines
4609 ofstream out;
4610 out.open(filename, ios::out);
4611 if (out.bad()) {
4612 Error("WriteProductionFile", "Bad file name: %s", filename);
4613 return;
4614 }
d3b18c4c 4615 TString workdir;
4616 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4617 workdir = gGrid->GetHomeDirectory();
5513444a 4618 workdir += fGridWorkingDir;
f5e8c702 4619 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4620 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4621 out << locjdl << " " << njobspermaster << endl;
5513444a 4622 Int_t nmasterjobs = fInputFiles->GetEntries();
4623 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4624 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4625 runOutDir.ReplaceAll(".xml", "");
4626 if (fOutputToRunNo)
4627 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4628 else
4629 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4630 }
d3b18c4c 4631 if (gGrid) {
4632 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4633 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4634// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4635 if (!copyLocal2Alien("WriteProductionFile", filename,
4636 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4637 }
5513444a 4638}
4639
4640//______________________________________________________________________________
0f389141 4641void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4642{
4643// Generate the alien validation script.
4644 // Generate the validation script
4645 TObjString *os;
d3b18c4c 4646 if (fValidationScript.IsNull()) {
4647 fValidationScript = fExecutable;
4648 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4649 }
4650 TString validationScript = fValidationScript;
4651 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4652 if (!Connect()) {
4653 Error("WriteValidationScript", "Alien connection required");
4654 return;
4655 }
d3b18c4c 4656 if (!fTerminateFiles.IsNull()) {
4657 fTerminateFiles.Strip();
4658 fTerminateFiles.ReplaceAll(" ",",");
4659 }
a2f5fc01 4660 TString outStream = "";
4661 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4662 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4663 ofstream out;
0d5d317c 4664 out.open(validationScript, ios::out);
c57f56b7 4665 out << "#!/bin/bash" << endl;
4666 out << "##################################################" << endl;
4667 out << "validateout=`dirname $0`" << endl;
4668 out << "validatetime=`date`" << endl;
4669 out << "validated=\"0\";" << endl;
4670 out << "error=0" << endl;
4671 out << "if [ -z $validateout ]" << endl;
4672 out << "then" << endl;
4673 out << " validateout=\".\"" << endl;
4674 out << "fi" << endl << endl;
4675 out << "cd $validateout;" << endl;
4676 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4677 out << "echo \"*******************************************************\"" << outStream << endl;
4678 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4679 out << "" << endl;
a2f5fc01 4680 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4681 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4682 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4683 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4684 out << "ls -la ./" << outStream << endl;
4685 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4686 out << "##################################################" << endl;
ebec370a 4687 out << "" << endl;
4688
4689 out << "if [ ! -f stderr ] ; then" << endl;
4690 out << " error=1" << endl;
a2f5fc01 4691 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4692 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4693 out << "fi" << endl;
4694
b34c9f51 4695 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4696 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4697 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4698 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4699 out << "" << endl;
4700
ebec370a 4701 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4702 out << " error=1" << endl;
a2f5fc01 4703 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4704 out << " echo \"$parArch\" " << outStream << endl;
4705 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4706 out << "fi" << endl;
4707
4708 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4709 out << " error=1" << endl;
a2f5fc01 4710 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4711 out << " echo \"$segViol\" " << outStream << endl;
4712 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4713 out << "fi" << endl;
4714
4715 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4716 out << " error=1" << endl;
a2f5fc01 4717 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4718 out << " echo \"$segFault\" " << outStream << endl;
4719 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4720 out << "fi" << endl;
4721
b34c9f51 4722 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4723 out << " error=1" << endl;
a2f5fc01 4724 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4725 out << " echo \"$glibcErr\" " << outStream << endl;
4726 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4727 out << "fi" << endl;
4728
ebec370a 4729 // Part dedicated to the specific analyses running into the train
4730
d3b18c4c 4731 TString outputFiles = fOutputFiles;
4732 if (merge && !fTerminateFiles.IsNull()) {
4733 outputFiles += ",";
4734 outputFiles += fTerminateFiles;
4735 }
4736 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4737 TIter next1(arr);
a2f5fc01 4738 TString outputFile;
bbd88db8 4739 while (!merge && (os=(TObjString*)next1())) {
4740 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4741 outputFile = os->GetString();
4742 Int_t index = outputFile.Index("@");
4743 if (index > 0) outputFile.Remove(index);
bbd88db8 4744 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4745 if (outputFile.Contains("*")) continue;
4746 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4747 out << " error=1" << endl;
7c2cd90a 4748 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4749 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4750 out << "fi" << endl;
4751 }
4752 delete arr;
7c2cd90a 4753 out << "if ! [ -f outputs_valid ] ; then" << endl;
4754 out << " error=1" << endl;
4755 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4756 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4757 out << "fi" << endl;
923e2ca5 4758
c57f56b7 4759 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4760 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4761 if (!IsKeepLogs()) {
4762 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4763 outStream = "";
149d288c 4764 out << " rm -f std*" << endl;
4765 }
c57f56b7 4766 out << "fi" << endl;
4767
a2f5fc01 4768 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4769 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4770 out << "cd -" << endl;
4771 out << "exit $error" << endl;
4772 }
4773 Bool_t copy = kTRUE;
d3b18c4c 4774 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4775 if (copy) {
4776 CdWork();
4777 TString workdir = gGrid->GetHomeDirectory();
4778 workdir += fGridWorkingDir;
d3b18c4c 4779 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4780 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 4781// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4782 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
4783 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 4784 }
4785}