]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Reverted "new AliPhysicsSelectionTask()" call to "new AliPhysicsSelectionTask("")"
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
c82bb898 51using std::ofstream;
52using std::ifstream;
53using std::ios;
54using std::endl;
c57f56b7 55ClassImp(AliAnalysisAlien)
6c395669 56#if 0
57;
58#endif
c57f56b7 59
6c395669 60namespace {
61 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
62 {
63 TString sl(Form("file:%s", loc));
64 TString sr(Form("alien://%s", rem));
65 Bool_t ret = TFile::Cp(sl, sr);
66 if (!ret) {
67 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
68 }
69 return ret;
70 }
71}
72
c57f56b7 73//______________________________________________________________________________
74AliAnalysisAlien::AliAnalysisAlien()
75 :AliAnalysisGrid(),
76 fGridJDL(NULL),
0f389141 77 fMergingJDL(NULL),
c57f56b7 78 fPrice(0),
79 fTTL(0),
80 fSplitMaxInputFileNumber(0),
81 fMaxInitFailed(0),
82 fMasterResubmitThreshold(0),
bb885a9e 83 fNtestFiles(0),
319593fb 84 fNrunsPerMaster(0),
16a4353c 85 fMaxMergeFiles(0),
b3e07543 86 fMaxMergeStages(0),
d2a409b2 87 fNsubmitted(0),
a3e84053 88 fProductionMode(0),
cd11251e 89 fOutputToRunNo(0),
0f389141 90 fMergeViaJDL(0),
91 fFastReadOption(0),
e1c22e21 92 fOverwriteMode(1),
149d288c 93 fNreplicas(2),
3bdcb562 94 fNproofWorkers(0),
27734f0e 95 fNproofWorkersPerSlave(0),
3bdcb562 96 fProofReset(0),
c57f56b7 97 fRunNumbers(),
98 fExecutable(),
0a1c1f7f 99 fExecutableCommand(),
c57f56b7 100 fArguments(),
631c0b05 101 fExecutableArgs(),
c57f56b7 102 fAnalysisMacro(),
103 fAnalysisSource(),
d3b18c4c 104 fValidationScript(),
d5c6455a 105 fAdditionalRootLibs(),
c57f56b7 106 fAdditionalLibs(),
107 fSplitMode(),
108 fAPIVersion(),
109 fROOTVersion(),
110 fAliROOTVersion(),
648174cf 111 fExternalPackages(),
c57f56b7 112 fUser(),
113 fGridWorkingDir(),
114 fGridDataDir(),
115 fDataPattern(),
116 fGridOutputDir(),
117 fOutputArchive(),
118 fOutputFiles(),
119 fInputFormat(),
e7c71df0 120 fDatasetName(),
c57f56b7 121 fJDLName(),
d3b18c4c 122 fTerminateFiles(),
bb885a9e 123 fMergeExcludes(),
37d21c01 124 fRegisterExcludes(),
f965131e 125 fIncludePath(),
bb885a9e 126 fCloseSE(),
0df6ccf2 127 fFriendChainName(),
c6cb3634 128 fJobTag(),
648174cf 129 fOutputSingle(),
5fce53f4 130 fRunPrefix(),
3bdcb562 131 fProofCluster(),
132 fProofDataSet(),
133 fFileForTestMode(),
134 fRootVersionForProof(),
135 fAliRootMode(),
f47d5cb4 136 fMergeDirName(),
4e5c5506 137 fInputFiles(0),
28ce6c9c 138 fPackages(0),
77f34eae 139 fModules(0),
28ce6c9c 140 fProofParam()
c57f56b7 141{
142// Dummy ctor.
143 SetDefaults();
144}
145
146//______________________________________________________________________________
147AliAnalysisAlien::AliAnalysisAlien(const char *name)
148 :AliAnalysisGrid(name),
149 fGridJDL(NULL),
0f389141 150 fMergingJDL(NULL),
c57f56b7 151 fPrice(0),
152 fTTL(0),
153 fSplitMaxInputFileNumber(0),
154 fMaxInitFailed(0),
155 fMasterResubmitThreshold(0),
bb885a9e 156 fNtestFiles(0),
319593fb 157 fNrunsPerMaster(0),
16a4353c 158 fMaxMergeFiles(0),
b3e07543 159 fMaxMergeStages(0),
d2a409b2 160 fNsubmitted(0),
a3e84053 161 fProductionMode(0),
cd11251e 162 fOutputToRunNo(0),
0f389141 163 fMergeViaJDL(0),
164 fFastReadOption(0),
e1c22e21 165 fOverwriteMode(1),
149d288c 166 fNreplicas(2),
3bdcb562 167 fNproofWorkers(0),
27734f0e 168 fNproofWorkersPerSlave(0),
3bdcb562 169 fProofReset(0),
c57f56b7 170 fRunNumbers(),
171 fExecutable(),
0a1c1f7f 172 fExecutableCommand(),
c57f56b7 173 fArguments(),
631c0b05 174 fExecutableArgs(),
c57f56b7 175 fAnalysisMacro(),
176 fAnalysisSource(),
d3b18c4c 177 fValidationScript(),
d5c6455a 178 fAdditionalRootLibs(),
c57f56b7 179 fAdditionalLibs(),
180 fSplitMode(),
181 fAPIVersion(),
182 fROOTVersion(),
183 fAliROOTVersion(),
648174cf 184 fExternalPackages(),
c57f56b7 185 fUser(),
186 fGridWorkingDir(),
187 fGridDataDir(),
188 fDataPattern(),
189 fGridOutputDir(),
190 fOutputArchive(),
191 fOutputFiles(),
192 fInputFormat(),
e7c71df0 193 fDatasetName(),
c57f56b7 194 fJDLName(),
d3b18c4c 195 fTerminateFiles(),
bb885a9e 196 fMergeExcludes(),
37d21c01 197 fRegisterExcludes(),
f965131e 198 fIncludePath(),
bb885a9e 199 fCloseSE(),
0df6ccf2 200 fFriendChainName(),
c6cb3634 201 fJobTag(),
648174cf 202 fOutputSingle(),
5fce53f4 203 fRunPrefix(),
3bdcb562 204 fProofCluster(),
205 fProofDataSet(),
206 fFileForTestMode(),
207 fRootVersionForProof(),
208 fAliRootMode(),
f47d5cb4 209 fMergeDirName(),
4e5c5506 210 fInputFiles(0),
28ce6c9c 211 fPackages(0),
77f34eae 212 fModules(0),
28ce6c9c 213 fProofParam()
c57f56b7 214{
215// Default ctor.
216 SetDefaults();
217}
218
219//______________________________________________________________________________
220AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
221 :AliAnalysisGrid(other),
222 fGridJDL(NULL),
0f389141 223 fMergingJDL(NULL),
c57f56b7 224 fPrice(other.fPrice),
225 fTTL(other.fTTL),
226 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
227 fMaxInitFailed(other.fMaxInitFailed),
228 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 229 fNtestFiles(other.fNtestFiles),
319593fb 230 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 231 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 232 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 233 fNsubmitted(other.fNsubmitted),
a3e84053 234 fProductionMode(other.fProductionMode),
cd11251e 235 fOutputToRunNo(other.fOutputToRunNo),
0f389141 236 fMergeViaJDL(other.fMergeViaJDL),
237 fFastReadOption(other.fFastReadOption),
238 fOverwriteMode(other.fOverwriteMode),
149d288c 239 fNreplicas(other.fNreplicas),
3bdcb562 240 fNproofWorkers(other.fNproofWorkers),
27734f0e 241 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 242 fProofReset(other.fProofReset),
c57f56b7 243 fRunNumbers(other.fRunNumbers),
244 fExecutable(other.fExecutable),
0a1c1f7f 245 fExecutableCommand(other.fExecutableCommand),
c57f56b7 246 fArguments(other.fArguments),
631c0b05 247 fExecutableArgs(other.fExecutableArgs),
c57f56b7 248 fAnalysisMacro(other.fAnalysisMacro),
249 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 250 fValidationScript(other.fValidationScript),
d5c6455a 251 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 252 fAdditionalLibs(other.fAdditionalLibs),
253 fSplitMode(other.fSplitMode),
254 fAPIVersion(other.fAPIVersion),
255 fROOTVersion(other.fROOTVersion),
256 fAliROOTVersion(other.fAliROOTVersion),
648174cf 257 fExternalPackages(other.fExternalPackages),
c57f56b7 258 fUser(other.fUser),
259 fGridWorkingDir(other.fGridWorkingDir),
260 fGridDataDir(other.fGridDataDir),
261 fDataPattern(other.fDataPattern),
262 fGridOutputDir(other.fGridOutputDir),
263 fOutputArchive(other.fOutputArchive),
264 fOutputFiles(other.fOutputFiles),
265 fInputFormat(other.fInputFormat),
e7c71df0 266 fDatasetName(other.fDatasetName),
c57f56b7 267 fJDLName(other.fJDLName),
d3b18c4c 268 fTerminateFiles(other.fTerminateFiles),
bb885a9e 269 fMergeExcludes(other.fMergeExcludes),
37d21c01 270 fRegisterExcludes(other.fRegisterExcludes),
f965131e 271 fIncludePath(other.fIncludePath),
bb885a9e 272 fCloseSE(other.fCloseSE),
0df6ccf2 273 fFriendChainName(other.fFriendChainName),
c6cb3634 274 fJobTag(other.fJobTag),
648174cf 275 fOutputSingle(other.fOutputSingle),
5fce53f4 276 fRunPrefix(other.fRunPrefix),
3bdcb562 277 fProofCluster(other.fProofCluster),
278 fProofDataSet(other.fProofDataSet),
279 fFileForTestMode(other.fFileForTestMode),
280 fRootVersionForProof(other.fRootVersionForProof),
281 fAliRootMode(other.fAliRootMode),
f47d5cb4 282 fMergeDirName(other.fMergeDirName),
4e5c5506 283 fInputFiles(0),
28ce6c9c 284 fPackages(0),
77f34eae 285 fModules(0),
28ce6c9c 286 fProofParam()
c57f56b7 287{
288// Copy ctor.
289 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 290 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 291 fRunRange[0] = other.fRunRange[0];
292 fRunRange[1] = other.fRunRange[1];
c57f56b7 293 if (other.fInputFiles) {
294 fInputFiles = new TObjArray();
295 TIter next(other.fInputFiles);
296 TObject *obj;
297 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
298 fInputFiles->SetOwner();
299 }
4e5c5506 300 if (other.fPackages) {
301 fPackages = new TObjArray();
302 TIter next(other.fPackages);
303 TObject *obj;
304 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
305 fPackages->SetOwner();
306 }
77f34eae 307 if (other.fModules) {
308 fModules = new TObjArray();
309 fModules->SetOwner();
310 TIter next(other.fModules);
311 AliAnalysisTaskCfg *mod, *crt;
312 while ((crt=(AliAnalysisTaskCfg*)next())) {
313 mod = new AliAnalysisTaskCfg(*crt);
314 fModules->Add(mod);
315 }
316 }
c57f56b7 317}
318
319//______________________________________________________________________________
320AliAnalysisAlien::~AliAnalysisAlien()
321{
322// Destructor.
77f34eae 323 delete fGridJDL;
324 delete fMergingJDL;
325 delete fInputFiles;
326 delete fPackages;
327 delete fModules;
28ce6c9c 328 fProofParam.DeleteAll();
c57f56b7 329}
330
331//______________________________________________________________________________
332AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
333{
334// Assignment.
335 if (this != &other) {
336 AliAnalysisGrid::operator=(other);
337 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 338 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 339 fPrice = other.fPrice;
340 fTTL = other.fTTL;
341 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
342 fMaxInitFailed = other.fMaxInitFailed;
343 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 344 fNtestFiles = other.fNtestFiles;
a3e84053 345 fNrunsPerMaster = other.fNrunsPerMaster;
346 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 347 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 348 fNsubmitted = other.fNsubmitted;
349 fProductionMode = other.fProductionMode;
cd11251e 350 fOutputToRunNo = other.fOutputToRunNo;
0f389141 351 fMergeViaJDL = other.fMergeViaJDL;
352 fFastReadOption = other.fFastReadOption;
353 fOverwriteMode = other.fOverwriteMode;
149d288c 354 fNreplicas = other.fNreplicas;
3bdcb562 355 fNproofWorkers = other.fNproofWorkers;
27734f0e 356 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 357 fProofReset = other.fProofReset;
c57f56b7 358 fRunNumbers = other.fRunNumbers;
359 fExecutable = other.fExecutable;
0a1c1f7f 360 fExecutableCommand = other.fExecutableCommand;
c57f56b7 361 fArguments = other.fArguments;
631c0b05 362 fExecutableArgs = other.fExecutableArgs;
c57f56b7 363 fAnalysisMacro = other.fAnalysisMacro;
364 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 365 fValidationScript = other.fValidationScript;
d5c6455a 366 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 367 fAdditionalLibs = other.fAdditionalLibs;
368 fSplitMode = other.fSplitMode;
369 fAPIVersion = other.fAPIVersion;
370 fROOTVersion = other.fROOTVersion;
371 fAliROOTVersion = other.fAliROOTVersion;
648174cf 372 fExternalPackages = other.fExternalPackages;
c57f56b7 373 fUser = other.fUser;
374 fGridWorkingDir = other.fGridWorkingDir;
375 fGridDataDir = other.fGridDataDir;
376 fDataPattern = other.fDataPattern;
377 fGridOutputDir = other.fGridOutputDir;
378 fOutputArchive = other.fOutputArchive;
379 fOutputFiles = other.fOutputFiles;
380 fInputFormat = other.fInputFormat;
e7c71df0 381 fDatasetName = other.fDatasetName;
c57f56b7 382 fJDLName = other.fJDLName;
d3b18c4c 383 fTerminateFiles = other.fTerminateFiles;
bb885a9e 384 fMergeExcludes = other.fMergeExcludes;
37d21c01 385 fRegisterExcludes = other.fRegisterExcludes;
f965131e 386 fIncludePath = other.fIncludePath;
bb885a9e 387 fCloseSE = other.fCloseSE;
0df6ccf2 388 fFriendChainName = other.fFriendChainName;
c6cb3634 389 fJobTag = other.fJobTag;
648174cf 390 fOutputSingle = other.fOutputSingle;
5fce53f4 391 fRunPrefix = other.fRunPrefix;
3bdcb562 392 fProofCluster = other.fProofCluster;
393 fProofDataSet = other.fProofDataSet;
394 fFileForTestMode = other.fFileForTestMode;
395 fRootVersionForProof = other.fRootVersionForProof;
396 fAliRootMode = other.fAliRootMode;
f47d5cb4 397 fMergeDirName = other.fMergeDirName;
c57f56b7 398 if (other.fInputFiles) {
399 fInputFiles = new TObjArray();
400 TIter next(other.fInputFiles);
401 TObject *obj;
402 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
403 fInputFiles->SetOwner();
404 }
4e5c5506 405 if (other.fPackages) {
406 fPackages = new TObjArray();
407 TIter next(other.fPackages);
408 TObject *obj;
409 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
410 fPackages->SetOwner();
411 }
77f34eae 412 if (other.fModules) {
413 fModules = new TObjArray();
414 fModules->SetOwner();
415 TIter next(other.fModules);
416 AliAnalysisTaskCfg *mod, *crt;
417 while ((crt=(AliAnalysisTaskCfg*)next())) {
418 mod = new AliAnalysisTaskCfg(*crt);
419 fModules->Add(mod);
420 }
421 }
c57f56b7 422 }
423 return *this;
424}
425
37d21c01 426//______________________________________________________________________________
427void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
428{
429// Add a single additional library to be loaded. Extension must be present.
430 TString lib(name);
431 if (!lib.Contains(".")) {
432 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
433 return;
434 }
435 if (fAdditionalLibs.Contains(name)) {
436 Warning("AddAdditionalLibrary", "Library %s already added.", name);
437 return;
438 }
439 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
440 fAdditionalLibs += lib;
441}
442
77f34eae 443//______________________________________________________________________________
444void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
445{
446// Adding a module. Checks if already existing. Becomes owned by this.
447 if (!module) return;
448 if (GetModule(module->GetName())) {
449 Error("AddModule", "A module having the same name %s already added", module->GetName());
450 return;
451 }
452 if (!fModules) {
453 fModules = new TObjArray();
454 fModules->SetOwner();
455 }
456 fModules->Add(module);
457}
458
459//______________________________________________________________________________
460void AliAnalysisAlien::AddModules(TObjArray *list)
461{
462// Adding a list of modules. Checks if already existing. Becomes owned by this.
463 TIter next(list);
464 AliAnalysisTaskCfg *module;
465 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
466}
467
468//______________________________________________________________________________
469Bool_t AliAnalysisAlien::CheckDependencies()
470{
471// Check if all dependencies are satisfied. Reorder modules if needed.
472 Int_t nmodules = GetNmodules();
473 if (!nmodules) {
474 Warning("CheckDependencies", "No modules added yet to check their dependencies");
475 return kTRUE;
476 }
477 AliAnalysisTaskCfg *mod = 0;
478 AliAnalysisTaskCfg *dep = 0;
479 TString depname;
480 Int_t i, j, k;
481 for (i=0; i<nmodules; i++) {
482 mod = (AliAnalysisTaskCfg*) fModules->At(i);
483 Int_t ndeps = mod->GetNdeps();
484 Int_t istart = i;
485 for (j=0; j<ndeps; j++) {
486 depname = mod->GetDependency(j);
487 dep = GetModule(depname);
488 if (!dep) {
489 Error("CheckDependencies","Dependency %s not added for module %s",
490 depname.Data(), mod->GetName());
491 return kFALSE;
492 }
493 if (dep->NeedsDependency(mod->GetName())) {
494 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
495 mod->GetName(), dep->GetName());
496 return kFALSE;
497 }
498 Int_t idep = fModules->IndexOf(dep);
499 // The dependency task must come first
500 if (idep>i) {
501 // Remove at idep and move all objects below up one slot
502 // down to index i included.
503 fModules->RemoveAt(idep);
3e40fd4c 504 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 505 fModules->AddAt(dep, i++);
506 }
507 //Redo from istart if dependencies were inserted
508 if (i>istart) i=istart-1;
509 }
510 }
511 return kTRUE;
512}
513
3e40fd4c 514//______________________________________________________________________________
515AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
516{
517// Create the analysis manager and optionally execute the macro in filename.
518 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
519 if (mgr) return mgr;
520 mgr = new AliAnalysisManager(name);
521 mgr->SetGridHandler((AliAnalysisGrid*)this);
522 if (strlen(filename)) {
523 TString line = gSystem->ExpandPathName(filename);
524 line.Prepend(".x ");
525 gROOT->ProcessLine(line.Data());
526 }
527 return mgr;
528}
529
77f34eae 530//______________________________________________________________________________
531Int_t AliAnalysisAlien::GetNmodules() const
532{
533// Get number of modules.
534 if (!fModules) return 0;
535 return fModules->GetEntries();
536}
537
538//______________________________________________________________________________
539AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
540{
541// Get a module by name.
542 if (!fModules) return 0;
543 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
544}
545
546//______________________________________________________________________________
547Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
548{
549// Load a given module.
550 if (mod->IsLoaded()) return kTRUE;
192ddca2 551 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
552 if (!mgr) {
553 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
554 return kFALSE;
555 }
77f34eae 556 Int_t ndeps = mod->GetNdeps();
557 TString depname;
558 for (Int_t j=0; j<ndeps; j++) {
559 depname = mod->GetDependency(j);
560 AliAnalysisTaskCfg *dep = GetModule(depname);
561 if (!dep) {
562 Error("LoadModule","Dependency %s not existing for module %s",
563 depname.Data(), mod->GetName());
564 return kFALSE;
565 }
566 if (!LoadModule(dep)) {
567 Error("LoadModule","Dependency %s for module %s could not be loaded",
568 depname.Data(), mod->GetName());
569 return kFALSE;
570 }
571 }
572 // Load libraries for the module
573 if (!mod->CheckLoadLibraries()) {
574 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
575 return kFALSE;
576 }
192ddca2 577 // Check if a custom file name was requested
578 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
579
580 // Check if a custom terminate file name was requested
581 if (strlen(mod->GetTerminateFileName())) {
582 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
583 fTerminateFiles += mod->GetTerminateFileName();
584 }
585
77f34eae 586 // Execute the macro
587 if (mod->ExecuteMacro()<0) {
588 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
589 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
590 return kFALSE;
591 }
592 // Configure dependencies
593 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
594 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
595 mod->GetConfigMacro()->GetTitle(), mod->GetName());
596 return kFALSE;
597 }
5e8c7c22 598 // Adjust extra libraries
599 Int_t nlibs = mod->GetNlibs();
600 TString lib;
601 for (Int_t i=0; i<nlibs; i++) {
602 lib = mod->GetLibrary(i);
5e8c7c22 603 lib = Form("lib%s.so", lib.Data());
c9d6bd12 604 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 605 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
606 fAdditionalLibs += lib;
607 }
77f34eae 608 return kTRUE;
609}
610
139fe6ba 611//______________________________________________________________________________
612Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
613{
614// Generate the full train.
615 fAdditionalLibs = "";
616 if (!LoadModules()) return kFALSE;
617 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
618 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 619 mgr->RunLocalInit();
139fe6ba 620 mgr->PrintStatus();
621 Int_t productionMode = fProductionMode;
622 SetProductionMode();
623 TString macro = fAnalysisMacro;
624 TString executable = fExecutable;
625 TString validation = fValidationScript;
626 TString execCommand = fExecutableCommand;
627 SetAnalysisMacro(Form("%s.C", name));
628 SetExecutable(Form("%s.sh", name));
a8e31fb0 629// SetExecutableCommand("aliroot -b -q ");
139fe6ba 630 SetValidationScript(Form("%s_validation.sh", name));
631 StartAnalysis();
632 SetProductionMode(productionMode);
633 fAnalysisMacro = macro;
634 fExecutable = executable;
635 fExecutableCommand = execCommand;
636 fValidationScript = validation;
637 return kTRUE;
638}
639
77f34eae 640//______________________________________________________________________________
21ca8e59 641Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 642{
643// Generate test macros for a single module or for the full train.
5e8c7c22 644 fAdditionalLibs = "";
77f34eae 645 if (strlen(modname)) {
21ca8e59 646 if (!CheckDependencies()) return kFALSE;
647 AliAnalysisTaskCfg *mod = GetModule(modname);
648 if (!mod) {
649 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
650 return kFALSE;
651 }
652 if (!LoadModule(mod)) return kFALSE;
653 } else if (!LoadModules()) return kFALSE;
654 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
655 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 656 mgr->RunLocalInit();
21ca8e59 657 mgr->PrintStatus();
658 SetLocalTest(kTRUE);
659 Int_t productionMode = fProductionMode;
660 SetProductionMode();
661 TString macro = fAnalysisMacro;
662 TString executable = fExecutable;
663 TString validation = fValidationScript;
664 TString execCommand = fExecutableCommand;
665 SetAnalysisMacro(Form("%s.C", name));
666 SetExecutable(Form("%s.sh", name));
a8e31fb0 667// SetExecutableCommand("aliroot -b -q ");
21ca8e59 668 SetValidationScript(Form("%s_validation.sh", name));
669 WriteAnalysisFile();
670 WriteAnalysisMacro();
671 WriteExecutable();
672 WriteValidationScript();
ffbe06b9 673 WriteMergingMacro();
674 WriteMergeExecutable();
675 WriteValidationScript(kTRUE);
21ca8e59 676 SetLocalTest(kFALSE);
677 SetProductionMode(productionMode);
678 fAnalysisMacro = macro;
679 fExecutable = executable;
680 fExecutableCommand = execCommand;
681 fValidationScript = validation;
77f34eae 682 return kTRUE;
683}
684
685//______________________________________________________________________________
686Bool_t AliAnalysisAlien::LoadModules()
687{
688// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 689 fAdditionalLibs = "";
21ca8e59 690 Int_t nmodules = GetNmodules();
691 if (!nmodules) {
692 Warning("LoadModules", "No module to be loaded");
693 return kTRUE;
694 }
3e40fd4c 695 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
696 if (!mgr) {
697 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
698 return kFALSE;
699 }
77f34eae 700 if (!CheckDependencies()) return kFALSE;
21ca8e59 701 nmodules = GetNmodules();
77f34eae 702 AliAnalysisTaskCfg *mod;
21ca8e59 703 for (Int_t imod=0; imod<nmodules; imod++) {
704 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 705 if (!LoadModule(mod)) return kFALSE;
706 }
77f34eae 707 return kTRUE;
708}
709
74013535 710//______________________________________________________________________________
711void AliAnalysisAlien::SetRunPrefix(const char *prefix)
712{
713// Set the run number format. Can be a prefix or a format like "%09d"
714 fRunPrefix = prefix;
715 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
716}
717
f965131e 718//______________________________________________________________________________
719void AliAnalysisAlien::AddIncludePath(const char *path)
720{
721// Add include path in the remote analysis macro.
722 TString p(path);
723 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
724 else fIncludePath += Form("-I%s ", path);
725}
726
c57f56b7 727//______________________________________________________________________________
728void AliAnalysisAlien::AddRunNumber(Int_t run)
729{
730// Add a run number to the list of runs to be processed.
731 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 732 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 733}
734
98690895 735//______________________________________________________________________________
736void AliAnalysisAlien::AddRunList(const char* runList)
737{
738// Add several runs into the list of runs; they are expected to be separated by a blank character.
739 TString sList = runList;
740 TObjArray *list = sList.Tokenize(" ");
741 Int_t n = list->GetEntries();
742 for (Int_t i = 0; i < n; i++) {
743 TObjString *os = (TObjString*)list->At(i);
744 AddRunNumber(os->GetString().Atoi());
745 }
746 delete list;
747}
748
ee75cfc3 749//______________________________________________________________________________
750void AliAnalysisAlien::AddRunNumber(const char* run)
751{
752// Add a run number to the list of runs to be processed.
7aa5d539 753 TString runs = run;
754 TObjString *os;
755 TObjArray *arr = runs.Tokenize(" ");
756 TIter next(arr);
757 TString prefix;
758 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
759 while ((os=(TObjString*)next())){
760 if (fRunNumbers.Length()) fRunNumbers += " ";
761 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
762 }
763 delete arr;
ee75cfc3 764}
765
c57f56b7 766//______________________________________________________________________________
767void AliAnalysisAlien::AddDataFile(const char *lfn)
768{
769// Adds a data file to the input to be analysed. The file should be a valid LFN
770// or point to an existing file in the alien workdir.
771 if (!fInputFiles) fInputFiles = new TObjArray();
772 fInputFiles->Add(new TObjString(lfn));
773}
648174cf 774
775//______________________________________________________________________________
776void AliAnalysisAlien::AddExternalPackage(const char *package)
777{
778// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
779 if (fExternalPackages) fExternalPackages += " ";
780 fExternalPackages += package;
781}
782
c57f56b7 783//______________________________________________________________________________
784Bool_t AliAnalysisAlien::Connect()
785{
786// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
787 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 788 if (fProductionMode) return kTRUE;
c57f56b7 789 if (!gGrid) {
790 Info("Connect", "Trying to connect to AliEn ...");
791 TGrid::Connect("alien://");
792 }
793 if (!gGrid || !gGrid->IsConnected()) {
794 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
795 return kFALSE;
796 }
797 fUser = gGrid->GetUser();
798 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
799 return kTRUE;
800}
801
802//______________________________________________________________________________
803void AliAnalysisAlien::CdWork()
804{
805// Check validity of alien workspace. Create directory if possible.
806 if (!Connect()) {
807 Error("CdWork", "Alien connection required");
808 return;
809 }
810 TString homedir = gGrid->GetHomeDirectory();
811 TString workdir = homedir + fGridWorkingDir;
923e2ca5 812 if (DirectoryExists(workdir)) {
813 gGrid->Cd(workdir);
814 return;
815 }
816 // Work directory not existing - create it
817 gGrid->Cd(homedir);
b93f8109 818 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 819 gGrid->Cd(fGridWorkingDir);
d3b18c4c 820 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 821 } else {
d3b18c4c 822 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 823 workdir.Data(), homedir.Data());
824 fGridWorkingDir = "";
825 }
c57f56b7 826}
827
348be253 828//______________________________________________________________________________
829Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
830{
831// Check if file copying is possible.
d3b18c4c 832 if (fProductionMode) return kTRUE;
b5ae4fe0 833 TString salienpath(alienpath);
834 if (salienpath.Contains(" ")) {
835 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
836 return kFALSE;
837 }
348be253 838 if (!Connect()) {
839 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
840 return kFALSE;
841 }
d3339be3 842 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
843 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 844 // Check if alien_CLOSE_SE is defined
845 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
846 if (!closeSE.IsNull()) {
847 Info("CheckFileCopy", "Your current close storage is pointing to: \
848 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
849 } else {
850 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
851 }
852 // Check if grid directory exists.
853 if (!DirectoryExists(alienpath)) {
854 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
855 return kFALSE;
856 }
b5ae4fe0 857 TString stest = "plugin_test_copy";
858 TFile f(stest, "RECREATE");
348be253 859 // User may not have write permissions to current directory
860 if (f.IsZombie()) {
861 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
862 gSystem->WorkingDirectory());
863 return kFALSE;
864 }
865 f.Close();
b5ae4fe0 866 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
867 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 868 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 869 \n# 1. Make sure you have write permissions there. If this is the case: \
870 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
871 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
872 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
873 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 874 gSystem->Unlink(stest.Data());
348be253 875 return kFALSE;
876 }
b5ae4fe0 877 gSystem->Unlink(stest.Data());
878 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 879 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 880 return kTRUE;
881}
882
c57f56b7 883//______________________________________________________________________________
884Bool_t AliAnalysisAlien::CheckInputData()
885{
886// Check validity of input data. If necessary, create xml files.
d3b18c4c 887 if (fProductionMode) return kTRUE;
a8739e8a 888 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 889 if (!fGridDataDir.Length()) {
890 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
891 return kFALSE;
892 }
9f020952 893 if (fMergeViaJDL) {
894 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
895 return kFALSE;
896 }
d2a409b2 897 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 898 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
899 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 900 return kTRUE;
c57f56b7 901 }
902 // Process declared files
a2f5fc01 903 Bool_t isCollection = kFALSE;
904 Bool_t isXml = kFALSE;
905 Bool_t useTags = kFALSE;
c57f56b7 906 Bool_t checked = kFALSE;
d3b18c4c 907 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 908 TString file;
909 TString workdir = gGrid->GetHomeDirectory();
910 workdir += fGridWorkingDir;
911 if (fInputFiles) {
912 TObjString *objstr;
913 TIter next(fInputFiles);
914 while ((objstr=(TObjString*)next())) {
915 file = workdir;
916 file += "/";
917 file += objstr->GetString();
918 // Store full lfn path
919 if (FileExists(file)) objstr->SetString(file);
920 else {
921 file = objstr->GetName();
922 if (!FileExists(objstr->GetName())) {
923 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
924 objstr->GetName(), workdir.Data());
925 return kFALSE;
926 }
927 }
928 Bool_t iscoll, isxml, usetags;
929 CheckDataType(file, iscoll, isxml, usetags);
930 if (!checked) {
931 checked = kTRUE;
a2f5fc01 932 isCollection = iscoll;
933 isXml = isxml;
934 useTags = usetags;
935 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 936 } else {
a2f5fc01 937 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 938 Error("CheckInputData", "Some conflict was found in the types of inputs");
939 return kFALSE;
940 }
941 }
942 }
943 }
944 // Process requested run numbers
a8739e8a 945 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 946 // Check validity of alien data directory
947 if (!fGridDataDir.Length()) {
948 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
949 return kFALSE;
950 }
923e2ca5 951 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 952 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
953 return kFALSE;
954 }
a2f5fc01 955 if (isCollection) {
c57f56b7 956 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
957 return kFALSE;
958 }
959
a2f5fc01 960 if (checked && !isXml) {
c57f56b7 961 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
962 return kFALSE;
963 }
964 // Check validity of run number(s)
965 TObjArray *arr;
966 TObjString *os;
74013535 967 TString format;
319593fb 968 Int_t nruns = 0;
904f9f5f 969 TString schunk, schunk2;
c57f56b7 970 TString path;
971 if (!checked) {
972 checked = kTRUE;
a2f5fc01 973 useTags = fDataPattern.Contains("tag");
974 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 975 }
a2f5fc01 976 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 977 Error("CheckInputData", "Cannot mix input files using/not using tags");
978 return kFALSE;
979 }
980 if (fRunNumbers.Length()) {
a8739e8a 981 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 982 arr = fRunNumbers.Tokenize(" ");
983 TIter next(arr);
984 while ((os=(TObjString*)next())) {
985 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 986 if (!DirectoryExists(path)) {
987 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 988 continue;
c57f56b7 989 }
990 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
991 TString msg = "\n##### file: ";
992 msg += path;
993 msg += " type: xml_collection;";
a2f5fc01 994 if (useTags) msg += " using_tags: Yes";
c57f56b7 995 else msg += " using_tags: No";
23329835 996 Info("CheckDataType", "%s", msg.Data());
319593fb 997 if (fNrunsPerMaster<2) {
d2a409b2 998 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 999 } else {
1000 nruns++;
1001 if (((nruns-1)%fNrunsPerMaster) == 0) {
1002 schunk = os->GetString();
1003 }
1004 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1005 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1006 AddDataFile(schunk);
319593fb 1007 }
c57f56b7 1008 }
1009 delete arr;
a8739e8a 1010 } else {
1011 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1012 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1013 format = Form("%%s/%s ", fRunPrefix.Data());
1014 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1015 if (!DirectoryExists(path)) {
a8739e8a 1016 continue;
1017 }
74013535 1018 format = Form("%%s/%s.xml", fRunPrefix.Data());
1019 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1020 TString msg = "\n##### file: ";
1021 msg += path;
1022 msg += " type: xml_collection;";
a2f5fc01 1023 if (useTags) msg += " using_tags: Yes";
a8739e8a 1024 else msg += " using_tags: No";
23329835 1025 Info("CheckDataType", "%s", msg.Data());
319593fb 1026 if (fNrunsPerMaster<2) {
74013535 1027 format = Form("%s.xml", fRunPrefix.Data());
1028 AddDataFile(Form(format.Data(),irun));
319593fb 1029 } else {
1030 nruns++;
1031 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1032 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1033 }
74013535 1034 format = Form("_%s.xml", fRunPrefix.Data());
1035 schunk2 = Form(format.Data(), irun);
319593fb 1036 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1037 schunk += schunk2;
d2a409b2 1038 AddDataFile(schunk);
319593fb 1039 }
a8739e8a 1040 }
904f9f5f 1041 if (!fInputFiles) {
1042 schunk += schunk2;
1043 AddDataFile(schunk);
1044 }
c57f56b7 1045 }
1046 return kTRUE;
1047}
1048
21ca8e59 1049//______________________________________________________________________________
4a8b667c 1050Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1051{
1052// Copy data from the given grid directory according a pattern and make a local
1053// dataset.
2ba4c287 1054// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1055 if (!Connect()) {
1056 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
4a8b667c 1057 return 0;
21ca8e59 1058 }
1059 if (!DirectoryExists(griddir)) {
1060 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
4a8b667c 1061 return 0;
21ca8e59 1062 }
1063 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1064 printf("Running command: %s\n", command.Data());
1065 TGridResult *res = gGrid->Command(command);
1066 Int_t nfound = res->GetEntries();
1067 if (!nfound) {
1068 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
4a8b667c 1069 return 0;
21ca8e59 1070 }
1071 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1072
1073 // archives
1074 TObjArray* additionalArchives = 0;
1075 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1076 additionalArchives = TString(archivefile).Tokenize(";");
1077 archivefile = additionalArchives->At(0)->GetName();
1078 additionalArchives->RemoveAt(0);
1079 additionalArchives->Compress();
1080 }
1081
21ca8e59 1082 // Copy files locally
1083 ofstream out;
1084 out.open(output, ios::out);
1085 TMap *map;
1086 TString turl, dirname, filename, temp;
1087 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1088 gSystem->MakeDirectory(outputdir);
1089 gSystem->ChangeDirectory(outputdir);
4a8b667c 1090 Int_t ncopied = 0;
21ca8e59 1091 for (Int_t i=0; i<nfound; i++) {
1092 map = (TMap*)res->At(i);
1093 turl = map->GetValue("turl")->GetName();
1094 filename = gSystem->BaseName(turl.Data());
1095 dirname = gSystem->DirName(turl.Data());
1096 dirname = gSystem->BaseName(dirname.Data());
1097 gSystem->MakeDirectory(dirname);
43b807f6 1098
1099 TString source(turl);
1100 TString targetFileName(filename);
1101
1102 if (strlen(archivefile) > 0) {
4a8b667c 1103// TODO here the archive in which the file resides should be determined
1104// however whereis returns only a guid, and guid2lfn does not work
1105// Therefore we use the one provided as argument for now
1106 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1107 targetFileName = archivefile;
43b807f6 1108 }
1109 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
4a8b667c 1110 Bool_t success = kTRUE;
1111 if (additionalArchives) {
1112 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1113 TString target;
1114 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1115 gSystem->MakeDirectory(gSystem->DirName(target));
1116 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1117 }
1118 }
2ba4c287 1119
4a8b667c 1120 if (success) {
1121 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1122 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1123 ncopied++;
1124 }
21ca8e59 1125 }
1126 }
1127 gSystem->ChangeDirectory(cdir);
1128 delete res;
4a8b667c 1129 delete additionalArchives;
1130 return ncopied;
21ca8e59 1131}
1132
c57f56b7 1133//______________________________________________________________________________
1134Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1135{
1136// Create dataset for the grid data directory + run number.
3af861ff 1137 const Int_t gMaxEntries = 15000;
d3b18c4c 1138 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1139 if (!Connect()) {
1140 Error("CreateDataset", "Cannot create dataset with no grid connection");
1141 return kFALSE;
1142 }
1143
1144 // Cd workspace
d3b18c4c 1145 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1146 TString workdir = gGrid->GetHomeDirectory();
1147 workdir += fGridWorkingDir;
1148
1149 // Compose the 'find' command arguments
74013535 1150 TString format;
c57f56b7 1151 TString command;
1152 TString options = "-x collection ";
bb885a9e 1153 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1154 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1155 TString conditions = "";
3af861ff 1156 Int_t nstart = 0;
1157 Int_t ncount = 0;
1158 Int_t stage = 0;
c57f56b7 1159 TString file;
1160 TString path;
319593fb 1161 Int_t nruns = 0;
904f9f5f 1162 TString schunk, schunk2;
ab254fd1 1163 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1164 if (!fRunNumbers.Length() && !fRunRange[0]) {
1165 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1166 // Make a single data collection from data directory.
1167 path = fGridDataDir;
923e2ca5 1168 if (!DirectoryExists(path)) {
d2a409b2 1169 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1170 return kFALSE;
1171 }
923e2ca5 1172// CdWork();
d2a409b2 1173 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1174 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1175 while (1) {
1176 ncount = 0;
1177 stage++;
1178 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1179 command = "find ";
1180 command += Form("%s -o %d ",options.Data(), nstart);
1181 command += path;
1182 command += " ";
1183 command += pattern;
1184 command += conditions;
1185 printf("command: %s\n", command.Data());
1186 TGridResult *res = gGrid->Command(command);
1187 if (res) delete res;
1188 // Write standard output to file
1189 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1190 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1191 Bool_t nullFile = kFALSE;
1192 if (!hasGrep) {
1193 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1194 } else {
1195 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1196 if (nullFile) {
1197 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1198 gSystem->Exec("rm -f __tmp*");
1199 return kFALSE;
1200 }
1201 TString line;
1202 ifstream in;
1203 in.open("__tmp__");
1204 in >> line;
1205 in.close();
1206 gSystem->Exec("rm -f __tmp__");
1207 ncount = line.Atoi();
1208 }
1209 }
1210 if (ncount == gMaxEntries) {
1211 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1212 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1213 if (!cbase) cbase = cadd;
1214 else {
1215 cbase->Add(cadd);
1216 delete cadd;
defd7a3a 1217 }
3af861ff 1218 nstart += ncount;
1219 } else {
1220 if (cbase) {
1221 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1222 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1223 cbase->Add(cadd);
1224 delete cadd;
1225 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1226 delete cbase; cbase = 0;
1227 } else {
1228 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1229 }
1230 gSystem->Exec("rm -f __tmp*");
1231 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1232 break;
1233 }
0f389141 1234 }
1235 Bool_t fileExists = FileExists(file);
1236 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1237 // Copy xml file to alien space
0f389141 1238 if (fileExists) gGrid->Rm(file);
d2a409b2 1239 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1240 if (!FileExists(file)) {
1241 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1242 return kFALSE;
1243 }
1244 // Update list of files to be processed.
1245 }
1246 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1247 return kTRUE;
1248 }
c57f56b7 1249 // Several runs
a2f5fc01 1250 Bool_t nullResult = kTRUE;
a8739e8a 1251 if (fRunNumbers.Length()) {
1252 TObjArray *arr = fRunNumbers.Tokenize(" ");
1253 TObjString *os;
1254 TIter next(arr);
1255 while ((os=(TObjString*)next())) {
3af861ff 1256 nstart = 0;
1257 stage = 0;
dc030a24 1258 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1259 if (!DirectoryExists(path)) continue;
1260// CdWork();
a8739e8a 1261 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1262 else file = Form("%s.xml", os->GetString().Data());
319593fb 1263 // If local collection file does not exist, create it via 'find' command.
3af861ff 1264 while (1) {
1265 ncount = 0;
1266 stage++;
1267 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1268 command = "find ";
1269 command += Form("%s -o %d ",options.Data(), nstart);
1270 command += path;
1271 command += pattern;
1272 command += conditions;
1273 TGridResult *res = gGrid->Command(command);
1274 if (res) delete res;
1275 // Write standard output to file
1276 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1277 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1278 Bool_t nullFile = kFALSE;
1279 if (!hasGrep) {
1280 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1281 } else {
1282 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1283 if (nullFile) {
1284 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1285 gSystem->Exec("rm -f __tmp*");
1286 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1287 break;
3af861ff 1288 }
1289 TString line;
1290 ifstream in;
1291 in.open("__tmp__");
1292 in >> line;
1293 in.close();
1294 gSystem->Exec("rm -f __tmp__");
1295 ncount = line.Atoi();
1296 }
1297 nullResult = kFALSE;
1298 }
1299 if (ncount == gMaxEntries) {
1300 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1301 if (fNrunsPerMaster > 1) {
1302 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1303 file.Data(),gMaxEntries);
1304 return kFALSE;
1305 }
1306 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1307 if (!cbase) cbase = cadd;
1308 else {
1309 cbase->Add(cadd);
1310 delete cadd;
defd7a3a 1311 }
3af861ff 1312 nstart += ncount;
1313 } else {
c89a84bc 1314 if (cbase && fNrunsPerMaster<2) {
3af861ff 1315 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1316 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1317 cbase->Add(cadd);
1318 delete cadd;
1319 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1320 delete cbase; cbase = 0;
1321 } else {
1322 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1323 }
1324 gSystem->Exec("rm -f __tmp*");
1325 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1326 break;
d3339be3 1327 }
3af861ff 1328 }
a8739e8a 1329 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1330 // Check if there is one run per master job.
1331 if (fNrunsPerMaster<2) {
1332 if (FileExists(file)) {
0f389141 1333 if (fOverwriteMode) gGrid->Rm(file);
1334 else {
1335 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1336 continue;
1337 }
319593fb 1338 }
1339 // Copy xml file to alien space
1340 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1341 if (!FileExists(file)) {
1342 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1343 delete arr;
1344 return kFALSE;
1345 }
1346 } else {
1347 nruns++;
1348 if (((nruns-1)%fNrunsPerMaster) == 0) {
1349 schunk = os->GetString();
1350 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1351 } else {
1352 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1353 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1354 cbase->Add(cadd);
1355 delete cadd;
1356 }
1357 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1358 continue;
1359 }
1360 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1361 if (FileExists(schunk)) {
1362 if (fOverwriteMode) gGrid->Rm(file);
1363 else {
1364 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1365 continue;
1366 }
319593fb 1367 }
84fcd93f 1368 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1369 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1370 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1371 if (!FileExists(schunk)) {
1372 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1373 delete arr;
1374 return kFALSE;
1375 }
d3339be3 1376 }
a8739e8a 1377 }
1378 delete arr;
a2f5fc01 1379 if (nullResult) {
d3339be3 1380 Error("CreateDataset", "No valid dataset corresponding to the query!");
1381 return kFALSE;
1382 }
a8739e8a 1383 } else {
1384 // Process a full run range.
1385 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1386 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1387 nstart = 0;
1388 stage = 0;
74013535 1389 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1390 if (!DirectoryExists(path)) continue;
1391// CdWork();
74013535 1392 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1393 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1394 else file = Form(format.Data(), irun);
0f389141 1395 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1396 if (fOverwriteMode) gGrid->Rm(file);
1397 else {
1398 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1399 continue;
1400 }
a8739e8a 1401 }
319593fb 1402 // If local collection file does not exist, create it via 'find' command.
3af861ff 1403 while (1) {
1404 ncount = 0;
1405 stage++;
1406 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1407 command = "find ";
1408 command += Form("%s -o %d ",options.Data(), nstart);
1409 command += path;
1410 command += pattern;
1411 command += conditions;
1412 TGridResult *res = gGrid->Command(command);
1413 if (res) delete res;
1414 // Write standard output to file
1415 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1416 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1417 Bool_t nullFile = kFALSE;
1418 if (!hasGrep) {
1419 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1420 } else {
1421 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1422 if (nullFile) {
1423 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1424 gSystem->Exec("rm -f __tmp*");
1604c9ed 1425 break;
3af861ff 1426 }
1427 TString line;
1428 ifstream in;
1429 in.open("__tmp__");
1430 in >> line;
1431 in.close();
1432 gSystem->Exec("rm -f __tmp__");
1433 ncount = line.Atoi();
1434 }
1435 nullResult = kFALSE;
1436 }
1437 if (ncount == gMaxEntries) {
1438 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1439 if (fNrunsPerMaster > 1) {
1440 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1441 file.Data(),gMaxEntries);
1442 return kFALSE;
1443 }
1444 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1445 if (!cbase) cbase = cadd;
1446 else {
1447 cbase->Add(cadd);
1448 delete cadd;
defd7a3a 1449 }
3af861ff 1450 nstart += ncount;
1451 } else {
c89a84bc 1452 if (cbase && fNrunsPerMaster<2) {
3af861ff 1453 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1454 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1455 cbase->Add(cadd);
1456 delete cadd;
1457 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1458 delete cbase; cbase = 0;
1459 } else {
1460 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1461 }
1462 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1463 break;
d3339be3 1464 }
319593fb 1465 }
a8739e8a 1466 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1467 // Check if there is one run per master job.
1468 if (fNrunsPerMaster<2) {
1469 if (FileExists(file)) {
0f389141 1470 if (fOverwriteMode) gGrid->Rm(file);
1471 else {
1472 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1473 continue;
1474 }
319593fb 1475 }
1476 // Copy xml file to alien space
1477 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1478 if (!FileExists(file)) {
1479 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1480 return kFALSE;
1481 }
1482 } else {
1483 nruns++;
95e5b448 1484 // Check if the collection for the chunk exist locally.
1485 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1486 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1487 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1488 else continue;
1489 }
84fcd93f 1490 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1491 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1492 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1493 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1494 } else {
1495 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1496 cbase->Add(cadd);
1497 delete cadd;
1498 }
74013535 1499 format = Form("%%s_%s.xml", fRunPrefix.Data());
1500 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1501 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1502 continue;
1503 }
904f9f5f 1504 schunk = schunk2;
319593fb 1505 if (FileExists(schunk)) {
0f389141 1506 if (fOverwriteMode) gGrid->Rm(schunk);
1507 else {
1508 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1509 continue;
1510 }
319593fb 1511 }
84fcd93f 1512 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1513 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1514 if (FileExists(schunk)) {
0f389141 1515 if (fOverwriteMode) gGrid->Rm(schunk);
1516 else {
1517 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1518 continue;
1519 }
95e5b448 1520 }
319593fb 1521 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1522 if (!FileExists(schunk)) {
1523 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1524 return kFALSE;
1525 }
1526 }
c57f56b7 1527 }
a2f5fc01 1528 if (nullResult) {
d3339be3 1529 Error("CreateDataset", "No valid dataset corresponding to the query!");
1530 return kFALSE;
1531 }
a8739e8a 1532 }
c57f56b7 1533 return kTRUE;
1534}
1535
1536//______________________________________________________________________________
1537Bool_t AliAnalysisAlien::CreateJDL()
1538{
1539// Generate a JDL file according to current settings. The name of the file is
1540// specified by fJDLName.
ec5acdce 1541 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
c57f56b7 1542 Bool_t error = kFALSE;
1543 TObjArray *arr = 0;
1544 Bool_t copy = kTRUE;
d3b18c4c 1545 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1546 Bool_t generate = kTRUE;
1547 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1548 if (!Connect()) {
1549 Error("CreateJDL", "Alien connection required");
1550 return kFALSE;
1551 }
1552 // Check validity of alien workspace
d3b18c4c 1553 TString workdir;
1554 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1555 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1556 workdir += fGridWorkingDir;
1557 if (generate) {
1558 TObjString *os;
1559 if (!fInputFiles) {
1560 Error("CreateJDL()", "Define some input files for your analysis.");
1561 error = kTRUE;
1562 }
1563 // Compose list of input files
1564 // Check if output files were defined
1565 if (!fOutputFiles.Length()) {
1566 Error("CreateJDL", "You must define at least one output file");
1567 error = kTRUE;
1568 }
1569 // Check if an output directory was defined and valid
1570 if (!fGridOutputDir.Length()) {
1571 Error("CreateJDL", "You must define AliEn output directory");
1572 error = kTRUE;
1573 } else {
d3b18c4c 1574 if (!fProductionMode) {
1575 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1576 if (!DirectoryExists(fGridOutputDir)) {
1577 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1578 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1579 } else {
1580 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1581 // error = kTRUE;
1582 }
67a3b254 1583 } else {
1584 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1585 }
1586 gGrid->Cd(workdir);
1587 }
c57f56b7 1588 }
1589 // Exit if any error up to now
1590 if (error) return kFALSE;
1591 // Set JDL fields
0f389141 1592 if (!fUser.IsNull()) {
1593 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1594 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1595 }
1596 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1597 TString mergeExec = fExecutable;
1598 mergeExec.ReplaceAll(".sh", "_merge.sh");
1599 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1600 mergeExec.ReplaceAll(".sh", ".C");
1601 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1602 if (!fArguments.IsNull())
1603 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1604 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1605 else {
1606 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1607 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1608 }
b3e07543 1609
1f0d1ca2 1610 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1611 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1612 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1613 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1614
0f389141 1615 if (fMaxInitFailed > 0) {
c57f56b7 1616 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1617 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1618 }
1619 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1620 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1621 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1622 }
1623 if (!IsOneStageMerging()) {
f47d5cb4 1624 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1625 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1626 }
1627 if (fSplitMode.Length()) {
c57f56b7 1628 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1629 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1630 }
1631 fMergingJDL->SetValue("Split", "\"se\"");
1632 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1633 if (!fAliROOTVersion.IsNull()) {
1634 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1635 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1636 }
1637 if (!fROOTVersion.IsNull()) {
c57f56b7 1638 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1639 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1640 }
1641 if (!fAPIVersion.IsNull()) {
c57f56b7 1642 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1643 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1644 }
648174cf 1645 if (!fExternalPackages.IsNull()) {
1646 arr = fExternalPackages.Tokenize(" ");
1647 TIter next(arr);
1648 while ((os=(TObjString*)next())) {
1649 TString pkgname = os->GetString();
1650 Int_t index = pkgname.Index("::");
1651 TString pkgversion = pkgname(index+2, pkgname.Length());
1652 pkgname.Remove(index);
1653 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1654 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1655 }
1656 delete arr;
1657 }
0f389141 1658 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1659 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1660 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1661 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1662 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1663 TString analysisFile = fExecutable;
1664 analysisFile.ReplaceAll(".sh", ".root");
1665 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1666 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1667 if (fAdditionalLibs.Length()) {
1668 arr = fAdditionalLibs.Tokenize(" ");
1669 TIter next(arr);
1670 while ((os=(TObjString*)next())) {
1671 if (os->GetString().Contains(".so")) continue;
1672 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1673 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1674 }
1675 delete arr;
1676 }
4e5c5506 1677 if (fPackages) {
1678 TIter next(fPackages);
1679 TObject *obj;
0f389141 1680 while ((obj=next())) {
4e5c5506 1681 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1682 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1683 }
4e5c5506 1684 }
c57f56b7 1685 if (fOutputArchive.Length()) {
e4d9a069 1686 TString outputArchive = fOutputArchive;
1687 if (!fRegisterExcludes.IsNull()) {
1688 arr = fRegisterExcludes.Tokenize(" ");
1689 TIter next1(arr);
1690 while ((os=(TObjString*)next1())) {
1691 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1692 outputArchive.ReplaceAll(os->GetString(),"");
1693 }
1694 delete arr;
1695 }
1696 arr = outputArchive.Tokenize(" ");
c57f56b7 1697 TIter next(arr);
0f389141 1698 Bool_t first = kTRUE;
1699 const char *comment = "Files to be archived";
1700 const char *comment1 = comment;
1701 while ((os=(TObjString*)next())) {
1702 if (!first) comment = NULL;
1703 if (!os->GetString().Contains("@") && fCloseSE.Length())
1704 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1705 else
1706 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1707 first = kFALSE;
1708 }
c57f56b7 1709 delete arr;
f790bc1b 1710 // Output archive for the merging jdl
f790bc1b 1711 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1712 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1713 // Add normal output files, extra files + terminate files
1714 TString files = GetListOfFiles("outextter");
37d21c01 1715 // Do not register files in fRegisterExcludes
1716 if (!fRegisterExcludes.IsNull()) {
1717 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1718 TIter next1(arr);
1719 while ((os=(TObjString*)next1())) {
1720 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1721 files.ReplaceAll(os->GetString(),"");
1722 }
66d71516 1723 delete arr;
0f389141 1724 }
f790bc1b 1725 files.ReplaceAll(".root", "*.root");
ec5acdce 1726
1727 if (mgr->IsCollectThroughput())
1728 outputArchive += Form("root_archive.zip:%s,*.stat,*%s@disk=%d",files.Data(),mgr->GetFileInfoLog(),fNreplicas);
1729 else
1730 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1731 } else {
95f84779 1732 TString files = fOutputArchive;
1733 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1734 outputArchive = files;
f790bc1b 1735 }
0f389141 1736 arr = outputArchive.Tokenize(" ");
705adb3e 1737 TIter next2(arr);
0f389141 1738 comment = comment1;
1739 first = kTRUE;
705adb3e 1740 while ((os=(TObjString*)next2())) {
0f389141 1741 if (!first) comment = NULL;
0cdf65a8 1742 TString currentfile = os->GetString();
0cdf65a8 1743 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1744 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1745 else
0cdf65a8 1746 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1747 first = kFALSE;
1748 }
1749 delete arr;
c57f56b7 1750 }
149d288c 1751 arr = fOutputFiles.Tokenize(",");
c57f56b7 1752 TIter next(arr);
0f389141 1753 Bool_t first = kTRUE;
f790bc1b 1754 const char *comment = "Files to be saved";
43da816a 1755 while ((os=(TObjString*)next())) {
e1eaf596 1756 // Ignore ouputs in jdl that are also in outputarchive
1757 TString sout = os->GetString();
f790bc1b 1758 sout.ReplaceAll("*", "");
1759 sout.ReplaceAll(".root", "");
e1eaf596 1760 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1761 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1762 // Ignore fRegisterExcludes
1763 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1764 if (!first) comment = NULL;
43da816a 1765 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1766 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1767 else
0f389141 1768 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1769 first = kFALSE;
f790bc1b 1770 if (fMergeExcludes.Contains(sout)) continue;
1771 if (!os->GetString().Contains("@") && fCloseSE.Length())
1772 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1773 else
1774 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1775 }
c57f56b7 1776 delete arr;
0f389141 1777 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1778 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1779 TString validationScript = fValidationScript;
0f389141 1780 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1781 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1782 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1783 if (fMasterResubmitThreshold) {
1784 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1785 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1786 }
d2a409b2 1787 // Write a jdl with 2 input parameters: collection name and output dir name.
1788 WriteJDL(copy);
c57f56b7 1789 }
1790 // Copy jdl to grid workspace
a8739e8a 1791 if (copy) {
b5e4aaa7 1792 // Check if an output directory was defined and valid
1793 if (!fGridOutputDir.Length()) {
1794 Error("CreateJDL", "You must define AliEn output directory");
1795 return kFALSE;
1796 } else {
1797 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1798 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1799 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1800 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1801 } else {
1802 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1803 return kFALSE;
1804 }
1805 }
1806 gGrid->Cd(workdir);
1807 }
648174cf 1808 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1809 TString mergeJDLName = fExecutable;
1810 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1811 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1812 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1813 if (fProductionMode) {
648174cf 1814 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1815 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1816 }
648174cf 1817 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1818 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1819 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1820 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1821 Fatal("","Terminating");
1822// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1823 if (fMergeViaJDL) {
1824 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1825// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1826 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1827 Fatal("","Terminating");
0f389141 1828 }
648174cf 1829 }
c57f56b7 1830 if (fAdditionalLibs.Length()) {
e7c71df0 1831 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1832 TObjString *os;
1833 TIter next(arr);
1834 while ((os=(TObjString*)next())) {
c57f56b7 1835 if (os->GetString().Contains(".so")) continue;
4e5c5506 1836 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1837 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1838// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1839 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1840 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1841 Fatal("","Terminating");
c57f56b7 1842 }
1843 delete arr;
1844 }
4e5c5506 1845 if (fPackages) {
1846 TIter next(fPackages);
1847 TObject *obj;
1848 while ((obj=next())) {
fdbbc7be 1849 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1850 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1851// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1852 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1853 Form("%s/%s", workdir.Data(), obj->GetName())))
1854 Fatal("","Terminating");
4e5c5506 1855 }
1856 }
c57f56b7 1857 }
1858 return kTRUE;
1859}
1860
a8739e8a 1861//______________________________________________________________________________
d2a409b2 1862Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1863{
1864// Writes one or more JDL's corresponding to findex. If findex is negative,
1865// all run numbers are considered in one go (jdl). For non-negative indices
1866// they correspond to the indices in the array fInputFiles.
1867 if (!fInputFiles) return kFALSE;
d3b18c4c 1868 TObject *os;
1869 TString workdir;
1870 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1871 workdir += fGridWorkingDir;
f47d5cb4 1872 TString stageName = "$2";
1873 if (fProductionMode) stageName = "$4";
1874 if (!fMergeDirName.IsNull()) {
1875 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1876 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1877 } else {
1878 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1879 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1880 }
d3b18c4c 1881 if (fProductionMode) {
a8739e8a 1882 TIter next(fInputFiles);
b3e07543 1883 while ((os=next())) {
d3b18c4c 1884 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1885 }
56754bf6 1886 if (!fOutputToRunNo)
1887 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1888 else
1889 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1890 } else {
1891 if (!fRunNumbers.Length() && !fRunRange[0]) {
1892 // One jdl with no parameters in case input data is specified by name.
1893 TIter next(fInputFiles);
1894 while ((os=next()))
1895 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1896 if (!fOutputSingle.IsNull())
1897 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1898 else {
1899 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1900 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1901 }
1902 } else {
1903 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1904 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1905 if (!fOutputSingle.IsNull()) {
1906 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1907 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1908 } else {
1909 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1910 }
1911 }
a8739e8a 1912 }
1913
a8739e8a 1914 // Generate the JDL as a string
1915 TString sjdl = fGridJDL->Generate();
0f389141 1916 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1917 // Final merge jdl
1918 if (!fMergeDirName.IsNull()) {
1919 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1920 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1921 } else {
1922 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1923 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1924 }
b3e07543 1925 TString sjdl2 = fMergingJDL->Generate();
1926 Int_t index, index1;
a8739e8a 1927 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1928 sjdl.ReplaceAll("(member", "\n (member");
1929 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1930 sjdl.ReplaceAll("{", "{\n ");
1931 sjdl.ReplaceAll("};", "\n};");
1932 sjdl.ReplaceAll("{\n \n", "{\n");
1933 sjdl.ReplaceAll("\n\n", "\n");
1934 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1935 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1936 sjdl1.ReplaceAll("(member", "\n (member");
1937 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1938 sjdl1.ReplaceAll("{", "{\n ");
1939 sjdl1.ReplaceAll("};", "\n};");
1940 sjdl1.ReplaceAll("{\n \n", "{\n");
1941 sjdl1.ReplaceAll("\n\n", "\n");
1942 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1943 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1944 sjdl2.ReplaceAll("(member", "\n (member");
1945 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1946 sjdl2.ReplaceAll("{", "{\n ");
1947 sjdl2.ReplaceAll("};", "\n};");
1948 sjdl2.ReplaceAll("{\n \n", "{\n");
1949 sjdl2.ReplaceAll("\n\n", "\n");
1950 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1951 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1952 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1953 index = sjdl.Index("JDLVariables");
1954 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1955 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1956 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1957 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1958 index = fJobTag.Index(":");
1959 if (index < 0) index = fJobTag.Length();
1960 TString jobTag = fJobTag;
f47d5cb4 1961 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1962 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1963 if (fProductionMode) {
1964 sjdl1.Prepend("# Generated merging jdl (production mode) \
1965 \n# $1 = full alien path to output directory to be merged \
1966 \n# $2 = train number \
1967 \n# $3 = production (like LHC10b) \
1968 \n# $4 = merging stage \
1969 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1970 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1971 sjdl2.Prepend("# Generated merging jdl \
1972 \n# $1 = full alien path to output directory to be merged \
1973 \n# $2 = train number \
1974 \n# $3 = production (like LHC10b) \
1975 \n# $4 = merging stage \
1976 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1977 } else {
1978 sjdl1.Prepend("# Generated merging jdl \
1979 \n# $1 = full alien path to output directory to be merged \
1980 \n# $2 = merging stage \
03335324 1981 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1982 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1983 sjdl2.Prepend("# Generated merging jdl \
1984 \n# $1 = full alien path to output directory to be merged \
1985 \n# $2 = merging stage \
03335324 1986 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1987 }
0f389141 1988 index = sjdl1.Index("JDLVariables");
1989 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1990 index = sjdl2.Index("JDLVariables");
1991 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1992 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1993 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1994 index = sjdl2.Index("Split =");
1995 if (index>=0) {
1996 index1 = sjdl2.Index("\n", index);
1997 sjdl2.Remove(index, index1-index+1);
1998 }
1999 index = sjdl2.Index("SplitMaxInputFileNumber");
2000 if (index>=0) {
2001 index1 = sjdl2.Index("\n", index);
2002 sjdl2.Remove(index, index1-index+1);
2003 }
2004 index = sjdl2.Index("InputDataCollection");
2005 if (index>=0) {
2006 index1 = sjdl2.Index(";", index);
2007 sjdl2.Remove(index, index1-index+1);
2008 }
2009 index = sjdl2.Index("InputDataListFormat");
2010 if (index>=0) {
2011 index1 = sjdl2.Index("\n", index);
2012 sjdl2.Remove(index, index1-index+1);
2013 }
2014 index = sjdl2.Index("InputDataList");
2015 if (index>=0) {
2016 index1 = sjdl2.Index("\n", index);
2017 sjdl2.Remove(index, index1-index+1);
2018 }
f47d5cb4 2019 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2020 // Write jdl to file
a8739e8a 2021 ofstream out;
d2a409b2 2022 out.open(fJDLName.Data(), ios::out);
a8739e8a 2023 if (out.bad()) {
d3b18c4c 2024 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2025 return kFALSE;
2026 }
2027 out << sjdl << endl;
b3e07543 2028 out.close();
0f389141 2029 TString mergeJDLName = fExecutable;
2030 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2031 if (fMergeViaJDL) {
2032 ofstream out1;
2033 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2034 if (out1.bad()) {
d3b18c4c 2035 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2036 return kFALSE;
2037 }
2038 out1 << sjdl1 << endl;
b3e07543 2039 out1.close();
2040 ofstream out2;
2041 TString finalJDL = mergeJDLName;
2042 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2043 out2.open(finalJDL.Data(), ios::out);
2044 if (out2.bad()) {
2045 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2046 return kFALSE;
2047 }
2048 out2 << sjdl2 << endl;
2049 out2.close();
0f389141 2050 }
a8739e8a 2051
2052 // Copy jdl to grid workspace
2053 if (!copy) {
d3b18c4c 2054 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2055 } else {
d2a409b2 2056 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2057 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2058 TString finalJDL = mergeJDLName;
2059 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2060 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2061 if (fProductionMode) {
b5fe9cba 2062 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2063 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2064 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2065 }
d2a409b2 2066 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2067 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2068 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2069 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2070// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2071 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2072 Fatal("","Terminating");
0f389141 2073 if (fMergeViaJDL) {
b3e07543 2074 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2075// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2076// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2077 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2078 Fatal("","Terminating");
2079 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2080 Fatal("","Terminating");
0f389141 2081 }
a8739e8a 2082 }
2083 return kTRUE;
2084}
2085
c57f56b7 2086//______________________________________________________________________________
5513444a 2087Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2088{
2089// Returns true if file exists.
5513444a 2090 if (!gGrid) return kFALSE;
2ea55496 2091 TString slfn = lfn;
2092 slfn.ReplaceAll("alien://","");
2093 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2094 if (!res) return kFALSE;
2095 TMap *map = dynamic_cast<TMap*>(res->At(0));
2096 if (!map) {
2097 delete res;
2098 return kFALSE;
2099 }
2100 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2101 if (!objs || !objs->GetString().Length()) {
2102 delete res;
2103 return kFALSE;
2104 }
2105 delete res;
2106 return kTRUE;
2107}
2108
923e2ca5 2109//______________________________________________________________________________
2110Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2111{
2112// Returns true if directory exists. Can be also a path.
2113 if (!gGrid) return kFALSE;
2114 // Check if dirname is a path
2115 TString dirstripped = dirname;
2116 dirstripped = dirstripped.Strip();
2117 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2118 TString dir = gSystem->BaseName(dirstripped);
2119 dir += "/";
2120 TString path = gSystem->DirName(dirstripped);
2121 TGridResult *res = gGrid->Ls(path, "-F");
2122 if (!res) return kFALSE;
2123 TIter next(res);
2124 TMap *map;
2125 TObject *obj;
2126 while ((map=dynamic_cast<TMap*>(next()))) {
2127 obj = map->GetValue("name");
2128 if (!obj) break;
2129 if (dir == obj->GetName()) {
2130 delete res;
2131 return kTRUE;
2132 }
2133 }
2134 delete res;
2135 return kFALSE;
2136}
2137
c57f56b7 2138//______________________________________________________________________________
a2f5fc01 2139void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2140{
2141// Check input data type.
a2f5fc01 2142 isCollection = kFALSE;
2143 isXml = kFALSE;
2144 useTags = kFALSE;
c57f56b7 2145 if (!gGrid) {
2146 Error("CheckDataType", "No connection to grid");
2147 return;
2148 }
a2f5fc01 2149 isCollection = IsCollection(lfn);
c57f56b7 2150 TString msg = "\n##### file: ";
2151 msg += lfn;
a2f5fc01 2152 if (isCollection) {
c57f56b7 2153 msg += " type: raw_collection;";
2154 // special treatment for collections
a2f5fc01 2155 isXml = kFALSE;
c57f56b7 2156 // check for tag files in the collection
2157 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2158 if (!res) {
2159 msg += " using_tags: No (unknown)";
23329835 2160 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2161 return;
2162 }
2163 const char* typeStr = res->GetKey(0, "origLFN");
2164 if (!typeStr || !strlen(typeStr)) {
2165 msg += " using_tags: No (unknown)";
23329835 2166 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2167 return;
2168 }
2169 TString file = typeStr;
a2f5fc01 2170 useTags = file.Contains(".tag");
2171 if (useTags) msg += " using_tags: Yes";
c57f56b7 2172 else msg += " using_tags: No";
23329835 2173 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2174 return;
2175 }
2176 TString slfn(lfn);
2177 slfn.ToLower();
a2f5fc01 2178 isXml = slfn.Contains(".xml");
2179 if (isXml) {
c57f56b7 2180 // Open xml collection and check if there are tag files inside
2181 msg += " type: xml_collection;";
2182 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2183 if (!coll) {
2184 msg += " using_tags: No (unknown)";
23329835 2185 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2186 return;
2187 }
2188 TMap *map = coll->Next();
2189 if (!map) {
2190 msg += " using_tags: No (unknown)";
23329835 2191 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2192 return;
2193 }
2194 map = (TMap*)map->GetValue("");
2195 TString file;
2196 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2197 useTags = file.Contains(".tag");
c57f56b7 2198 delete coll;
a2f5fc01 2199 if (useTags) msg += " using_tags: Yes";
c57f56b7 2200 else msg += " using_tags: No";
23329835 2201 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2202 return;
2203 }
a2f5fc01 2204 useTags = slfn.Contains(".tag");
c57f56b7 2205 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2206 else msg += " type: unknown file;";
a2f5fc01 2207 if (useTags) msg += " using_tags: Yes";
c57f56b7 2208 else msg += " using_tags: No";
23329835 2209 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2210}
2211
4e5c5506 2212//______________________________________________________________________________
2213void AliAnalysisAlien::EnablePackage(const char *package)
2214{
2215// Enables a par file supposed to exist in the current directory.
2216 TString pkg(package);
2217 pkg.ReplaceAll(".par", "");
2218 pkg += ".par";
2219 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2220 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2221 return;
2222 }
2223 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2224 Info("EnablePackage", "AliEn plugin will use .par packages");
2225 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2226 if (!fPackages) {
2227 fPackages = new TObjArray();
2228 fPackages->SetOwner();
2229 }
2230 fPackages->Add(new TObjString(pkg));
2231}
2232
3bdcb562 2233//______________________________________________________________________________
2234TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2235{
2236// Make a tree from files having the location specified in fFileForTestMode.
2237// Inspired from JF's CreateESDChain.
2238 if (fFileForTestMode.IsNull()) {
2239 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2240 return NULL;
2241 }
2242 if (gSystem->AccessPathName(fFileForTestMode)) {
2243 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2244 return NULL;
2245 }
2246 // Open the file
2247 ifstream in;
2248 in.open(fFileForTestMode);
2249 Int_t count = 0;
2250 // Read the input list of files and add them to the chain
b505f1f8 2251 TString line;
d6c60197 2252 TString streeName(treeName);
2253 if (IsUseMCchain()) streeName = "TE";
2254 TChain *chain = new TChain(streeName);
b505f1f8 2255 TChain *chainFriend = 0;
d6c60197 2256 if (!fFriendChainName.IsNull()) chainFriend = new TChain(streeName);
b505f1f8 2257 while (in.good())
2258 {
3bdcb562 2259 in >> line;
ffbe06b9 2260 if (line.IsNull() || line.BeginsWith("#")) continue;
3bdcb562 2261 if (count++ == fNtestFiles) break;
2262 TString esdFile(line);
2263 TFile *file = TFile::Open(esdFile);
b505f1f8 2264 if (file && !file->IsZombie()) {
2265 chain->Add(esdFile);
3bdcb562 2266 file->Close();
b505f1f8 2267 if (!fFriendChainName.IsNull()) {
4b5d9e21 2268 if (esdFile.Index("#") > -1)
2269 esdFile.Remove(esdFile.Index("#"));
2270 esdFile = gSystem->DirName(esdFile);
2271 esdFile += "/" + fFriendChainName;
b505f1f8 2272 file = TFile::Open(esdFile);
2273 if (file && !file->IsZombie()) {
2274 file->Close();
2275 chainFriend->Add(esdFile);
2276 } else {
2277 Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data());
2278 return 0;
2279 }
4b5d9e21 2280 }
3bdcb562 2281 } else {
2282 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2283 }
b505f1f8 2284 }
2285 in.close();
2286 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2287 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2288 delete chain;
b505f1f8 2289 delete chainFriend;
3bdcb562 2290 return NULL;
b505f1f8 2291 }
3bdcb562 2292// chain->ls();
b505f1f8 2293 if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend);
2294 return chain;
3bdcb562 2295}
2296
d2a409b2 2297//______________________________________________________________________________
2298const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2299{
2300// Get job status for all jobs with jobid>jobidstart.
2301 static char mstatus[20];
2302 mstatus[0] = '\0';
2303 nrunning = 0;
2304 nwaiting = 0;
2305 nerror = 0;
2306 ndone = 0;
2307 TGridJobStatusList *list = gGrid->Ps("");
2308 if (!list) return mstatus;
2309 Int_t nentries = list->GetSize();
2310 TGridJobStatus *status;
2311 Int_t pid;
2312 for (Int_t ijob=0; ijob<nentries; ijob++) {
2313 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2314 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2315 if (pid<jobidstart) continue;
2316 if (pid == lastid) {
8ddad121 2317 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2318 }
2319 switch (status->GetStatus()) {
2320 case TGridJobStatus::kWAITING:
2321 nwaiting++; break;
2322 case TGridJobStatus::kRUNNING:
2323 nrunning++; break;
2324 case TGridJobStatus::kABORTED:
2325 case TGridJobStatus::kFAIL:
2326 case TGridJobStatus::kUNKNOWN:
2327 nerror++; break;
2328 case TGridJobStatus::kDONE:
2329 ndone++;
2330 }
2331 }
2332 list->Delete();
2333 delete list;
2334 return mstatus;
2335}
2336
c57f56b7 2337//______________________________________________________________________________
2338Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2339{
2340// Returns true if file is a collection. Functionality duplicated from
2341// TAlien::Type() because we don't want to directly depend on TAlien.
2342 if (!gGrid) {
2343 Error("IsCollection", "No connection to grid");
2344 return kFALSE;
2345 }
2346 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2347 if (!res) return kFALSE;
2348 const char* typeStr = res->GetKey(0, "type");
2349 if (!typeStr || !strlen(typeStr)) return kFALSE;
2350 if (!strcmp(typeStr, "collection")) return kTRUE;
2351 delete res;
2352 return kFALSE;
2353}
2354
fe2d7fc2 2355//______________________________________________________________________________
2356Bool_t AliAnalysisAlien::IsSingleOutput() const
2357{
2358// Check if single-ouput option is on.
2359 return (!fOutputSingle.IsNull());
2360}
2361
16a4353c 2362//______________________________________________________________________________
2363void AliAnalysisAlien::Print(Option_t *) const
2364{
2365// Print current plugin settings.
84fcd93f 2366 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2367 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2368 if (mgr && mgr->IsProofMode()) {
2369 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2370 if (TestBit(AliAnalysisGrid::kTest))
2371 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2372 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2373 if (!fProofDataSet.IsNull())
2374 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2375 if (fProofReset==1)
2376 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2377 if (fProofReset>1)
2378 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2379 if (!fRootVersionForProof.IsNull())
2380 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2381 else
2382 printf("= ROOT version requested________________________ default\n");
2383 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2384 if (!fAliRootMode.IsNull())
2385 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2386 if (fNproofWorkers)
2387 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2388 if (fNproofWorkersPerSlave)
2389 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2390 if (TestSpecialBit(kClearPackages))
2391 printf("= ClearPackages requested...\n");
2392 if (fIncludePath.Data())
2393 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2394 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2395 if (fPackages && fPackages->GetEntries()) {
2396 TIter next(fPackages);
2397 TObject *obj;
2398 TString list;
2399 while ((obj=next())) list += obj->GetName();
2400 printf("= Par files to be used: ________________________ %s\n", list.Data());
2401 }
2402 if (TestSpecialBit(kProofConnectGrid))
2403 printf("= Requested PROOF connection to grid\n");
2404 return;
2405 }
e1c22e21 2406 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2407 if (fOverwriteMode) {
2408 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2409 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2410 }
348be253 2411 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2412 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2413 printf("= Production mode:______________________________ %d\n", fProductionMode);
2414 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2415 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2416 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2417 if (fUser.Length())
84fcd93f 2418 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2419 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2420 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2421 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2422 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2423 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2424 if (fRunNumbers.Length())
84fcd93f 2425 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2426 if (fRunRange[0])
74013535 2427 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2428 if (!fRunRange[0] && !fRunNumbers.Length()) {
2429 TIter next(fInputFiles);
2430 TObject *obj;
2431 TString list;
2432 while ((obj=next())) list += obj->GetName();
84fcd93f 2433 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2434 }
2435 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2436 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2437 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2438 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2439 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2440 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2441 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2442 printf("=====================================================================\n");
2443 printf("= Job price: ___________________________________ %d\n", fPrice);
2444 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2445 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2446 if (fMaxInitFailed>0)
84fcd93f 2447 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2448 if (fMasterResubmitThreshold>0)
84fcd93f 2449 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2450 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2451 if (fNrunsPerMaster>0)
84fcd93f 2452 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2453 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2454 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2455 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2456 if (fArguments.Length())
84fcd93f 2457 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2458 if (fExecutableArgs.Length())
2459 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2460 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2461 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2462 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2463 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2464 if (fDatasetName)
84fcd93f 2465 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2466 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2467 if (fIncludePath.Data())
84fcd93f 2468 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2469 if (fCloseSE.Length())
84fcd93f 2470 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2471 if (fFriendChainName.Length())
84fcd93f 2472 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2473 if (fPackages && fPackages->GetEntries()) {
16a4353c 2474 TIter next(fPackages);
2475 TObject *obj;
2476 TString list;
2477 while ((obj=next())) list += obj->GetName();
84fcd93f 2478 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2479 }
2480}
2481
c57f56b7 2482//______________________________________________________________________________
2483void AliAnalysisAlien::SetDefaults()
2484{
2485// Set default values for everything. What cannot be filled will be left empty.
2486 if (fGridJDL) delete fGridJDL;
2487 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2488 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2489 fPrice = 1;
2490 fTTL = 30000;
2491 fSplitMaxInputFileNumber = 100;
2492 fMaxInitFailed = 0;
2493 fMasterResubmitThreshold = 0;
bb885a9e 2494 fNtestFiles = 10;
149d288c 2495 fNreplicas = 2;
a8739e8a 2496 fRunRange[0] = 0;
2497 fRunRange[1] = 0;
8e3bd06a 2498 fRunPrefix = "%d";
319593fb 2499 fNrunsPerMaster = 1;
16a4353c 2500 fMaxMergeFiles = 100;
c57f56b7 2501 fRunNumbers = "";
2502 fExecutable = "analysis.sh";
0a1c1f7f 2503 fExecutableCommand = "root -b -q";
c57f56b7 2504 fArguments = "";
631c0b05 2505 fExecutableArgs = "";
c57f56b7 2506 fAnalysisMacro = "myAnalysis.C";
2507 fAnalysisSource = "";
2508 fAdditionalLibs = "";
2509 fSplitMode = "se";
2510 fAPIVersion = "";
2511 fROOTVersion = "";
2512 fAliROOTVersion = "";
2513 fUser = ""; // Your alien user name
2514 fGridWorkingDir = "";
2515 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2516 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2517 fFriendChainName = "";
c57f56b7 2518 fGridOutputDir = "output";
b83f84f0 2519 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2520 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2521 fInputFormat = "xml-single";
2522 fJDLName = "analysis.jdl";
c6cb3634 2523 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2524 fMergeExcludes = "";
0f389141 2525 fMergeViaJDL = 0;
348be253 2526 SetUseCopy(kTRUE);
2527 SetCheckCopy(kTRUE);
149d288c 2528 SetDefaultOutputs(kTRUE);
e1c22e21 2529 fOverwriteMode = 1;
c57f56b7 2530}
2531
0f389141 2532//______________________________________________________________________________
b3e07543 2533Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2534{
b3e07543 2535// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2536 // First check if the result is already in the output directory.
2537 if (FileExists(Form("%s/%s",aliendir,filename))) {
2538 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2539 return kFALSE;
2540 }
b3e07543 2541 // Now check the last stage done.
2542 Int_t stage = 0;
2543 while (1) {
2544 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2545 stage++;
2546 }
2547 // Next stage of merging
2548 stage++;
2549 TString pattern = "*root_archive.zip";
2550 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2551 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2552 if (res) delete res;
2553 // Write standard output to file
2554 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2555 // Count the number of files inside
2556 ifstream ifile;
2557 ifile.open(Form("Stage_%d.xml",stage));
2558 if (!ifile.good()) {
2559 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2560 return kFALSE;
b3e07543 2561 }
2562 TString line;
2563 Int_t nfiles = 0;
2564 while (!ifile.eof()) {
2565 ifile >> line;
2566 if (line.Contains("/event")) nfiles++;
7c2cd90a 2567 }
b3e07543 2568 ifile.close();
2569 if (!nfiles) {
2570 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2571 return kFALSE;
2572 } else {
2573 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2574 }
b3e07543 2575 // Copy the file in the output directory
2576 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2577// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2578 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2579 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2580 // Check if this is the last stage to be done.
2581 Bool_t laststage = (nfiles<nperchunk);
2582 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2583 if (laststage) {
2584 printf("### Submiting final merging stage %d\n", stage);
2585 TString finalJDL = jdl;
2586 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2587 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2588 Int_t jobId = SubmitSingleJob(query);
2589 if (!jobId) return kFALSE;
2590 } else {
2591 printf("### Submiting merging stage %d\n", stage);
03335324 2592 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2593 Int_t jobId = SubmitSingleJob(query);
2594 if (!jobId) return kFALSE;
7c2cd90a 2595 }
b3e07543 2596 return kTRUE;
2597}
7c2cd90a 2598
105631b5 2599//______________________________________________________________________________
2600AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2601{
2602// Loat the analysis manager from a file.
2603 TFile *file = TFile::Open(fname);
2604 if (!file) {
2605 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2606 return 0;
2607 }
2608 TIter nextkey(file->GetListOfKeys());
2609 AliAnalysisManager *mgr = 0;
2610 TKey *key;
2611 while ((key=(TKey*)nextkey())) {
2612 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2613 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2614 }
2615 if (!mgr)
2616 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2617 return mgr;
2618}
2619
7c2cd90a 2620//______________________________________________________________________________
2621Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2622{
2623// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2624 if (!gGrid) return 0;
2625 printf("=> %s ------> ",query);
2626 TGridResult *res = gGrid->Command(query);
2627 if (!res) return 0;
2628 TString jobId = res->GetKey(0,"jobId");
2629 delete res;
2630 if (jobId.IsNull()) {
2631 printf("submission failed. Reason:\n");
2632 gGrid->Stdout();
2633 gGrid->Stderr();
2634 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2635 return 0;
2636 }
2637 printf(" Job id: %s\n", jobId.Data());
2638 return atoi(jobId);
2639}
2640
ec5acdce 2641//______________________________________________________________________________
2642Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2643{
2644// Merges a collection of output files using concatenation.
2645 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2646 if (!coll) {
2647 ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2648 return kFALSE;
2649 }
2650 // Iterate grid collection
2651 TString outtmp;
2652 Bool_t merged = kFALSE;
2653 Int_t ifile = 0;
2654 while (coll->Next()) {
2655 TString fname = gSystem->DirName(coll->GetTURL());
2656 fname += "/";
2657 fname += output;
2658 outtmp = Form("%d_%s", ifile, output);
2659 if (!TFile::Cp(fname, outtmp)) {
2660 ::Error("MergeInfo", "Could not copy %s", fname.Data());
2661 continue;
2662 }
2663 ifile++;
2664 if (ifile<2) {
2665 gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2666 continue;
2667 }
2668 gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2669 gSystem->Exec("cp tempmerged lastmerged");
2670 }
2671 if (ifile) {
2672 gSystem->Exec(Form("cp lastmerged %s", output));
2673 gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2674 merged = kTRUE;
2675 }
2676 return merged;
2677}
2678
7c2cd90a 2679//______________________________________________________________________________
b3e07543 2680Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2681{
b3e07543 2682// Merge given output files from basedir. Basedir can be an alien output directory
2683// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2684// files in a group (ignored for xml input). Merging can be done in stages:
2685// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2686// stage=1 : works with an xml of all root_archive.zip in the output directory
2687// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2688 TString outputFile = output;
0f389141 2689 TString command;
a2f5fc01 2690 TString outputChunk;
2691 TString previousChunk = "";
b3e07543 2692 TObjArray *listoffiles = new TObjArray();
2693// listoffiles->SetOwner();
a2f5fc01 2694 Int_t countChunk = 0;
2695 Int_t countZero = nmaxmerge;
0f389141 2696 Bool_t merged = kTRUE;
a2f5fc01 2697 Int_t index = outputFile.Index("@");
2698 if (index > 0) outputFile.Remove(index);
7c2cd90a 2699 TString inputFile = outputFile;
b3e07543 2700 TString sbasedir = basedir;
2701 if (sbasedir.Contains(".xml")) {
2702 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2703 nmaxmerge = 9999999;
2704 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2705 if (!coll) {
2706 ::Error("MergeOutput", "Input XML collection empty.");
2707 return kFALSE;
2708 }
2709 // Iterate grid collection
2710 while (coll->Next()) {
2711 TString fname = gSystem->DirName(coll->GetTURL());
2712 fname += "/";
2713 fname += inputFile;
2714 listoffiles->Add(new TNamed(fname.Data(),""));
2715 }
ffbe06b9 2716 } else if (sbasedir.Contains(".txt")) {
2717 // Assume lfn's on each line
2718 TString line;
2719 ifstream in;
2720 in.open(sbasedir);
2721 Int_t nfiles = 0;
2722 while (in.good()) {
2723 in >> line;
2724 if (line.IsNull() || line.BeginsWith("#")) continue;
2725 nfiles++;
2726 listoffiles->Add(new TNamed(line.Data(),""));
2727 }
2728 if (!nfiles) {
2729 ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data());
2730 delete listoffiles;
2731 return kFALSE;
2732 }
b3e07543 2733 } else {
2734 command = Form("find %s/ *%s", basedir, inputFile.Data());
2735 printf("command: %s\n", command.Data());
2736 TGridResult *res = gGrid->Command(command);
2737 if (!res) {
2738 ::Error("MergeOutput","No result for the find command\n");
2739 delete listoffiles;
2740 return kFALSE;
2741 }
2742 TIter nextmap(res);
2743 TMap *map = 0;
2744 while ((map=(TMap*)nextmap())) {
2745 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2746 if (!objs || !objs->GetString().Length()) {
2747 // Nothing found - skip this output
2748 delete res;
2749 delete listoffiles;
2750 return kFALSE;
2751 }
2752 listoffiles->Add(new TNamed(objs->GetName(),""));
2753 }
2754 delete res;
2755 }
2756 if (!listoffiles->GetEntries()) {
7c2cd90a 2757 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2758 delete listoffiles;
0f389141 2759 return kFALSE;
2760 }
2761
2762 TFileMerger *fm = 0;
b3e07543 2763 TIter next0(listoffiles);
2764 TObjArray *listoffilestmp = new TObjArray();
2765 listoffilestmp->SetOwner();
2766 TObject *nextfile;
2767 TString snextfile;
2768 // Keep only the files at upper level
2769 Int_t countChar = 0;
2770 while ((nextfile=next0())) {
2771 snextfile = nextfile->GetName();
2772 Int_t crtCount = snextfile.CountChar('/');
2773 if (nextfile == listoffiles->First()) countChar = crtCount;
2774 if (crtCount < countChar) countChar = crtCount;
2775 }
2776 next0.Reset();
2777 while ((nextfile=next0())) {
2778 snextfile = nextfile->GetName();
2779 Int_t crtCount = snextfile.CountChar('/');
2780 if (crtCount > countChar) {
2781 delete nextfile;
2782 continue;
2783 }
2784 listoffilestmp->Add(nextfile);
2785 }
2786 delete listoffiles;
2787 listoffiles = listoffilestmp; // Now contains 'good' files
2788 listoffiles->Print();
2789 TIter next(listoffiles);
7c2cd90a 2790 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2791 outputChunk = outputFile;
2792 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2793 // Check for existent temporary merge files
e1c22e21 2794 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2795 // Preserve old merging functionality for stage 0.
2796 if (stage==0) {
2797 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2798 while (1) {
2799 // Skip as many input files as in a chunk
2ea55496 2800 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2801 nextfile = next();
2802 if (!nextfile) {
2ea55496 2803 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2804 delete listoffiles;
2ea55496 2805 return kFALSE;
2806 }
b3e07543 2807 snextfile = nextfile->GetName();
7c2cd90a 2808 }
2809 outputChunk = outputFile;
2810 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2811 countChunk++;
2812 if (gSystem->AccessPathName(outputChunk)) continue;
2813 // Merged file with chunks up to <countChunk> found
2814 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2815 previousChunk = outputChunk;
2816 break;
2817 }
2818 }
2819 countZero = nmaxmerge;
2820
b3e07543 2821 while ((nextfile=next())) {
2822 snextfile = nextfile->GetName();
2ea55496 2823 // Loop 'find' results and get next LFN
7c2cd90a 2824 if (countZero == nmaxmerge) {
2825 // First file in chunk - create file merger and add previous chunk if any.
2b895f34 2826 fm = new TFileMerger(kTRUE);
7c2cd90a 2827 fm->SetFastMethod(kTRUE);
2828 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2829 outputChunk = outputFile;
2830 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2831 }
2832 // If last file found, put merged results in the output file
b3e07543 2833 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2834 // Add file to be merged and decrement chunk counter.
b3e07543 2835 fm->AddFile(snextfile);
7c2cd90a 2836 countZero--;
b3e07543 2837 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2838 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2839 // Nothing found - skip this output
2840 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2841 merged = kFALSE;
2842 break;
7c2cd90a 2843 }
2844 fm->OutputFile(outputChunk);
2845 // Merge the outputs, then go to next chunk
2846 if (!fm->Merge()) {
2847 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2848 merged = kFALSE;
2849 break;
7c2cd90a 2850 } else {
2851 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2852 gSystem->Unlink(previousChunk);
2853 }
b3e07543 2854 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2855 countChunk++;
2856 countZero = nmaxmerge;
2857 previousChunk = outputChunk;
0f389141 2858 }
0f389141 2859 }
b3e07543 2860 delete listoffiles;
2861 delete fm;
7c2cd90a 2862 return merged;
2863 }
2864 // Merging stage different than 0.
2865 // Move to the begining of the requested chunk.
2b895f34 2866 fm = new TFileMerger(kTRUE);
7c2cd90a 2867 fm->SetFastMethod(kTRUE);
b3e07543 2868 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2869 delete listoffiles;
7c2cd90a 2870 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2871 // Nothing found - skip this output
2872 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2873 delete fm;
2874 return kFALSE;
2875 }
b3e07543 2876 fm->OutputFile(outputFile);
7c2cd90a 2877 // Merge the outputs
2878 if (!fm->Merge()) {
2879 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2880 delete fm;
2881 return kFALSE;
2882 } else {
b3e07543 2883 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2884 }
7c2cd90a 2885 delete fm;
2886 return kTRUE;
0f389141 2887}
2888
c57f56b7 2889//______________________________________________________________________________
2890Bool_t AliAnalysisAlien::MergeOutputs()
2891{
2892// Merge analysis outputs existing in the AliEn space.
2893 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2894 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2895 if (!Connect()) {
2896 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2897 return kFALSE;
0f389141 2898 }
c9e8f7fd 2899 if (fMergeViaJDL) {
2900 if (!TestBit(AliAnalysisGrid::kMerge)) {
2901 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2902 return kFALSE;
2903 }
2904 if (fProductionMode) {
2905 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2906 return kFALSE;
2907 }
0f389141 2908 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2909 if (!SubmitMerging()) return kFALSE;
0f389141 2910 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2911 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2912 return kFALSE;
c57f56b7 2913 }
2914 // Get the output path
dc030a24 2915 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2916 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2917 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2918 return kFALSE;
2919 }
2920 if (!fOutputFiles.Length()) {
2921 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2922 return kFALSE;
0f389141 2923 }
2924 // Check if fast read option was requested
e1c22e21 2925 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2926 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2927 if (fFastReadOption) {
d3339be3 2928 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2929 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2930 gEnv->SetValue("XNet.ConnectTimeout",50);
2931 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2932 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2933 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2934 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2935 }
e8b839ab 2936 // Make sure we change the temporary directory
2937 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2938 // Set temporary compilation directory to current one
2939 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2940 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2941 TIter next(list);
2942 TObjString *str;
a2f5fc01 2943 TString outputFile;
c57f56b7 2944 Bool_t merged = kTRUE;
2945 while((str=(TObjString*)next())) {
a2f5fc01 2946 outputFile = str->GetString();
2947 Int_t index = outputFile.Index("@");
2948 if (index > 0) outputFile.Remove(index);
2949 TString outputChunk = outputFile;
2950 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2951 // Skip already merged outputs
a2f5fc01 2952 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2953 if (fOverwriteMode) {
a2f5fc01 2954 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2955 gSystem->Unlink(outputFile);
2956 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2957 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2958 outputChunk.Data());
2959 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2960 }
2961 } else {
a2f5fc01 2962 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2963 continue;
2964 }
2965 } else {
a2f5fc01 2966 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2967 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2968 outputChunk.Data());
2969 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2970 }
2971 }
37d21c01 2972 if (fMergeExcludes.Contains(outputFile.Data()) ||
2973 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 2974 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2975 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2976 if (!merged) {
2977 Error("MergeOutputs", "Terminate() will NOT be executed");
2978 return kFALSE;
ff07ec61 2979 }
2980 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2981 if (fileOpened) fileOpened->Close();
c57f56b7 2982 }
0f389141 2983 return kTRUE;
c57f56b7 2984}
2985
bb885a9e 2986//______________________________________________________________________________
2987void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2988{
2989// Use the output files connected to output containers from the analysis manager
2990// rather than the files defined by SetOutputFiles
2991 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2992 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2993 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2994}
2995
149d288c 2996//______________________________________________________________________________
2997void AliAnalysisAlien::SetOutputFiles(const char *list)
2998{
2999// Manually set the output files list.
3000// Removes duplicates. Not allowed if default outputs are not disabled.
3001 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3002 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
3003 return;
3004 }
3005 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
3006 fOutputFiles = "";
3007 TString slist = list;
3008 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
3009 TObjArray *arr = slist.Tokenize(" ");
3010 TObjString *os;
3011 TIter next(arr);
3012 TString sout;
3013 while ((os=(TObjString*)next())) {
3014 sout = os->GetString();
3015 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
3016 if (fOutputFiles.Contains(sout)) continue;
3017 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
3018 fOutputFiles += sout;
3019 }
3020 delete arr;
f790bc1b 3021}
149d288c 3022
3023//______________________________________________________________________________
3024void AliAnalysisAlien::SetOutputArchive(const char *list)
3025{
3026// Manually set the output archive list. Free text - you are on your own...
3027// Not allowed if default outputs are not disabled.
3028 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3029 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
3030 return;
3031 }
3032 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
3033 fOutputArchive = list;
3034}
3035
3036//______________________________________________________________________________
3037void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
3038{
3039// Setting a prefered output SE is not allowed anymore.
3040 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
3041}
3042
28ce6c9c 3043//______________________________________________________________________________
3044void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
3045{
3046// Set some PROOF special parameter.
3047 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3048 if (pair) {
3049 TObject *old = pair->Key();
3050 TObject *val = pair->Value();
3051 fProofParam.Remove(old);
3052 delete old;
3053 delete val;
3054 }
3055 fProofParam.Add(new TObjString(pname), new TObjString(value));
3056}
3057
3058//______________________________________________________________________________
3059const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
3060{
3061// Returns a special PROOF parameter.
3062 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3063 if (!pair) return 0;
3064 return pair->Value()->GetName();
3065}
3066
5513444a 3067//______________________________________________________________________________
3068Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 3069{
3070// Start remote grid analysis.
3bdcb562 3071 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3072 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
3073 if (!mgr || !mgr->IsInitialized()) {
3074 Error("StartAnalysis", "You need an initialized analysis manager for this");
3075 return kFALSE;
3076 }
3077 // Are we in PROOF mode ?
3078 if (mgr->IsProofMode()) {
baa34407 3079 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3080 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3081 if (fProofCluster.IsNull()) {
3082 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3083 return kFALSE;
3084 }
3085 if (fProofDataSet.IsNull() && !testMode) {
3086 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3087 return kFALSE;
3088 }
3089 // Set the needed environment
3090 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3091 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3092 if (fProofReset && !testMode) {
3093 if (fProofReset==1) {
3094 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3095 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3096 } else {
3097 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3098 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3099 }
3100 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3101 return kFALSE;
3102 }
baa34407 3103
3104 if (!testMode) {
3105 // Check if there is an old active session
3106 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3107 if (nsessions) {
3108 Error("StartAnalysis","You have to reset your old session first\n");
3109 return kFALSE;
3110 }
53911ca6 3111 }
3bdcb562 3112 // Do we need to change the ROOT version ? The success of this cannot be checked.
3113 if (!fRootVersionForProof.IsNull() && !testMode) {
3114 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
3115 fProofCluster.Data(), fRootVersionForProof.Data()));
3116 }
3117 // Connect to PROOF and check the status
3118 Long_t proof = 0;
27734f0e 3119 TString sworkers;
3120 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3121 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3122 if (!testMode) {
27734f0e 3123 if (!sworkers.IsNull())
3124 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3125 else
3126 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3127 } else {
3128 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3129 if (!proof) {
3130 Error("StartAnalysis", "Could not start PROOF in test mode");
3131 return kFALSE;
3132 }
3133 }
3134 if (!proof) {
3135 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3136 return kFALSE;
3137 }
27734f0e 3138 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3139 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3140 // Set proof special parameters if any
3141 TIter nextpp(&fProofParam);
3142 TObject *proofparam;
3143 while ((proofparam=nextpp())) {
3144 TString svalue = GetProofParameter(proofparam->GetName());
3145 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3146 }
3bdcb562 3147 // Is dataset existing ?
3148 if (!testMode) {
3149 TString dataset = fProofDataSet;
3150 Int_t index = dataset.Index("#");
3151 if (index>=0) dataset.Remove(index);
830acc4c 3152// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3153// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3154// return kFALSE;
3155// }
3156// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3157 }
3158 // Is ClearPackages() needed ?
3159 if (TestSpecialBit(kClearPackages)) {
3160 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3161 gROOT->ProcessLine("gProof->ClearPackages();");
3162 }
3163 // Is a given aliroot mode requested ?
3164 TList optionsList;
1ac846e9 3165 TString parLibs;
830acc4c 3166 if (!fAliRootMode.IsNull()) {
3bdcb562 3167 TString alirootMode = fAliRootMode;
3168 if (alirootMode == "default") alirootMode = "";
3169 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3170 optionsList.SetOwner();
3171 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3172 // Check the additional libs to be loaded
3173 TString extraLibs;
1ac846e9 3174 Bool_t parMode = kFALSE;
dc1ffbcb 3175 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3176 // Parse the extra libs for .so
3177 if (fAdditionalLibs.Length()) {
3178 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3179 TIter next(list);
3180 TObjString *str;
1ac846e9 3181 while((str=(TObjString*)next())) {
3182 if (str->GetString().Contains(".so")) {
3183 if (parMode) {
3184 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3185 break;
3186 }
3187 TString stmp = str->GetName();
3188 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3189 stmp.ReplaceAll(".so","");
3190 if (!extraLibs.IsNull()) extraLibs += ":";
3191 extraLibs += stmp;
3192 continue;
3193 }
3194 if (str->GetString().Contains(".par")) {
3195 // The first par file found in the list will not allow any further .so
3196 parMode = kTRUE;
3197 if (!parLibs.IsNull()) parLibs += ":";
3198 parLibs += str->GetName();
3199 continue;
3200 }
3bdcb562 3201 }
3202 if (list) delete list;
3203 }
dc030a24 3204 if (!extraLibs.IsNull()) {
3205 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3206 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3207 }
830acc4c 3208 // Check extra includes
3209 if (!fIncludePath.IsNull()) {
3210 TString includePath = fIncludePath;
3211 includePath.ReplaceAll(" ",":");
dc030a24 3212 includePath.ReplaceAll("$ALICE_ROOT/","");
3213 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3214 includePath.ReplaceAll("-I","");
dc030a24 3215 includePath.Remove(TString::kTrailing, ':');
830acc4c 3216 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3217 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3218 }
27734f0e 3219 // Check if connection to grid is requested
3220 if (TestSpecialBit(kProofConnectGrid))
3221 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3222 // Enable AliRoot par
3223 if (testMode) {
3224 // Enable proof lite package
3225 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3226 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3227 TNamed *obj = (TNamed*)optionsList.At(i);
3228 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3229 }
3230 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3231 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3232 Info("StartAnalysis", "AliRootProofLite enabled");
3233 } else {
3234 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3235 return kFALSE;
3236 }
3237 } else {
dc030a24 3238 if ( ! fAliROOTVersion.IsNull() ) {
3239 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3240 fAliROOTVersion.Data(), &optionsList))) {
3241 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3242 return kFALSE;
3243 }
3244 }
27734f0e 3245 }
1ac846e9 3246 // Enable first par files from fAdditionalLibs
3247 if (!parLibs.IsNull()) {
3248 TObjArray *list = parLibs.Tokenize(":");
3249 TIter next(list);
3250 TObjString *package;
3251 while((package=(TObjString*)next())) {
40501bb7 3252 TString spkg = package->GetName();
3253 spkg.ReplaceAll(".par", "");
3254 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3255 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3256 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3257 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3258 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3259 return kFALSE;
3260 }
3261 } else {
3262 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3263 return kFALSE;
3264 }
3265 }
3266 if (list) delete list;
3267 }
3bdcb562 3268 } else {
3269 if (fAdditionalLibs.Contains(".so") && !testMode) {
3270 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3271 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3272 return kFALSE;
3273 }
3274 }
3275 // Enable par files if requested
3276 if (fPackages && fPackages->GetEntries()) {
3277 TIter next(fPackages);
3278 TObject *package;
3279 while ((package=next())) {
1ac846e9 3280 // Skip packages already enabled
3281 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3282 TString spkg = package->GetName();
3283 spkg.ReplaceAll(".par", "");
3284 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3285 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3286 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3287 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3288 return kFALSE;
3289 }
3290 } else {
3291 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3292 return kFALSE;
3293 }
3294 }
3295 }
3296 // Do we need to load analysis source files ?
3297 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3298 if (fAnalysisSource.Length()) {
3299 TObjArray *list = fAnalysisSource.Tokenize(" ");
3300 TIter next(list);
3301 TObjString *str;
3302 while((str=(TObjString*)next())) {
3303 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3304 }
3bdcb562 3305 if (list) delete list;
3306 }
830acc4c 3307 if (testMode) {
3308 // Register dataset to proof lite.
3309 if (fFileForTestMode.IsNull()) {
3310 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3311 return kFALSE;
3312 }
3313 if (gSystem->AccessPathName(fFileForTestMode)) {
3314 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3315 return kFALSE;
3316 }
3317 TFileCollection *coll = new TFileCollection();
3318 coll->AddFromFile(fFileForTestMode);
8ddad121 3319 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3320 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3321 }
3bdcb562 3322 return kTRUE;
3323 }
c57f56b7 3324
43da816a 3325 // Check if output files have to be taken from the analysis manager
3326 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3327 // Add output files and AOD files
3328 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3329 // Add extra files registered to the analysis manager
f790bc1b 3330 TString extra = GetListOfFiles("ext");
3331 if (!extra.IsNull()) {
149d288c 3332 extra.ReplaceAll(".root", "*.root");
f790bc1b 3333 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3334 fOutputFiles += extra;
84fcd93f 3335 }
149d288c 3336 // Compose the output archive.
b83f84f0 3337 fOutputArchive = "log_archive.zip:std*@disk=1 ";
ec5acdce 3338 if (mgr->IsCollectThroughput())
3339 fOutputArchive += Form("root_archive.zip:%s,*.stat,*%s@disk=%d",fOutputFiles.Data(),mgr->GetFileInfoLog(),fNreplicas);
3340 else
3341 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3342 }
f7b1cbc2 3343// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3344 if (TestBit(AliAnalysisGrid::kOffline)) {
3345 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3346 \n there nor any job run. You can revise the JDL and analysis \
3347 \n macro then run the same in \"submit\" mode.");
3348 } else if (TestBit(AliAnalysisGrid::kTest)) {
3349 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3350 \n dataset.");
3351 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3352 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3353 \n space and job submitted.");
3354 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3355 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3356 if (fMergeViaJDL) CheckInputData();
5513444a 3357 return kTRUE;
c57f56b7 3358 } else {
3359 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3360 }
3361
348be253 3362 Print();
c57f56b7 3363 if (!Connect()) {
3364 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3365 return kFALSE;
16a4353c 3366 }
d3b18c4c 3367 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3368 if (!CheckInputData()) {
3369 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3370 return kFALSE;
c57f56b7 3371 }
d3339be3 3372 if (!CreateDataset(fDataPattern)) {
3373 TString serror;
3374 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3375 if (fRunNumbers.Length()) serror = "run numbers";
3376 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3377 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3378 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3379 return kFALSE;
3380 }
21ca8e59 3381 WriteAnalysisFile();
c57f56b7 3382 WriteAnalysisMacro();
3383 WriteExecutable();
3384 WriteValidationScript();
0f389141 3385 if (fMergeViaJDL) {
3386 WriteMergingMacro();
3387 WriteMergeExecutable();
3388 WriteValidationScript(kTRUE);
3389 }
5513444a 3390 if (!CreateJDL()) return kFALSE;
3391 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3392 if (testMode) {
c57f56b7 3393 // Locally testing the analysis
3394 Info("StartAnalysis", "\n_______________________________________________________________________ \
3395 \n Running analysis script in a daughter shell as on a worker node \
3396 \n_______________________________________________________________________");
149d288c 3397 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3398 TIter next(list);
3399 TObjString *str;
a2f5fc01 3400 TString outputFile;
c57f56b7 3401 while((str=(TObjString*)next())) {
a2f5fc01 3402 outputFile = str->GetString();
3403 Int_t index = outputFile.Index("@");
3404 if (index > 0) outputFile.Remove(index);
3405 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3406 }
3407 delete list;
3408 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3409 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3410// gSystem->Exec("cat stdout");
5513444a 3411 return kFALSE;
c57f56b7 3412 }
5513444a 3413 // Check if submitting is managed by LPM manager
a3e84053 3414 if (fProductionMode) {
7ac61826 3415 //TString prodfile = fJDLName;
3416 //prodfile.ReplaceAll(".jdl", ".prod");
3417 //WriteProductionFile(prodfile);
5513444a 3418 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3419 return kFALSE;
3420 }
a8739e8a 3421 // Submit AliEn job(s)
d2a409b2 3422 gGrid->Cd(fGridOutputDir);
a8739e8a 3423 TGridResult *res;
c57f56b7 3424 TString jobID = "";
d2a409b2 3425 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3426 // Submit a given xml or a set of runs
a8739e8a 3427 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3428 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3429 if (res) {
3430 const char *cjobId = res->GetKey(0,"jobId");
3431 if (!cjobId) {
a03be957 3432 gGrid->Stdout();
3433 gGrid->Stderr();
a8739e8a 3434 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3435 return kFALSE;
a8739e8a 3436 } else {
3437 Info("StartAnalysis", "\n_______________________________________________________________________ \
3438 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3439 \n_______________________________________________________________________",
3440 fJDLName.Data(), cjobId);
3441 jobID = cjobId;
3442 }
3443 delete res;
a03be957 3444 } else {
3445 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3446 return kFALSE;
a8739e8a 3447 }
3448 } else {
d2a409b2 3449 // Submit for a range of enumeration of runs.
a03be957 3450 if (!Submit()) return kFALSE;
c57f56b7 3451 }
a8739e8a 3452
c57f56b7 3453 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3454 \n You may exit at any time and terminate the job later using the option <terminate> \
3455 \n ##################################################################################", jobID.Data());
bb885a9e 3456 gSystem->Exec("aliensh");
5513444a 3457 return kTRUE;
c57f56b7 3458}
3459
f790bc1b 3460//______________________________________________________________________________
3461const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3462{
3463// Get a comma-separated list of output files of the requested type.
3464// Type can be (case unsensitive):
3465// aod - list of aod files (std, extensions and filters)
3466// out - list of output files connected to containers (but not aod's or extras)
3467// ext - list of extra files registered to the manager
3468// ter - list of files produced in terminate
3469 static TString files;
3470 files = "";
3471 TString stype = type;
3472 stype.ToLower();
3473 TString aodfiles, extra;
3474 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3475 if (!mgr) {
3476 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3477 return files.Data();
3478 }
3479 if (mgr->GetOutputEventHandler()) {
3480 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3481 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3482 if (!extraaod.IsNull()) {
3483 aodfiles += ",";
3484 aodfiles += extraaod;
3485 }
3486 }
3487 if (stype.Contains("aod")) {
3488 files = aodfiles;
3489 if (stype == "aod") return files.Data();
3490 }
3491 // Add output files that are not in the list of AOD files
3492 TString outputfiles = "";
3493 TIter next(mgr->GetOutputs());
3494 AliAnalysisDataContainer *output;
3495 const char *filename = 0;
3496 while ((output=(AliAnalysisDataContainer*)next())) {
3497 filename = output->GetFileName();
3498 if (!(strcmp(filename, "default"))) continue;
3499 if (outputfiles.Contains(filename)) continue;
3500 if (aodfiles.Contains(filename)) continue;
3501 if (!outputfiles.IsNull()) outputfiles += ",";
3502 outputfiles += filename;
3503 }
3504 if (stype.Contains("out")) {
3505 if (!files.IsNull()) files += ",";
3506 files += outputfiles;
3507 if (stype == "out") return files.Data();
3508 }
3509 // Add extra files registered to the analysis manager
3510 TString sextra;
3511 extra = mgr->GetExtraFiles();
3512 if (!extra.IsNull()) {
3513 extra.Strip();
3514 extra.ReplaceAll(" ", ",");
3515 TObjArray *fextra = extra.Tokenize(",");
3516 TIter nextx(fextra);
3517 TObject *obj;
3518 while ((obj=nextx())) {
3519 if (aodfiles.Contains(obj->GetName())) continue;
3520 if (outputfiles.Contains(obj->GetName())) continue;
3521 if (sextra.Contains(obj->GetName())) continue;
3522 if (!sextra.IsNull()) sextra += ",";
3523 sextra += obj->GetName();
3524 }
3525 delete fextra;
3526 if (stype.Contains("ext")) {
3527 if (!files.IsNull()) files += ",";
3528 files += sextra;
3529 }
3530 }
3531 if (stype == "ext") return files.Data();
3532 TString termfiles;
3533 if (!fTerminateFiles.IsNull()) {
3534 fTerminateFiles.Strip();
3535 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3536 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3537 TIter nextx(fextra);
3538 TObject *obj;
3539 while ((obj=nextx())) {
3540 if (aodfiles.Contains(obj->GetName())) continue;
3541 if (outputfiles.Contains(obj->GetName())) continue;
3542 if (termfiles.Contains(obj->GetName())) continue;
3543 if (sextra.Contains(obj->GetName())) continue;
3544 if (!termfiles.IsNull()) termfiles += ",";
3545 termfiles += obj->GetName();
3546 }
3547 delete fextra;
3548 }
3549 if (stype.Contains("ter")) {
3550 if (!files.IsNull() && !termfiles.IsNull()) {
3551 files += ",";
3552 files += termfiles;
3553 }
3554 }
3555 return files.Data();
3556}
3557
d2a409b2 3558//______________________________________________________________________________
a03be957 3559Bool_t AliAnalysisAlien::Submit()
d2a409b2 3560{
3561// Submit all master jobs.
3562 Int_t nmasterjobs = fInputFiles->GetEntries();
3563 Long_t tshoot = gSystem->Now();
a03be957 3564 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3565 while (fNsubmitted < nmasterjobs) {
3566 Long_t now = gSystem->Now();
3567 if ((now-tshoot)>30000) {
3568 tshoot = now;
a03be957 3569 if (!SubmitNext()) return kFALSE;
d2a409b2 3570 }
3571 }
a03be957 3572 return kTRUE;
d2a409b2 3573}
3574
0f389141 3575//______________________________________________________________________________
a03be957 3576Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3577{
3578// Submit all merging jobs.
dc030a24 3579 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3580 gGrid->Cd(fGridOutputDir);
3581 TString mergeJDLName = fExecutable;
3582 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3583 if (!fInputFiles) {
3584 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3585 return kFALSE;
3586 }
0f389141 3587 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3588 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3589 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3590 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3591 if (fOutputToRunNo) {
3592 // The output directory is the run number
3593 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3594 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3595 } else {
17b4e9a4 3596 if (!fRunNumbers.Length() && !fRunRange[0]) {
3597 // The output directory is the grid outdir
3598 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3599 runOutDir = fGridOutputDir;
3600 } else {
3601 // The output directory is the master number in 3 digits format
3602 printf("### Submitting merging job for master <%03d>\n", i);
3603 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3604 }
7c2cd90a 3605 }
3606 // Check now the number of merging stages.
2b222f7c 3607 TObjArray *list = fOutputFiles.Tokenize(",");
3608 TIter next(list);
3609 TObjString *str;
3610 TString outputFile;
3611 while((str=(TObjString*)next())) {
3612 outputFile = str->GetString();
3613 Int_t index = outputFile.Index("@");
3614 if (index > 0) outputFile.Remove(index);
37d21c01 3615 if (!fMergeExcludes.Contains(outputFile) &&
3616 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3617 }
3618 delete list;
b3e07543 3619 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3620 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3621 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3622 }
a03be957 3623 if (!ntosubmit) return kTRUE;
9f020952 3624 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3625 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3626 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3627 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3628 \n ################################################################################################################");
0f389141 3629 gSystem->Exec("aliensh");
a03be957 3630 return kTRUE;
0f389141 3631}
3632
d2a409b2 3633//______________________________________________________________________________
a03be957 3634Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3635{
08d5b699 3636// Submit next bunch of master jobs if the queue is free. The first master job is
3637// submitted right away, while the next will not be unless the previous was split.
3638// The plugin will not submit new master jobs if there are more that 500 jobs in
3639// waiting phase.
d2a409b2 3640 static Bool_t iscalled = kFALSE;
3641 static Int_t firstmaster = 0;
3642 static Int_t lastmaster = 0;
3643 static Int_t npermaster = 0;
a03be957 3644 if (iscalled) return kTRUE;
d2a409b2 3645 iscalled = kTRUE;
3646 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3647 Int_t ntosubmit = 0;
3648 TGridResult *res;
3649 TString jobID = "";
3bdcb562 3650 Int_t nmasterjobs = fInputFiles->GetEntries();
3651 if (!fNsubmitted) {
3652 ntosubmit = 1;
3653 if (!IsUseSubmitPolicy()) {
90d50a8c 3654 if (nmasterjobs>5)
3bdcb562 3655 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3656 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3657 ntosubmit = nmasterjobs;
3658 }
3659 } else {
d2a409b2 3660 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3661 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3662 // If last master not split, just return
a03be957 3663 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3664 // No more than 100 waiting jobs
08d5b699 3665 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3666 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3667 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3668 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3669 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3670 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3671 }
d2a409b2 3672 for (Int_t i=0; i<ntosubmit; i++) {
3673 // Submit for a range of enumeration of runs.
a03be957 3674 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3675 TString query;
cd11251e 3676 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3677 runOutDir.ReplaceAll(".xml", "");
3678 if (fOutputToRunNo)
3679 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3680 else
3681 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3682 printf("********* %s\n",query.Data());
d2a409b2 3683 res = gGrid->Command(query);
3684 if (res) {
98ca124f 3685 TString cjobId1 = res->GetKey(0,"jobId");
3686 if (!cjobId1.Length()) {
d2a409b2 3687 iscalled = kFALSE;
a03be957 3688 gGrid->Stdout();
3689 gGrid->Stderr();
3690 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3691 return kFALSE;
d2a409b2 3692 } else {
3693 Info("StartAnalysis", "\n_______________________________________________________________________ \
3694 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3695 \n_______________________________________________________________________",
98ca124f 3696 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 3697 jobID += cjobId1;
3698 jobID += " ";
98ca124f 3699 lastmaster = cjobId1.Atoi();
d2a409b2 3700 if (!firstmaster) firstmaster = lastmaster;
3701 fNsubmitted++;
3702 }
3703 delete res;
a03be957 3704 } else {
3705 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3706 return kFALSE;
d2a409b2 3707 }
3708 }
3709 iscalled = kFALSE;
a03be957 3710 return kTRUE;
d2a409b2 3711}
3712
c57f56b7 3713//______________________________________________________________________________
3714void AliAnalysisAlien::WriteAnalysisFile()
3715{
f10e8481 3716// Write current analysis manager into the file <analysisFile>
3717 TString analysisFile = fExecutable;
3718 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3719 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3720 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3721 if (!mgr || !mgr->IsInitialized()) {
3722 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3723 return;
3724 }
3725 // Check analysis type
3726 TObject *handler;
3727 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3728 handler = (TObject*)mgr->GetInputEventHandler();
3729 if (handler) {
65eb22f8 3730 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3731 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3732 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3733 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3734 } else {
3735 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3736 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3737 }
c57f56b7 3738 }
3739 TDirectory *cdir = gDirectory;
f10e8481 3740 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3741 if (file) {
e16a394c 3742 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3743 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3744 // Unless merging makes no sense
3745 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3746 mgr->Write();
3747 delete file;
fe2d7fc2 3748 // Enable termination for local jobs
3749 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3750 }
3751 if (cdir) cdir->cd();
f10e8481 3752 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3753 }
3754 Bool_t copy = kTRUE;
d3b18c4c 3755 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3756 if (copy) {
3757 CdWork();
3758 TString workdir = gGrid->GetHomeDirectory();
3759 workdir += fGridWorkingDir;
d3b18c4c 3760 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3761 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3762 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3763 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3764 }
3765}
3766
3767//______________________________________________________________________________
3768void AliAnalysisAlien::WriteAnalysisMacro()
3769{
3770// Write the analysis macro that will steer the analysis in grid mode.
3771 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3772 ofstream out;
3773 out.open(fAnalysisMacro.Data(), ios::out);
3774 if (!out.good()) {
3775 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3776 return;
3777 }
5b273635 3778 Bool_t hasSTEERBase = kFALSE;
3779 Bool_t hasESD = kFALSE;
3780 Bool_t hasAOD = kFALSE;
3781 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3782 Bool_t hasOADB = kFALSE;
5b273635 3783 Bool_t hasANALYSISalice = kFALSE;
3784 Bool_t hasCORRFW = kFALSE;
c57f56b7 3785 TString func = fAnalysisMacro;
3786 TString type = "ESD";
3787 TString comment = "// Analysis using ";
c85cfc0f 3788 if (IsUseMCchain()) {
3789 type = "MC";
3790 comment += "MC";
3791 } else {
3792 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3793 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3794 type = "AOD";
3795 comment += "AOD";
3796 }
3797 }
0df6ccf2 3798 if (type!="AOD" && fFriendChainName!="") {
3799 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3800 return;
3801 }
c57f56b7 3802 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3803 else comment += " data";
3804 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3805 func.ReplaceAll(".C", "");
3806 out << "void " << func.Data() << "()" << endl;
3807 out << "{" << endl;
3808 out << comment.Data() << endl;
3809 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3810 out << " TStopwatch timer;" << endl;
3811 out << " timer.Start();" << endl << endl;
13f0c00b 3812 // Change temp directory to current one
fdee57f1 3813 if (!IsLocalTest()) {
3814 out << "// connect to AliEn and make the chain" << endl;
3815 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
3816 }
13f0c00b 3817 out << "// Set temporary merging directory to current one" << endl;
3818 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3819 out << "// Set temporary compilation directory to current one" << endl;
3820 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3821 // Reset existing include path
3822 out << "// Reset existing include path and add current directory first in the search" << endl;
3823 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3824 if (!fExecutableCommand.Contains("aliroot")) {
3825 out << "// load base root libraries" << endl;
3826 out << " gSystem->Load(\"libTree\");" << endl;
3827 out << " gSystem->Load(\"libGeom\");" << endl;
3828 out << " gSystem->Load(\"libVMC\");" << endl;
3829 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3830 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3831 }
d5c6455a 3832 if (fAdditionalRootLibs.Length()) {
47a4137d 3833 // in principle libtree /lib geom libvmc etc. can go into this list, too
3834 out << "// Add aditional libraries" << endl;
3835 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3836 TIter next(list);
3837 TObjString *str;
3838 while((str=(TObjString*)next())) {
3839 if (str->GetString().Contains(".so"))
3840 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3841 }
47a4137d 3842 if (list) delete list;
d5c6455a 3843 }
57377eb5 3844 out << "// Load analysis framework libraries" << endl;
652561d5 3845 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3846 if (!fPackages) {
f790bc1b 3847 if (!fExecutableCommand.Contains("aliroot")) {
3848 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3849 out << " gSystem->Load(\"libESD\");" << endl;
3850 out << " gSystem->Load(\"libAOD\");" << endl;
3851 }
4e5c5506 3852 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3853 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3854 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3855 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3856 } else {
4e5c5506 3857 TIter next(fPackages);
3858 TObject *obj;
57377eb5 3859 TString pkgname;
57377eb5 3860 while ((obj=next())) {
3861 pkgname = obj->GetName();
4478e6f1 3862 if (pkgname == "STEERBase" ||
3863 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3864 if (pkgname == "ESD" ||
3865 pkgname == "ESD.par") hasESD = kTRUE;
3866 if (pkgname == "AOD" ||
3867 pkgname == "AOD.par") hasAOD = kTRUE;
3868 if (pkgname == "ANALYSIS" ||
3869 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 3870 if (pkgname == "OADB" ||
3871 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 3872 if (pkgname == "ANALYSISalice" ||
3873 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3874 if (pkgname == "CORRFW" ||
3875 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3876 }
3877 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3878 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3879 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3880 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3881 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3882 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3883 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3884 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3885 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 3886 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
3887 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 3888 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3889 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3890 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3891 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3892 out << "// Compile other par packages" << endl;
3893 next.Reset();
fcc9bb6f 3894 while ((obj=next())) {
3895 pkgname = obj->GetName();
4478e6f1 3896 if (pkgname == "STEERBase" ||
3897 pkgname == "STEERBase.par" ||
3898 pkgname == "ESD" ||
3899 pkgname == "ESD.par" ||
3900 pkgname == "AOD" ||
3901 pkgname == "AOD.par" ||
3902 pkgname == "ANALYSIS" ||
3903 pkgname == "ANALYSIS.par" ||
7ea40ac8 3904 pkgname == "OADB" ||
3905 pkgname == "OADB.par" ||
4478e6f1 3906 pkgname == "ANALYSISalice" ||
3907 pkgname == "ANALYSISalice.par" ||
3908 pkgname == "CORRFW" ||
3909 pkgname == "CORRFW.par") continue;
5b273635 3910 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3911 }
4e5c5506 3912 }
b3e07543 3913 out << "// include path" << endl;
3914 // Get the include path from the interpreter and remove entries pointing to AliRoot
3915 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3916 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3917 out << " TIter nextpath(listpaths);" << endl;
3918 out << " TObjString *pname;" << endl;
3919 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3920 out << " TString current = pname->GetName();" << endl;
3921 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3922 out << " gSystem->AddIncludePath(current);" << endl;
3923 out << " }" << endl;
3924 out << " if (listpaths) delete listpaths;" << endl;
3925 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3926 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3927 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3928 if (fAdditionalLibs.Length()) {
3929 out << "// Add aditional AliRoot libraries" << endl;
3930 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3931 TIter next(list);
3932 TObjString *str;
3933 while((str=(TObjString*)next())) {
3934 if (str->GetString().Contains(".so"))
3935 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3936 if (str->GetString().Contains(".par"))
3937 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3938 }
3939 if (list) delete list;
3940 }
3941 out << endl;
c57f56b7 3942 out << "// analysis source to be compiled at runtime (if any)" << endl;
3943 if (fAnalysisSource.Length()) {
3944 TObjArray *list = fAnalysisSource.Tokenize(" ");
3945 TIter next(list);
3946 TObjString *str;
3947 while((str=(TObjString*)next())) {
3948 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3949 }
3950 if (list) delete list;
3951 }
3952 out << endl;
b3e07543 3953// out << " printf(\"Currently load libraries:\\n\");" << endl;
3954// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3955 if (fFastReadOption) {
d3339be3 3956 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3957 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3958 out << "// fast xrootd reading enabled" << endl;
3959 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3960 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3961 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3962 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3963 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3964 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 3965 }
c57f56b7 3966 out << "// read the analysis manager from file" << endl;
f10e8481 3967 TString analysisFile = fExecutable;
3968 analysisFile.ReplaceAll(".sh", ".root");
105631b5 3969 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
3970 << analysisFile << "\");" << endl;
3971 out << " if (!mgr) return;" << endl;
21ca8e59 3972 if (IsLocalTest()) {
3973 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
3974 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 3975 if (fFileForTestMode.IsNull())
3976 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
3977 else
3978 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 3979 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4b5d9e21 3980 if (!fFriendChainName.IsNull())
3981 out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\");" << endl;
024dfa52 3982 if (IsUseMCchain())
3983 out << " plugin->SetUseMCchain();" << endl;
21ca8e59 3984 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 3985 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 3986 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 3987 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
3988 } else {
7a366e0b 3989 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 3990 out << " mgr->SetNSysInfo(100);" << endl;
3991 }
21ca8e59 3992 }
c57f56b7 3993 out << " mgr->PrintStatus();" << endl;
52b6a92b 3994 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3995 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3996 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3997 } else {
9a8288ff 3998 if (TestBit(AliAnalysisGrid::kTest))
3999 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4000 else
4001 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 4002 }
4003 }
21ca8e59 4004 if (!IsLocalTest()) {
242accb2 4005 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 4006 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
4007 } else {
4008 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 4009 }
f7498086 4010 out << " timer.Stop();" << endl;
4011 out << " timer.Print();" << endl;
c57f56b7 4012 out << "}" << endl << endl;
21ca8e59 4013 if (!IsLocalTest()) {
fcc9bb6f 4014 out <<"//________________________________________________________________________________" << endl;
c57f56b7 4015 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
4016 out << "{" << endl;
4017 out << "// Create a chain using url's from xml file" << endl;
242accb2 4018 out << " TString filename;" << endl;
4019 out << " Int_t run = 0;" << endl;
c85cfc0f 4020 if (IsUseMCchain()) {
4021 out << " TString treename = \"TE\";" << endl;
4022 } else {
4023 out << " TString treename = type;" << endl;
4024 out << " treename.ToLower();" << endl;
4025 out << " treename += \"Tree\";" << endl;
4026 }
e02fee64 4027 out << " printf(\"***************************************\\n\");" << endl;
4028 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
4029 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 4030 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
4031 out << " if (!coll) {" << endl;
4032 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
4033 out << " return NULL;" << endl;
4034 out << " }" << endl;
242accb2 4035 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 4036 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 4037 if(fFriendChainName!="") {
4038 out << " TChain *chainFriend = new TChain(treename);" << endl;
4039 }
c57f56b7 4040 out << " coll->Reset();" << endl;
0df6ccf2 4041 out << " while (coll->Next()) {" << endl;
242accb2 4042 out << " filename = coll->GetTURL("");" << endl;
4043 out << " if (mgr) {" << endl;
4044 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
4045 out << " if (nrun && nrun != run) {" << endl;
4046 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
4047 out << " mgr->SetRunFromPath(nrun);" << endl;
4048 out << " run = nrun;" << endl;
4049 out << " }" << endl;
4050 out << " }" << endl;
4051 out << " chain->Add(filename);" << endl;
0df6ccf2 4052 if(fFriendChainName!="") {
4053 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
4b5d9e21 4054 out << " if (fileFriend.Index(\"#\") > -1) fileFriend.Remove(fileFriend.Index(\"#\"));" << endl;
4055 out << " fileFriend = gSystem->DirName(fileFriend);" << endl;
4056 out << " fileFriend += \"/\";" << endl;
4057 out << " fileFriend += \"" << fFriendChainName << "\";";
4058 out << " TFile *file = TFile::Open(fileFriend);" << endl;
4059 out << " if (file) {" << endl;
4060 out << " file->Close();" << endl;
4061 out << " chainFriend->Add(fileFriend.Data());" << endl;
4062 out << " } else {" << endl;
4063 out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
4064 out << " return 0;" << endl;
4065 out << " }" << endl;
0df6ccf2 4066 }
4067 out << " }" << endl;
c57f56b7 4068 out << " if (!chain->GetNtrees()) {" << endl;
4069 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
4070 out << " return NULL;" << endl;
4071 out << " }" << endl;
0df6ccf2 4072 if(fFriendChainName!="") {
4073 out << " chain->AddFriend(chainFriend);" << endl;
4074 }
c57f56b7 4075 out << " return chain;" << endl;
fcc9bb6f 4076 out << "}" << endl << endl;
c57f56b7 4077 }
5b273635 4078 if (hasANALYSISalice) {
4079 out <<"//________________________________________________________________________________" << endl;
4080 out << "Bool_t SetupPar(const char *package) {" << endl;
4081 out << "// Compile the package and set it up." << endl;
4082 out << " TString pkgdir = package;" << endl;
4083 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4084 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4085 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4086 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4087 out << " // Check for BUILD.sh and execute" << endl;
4088 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4089 out << " printf(\"*******************************\\n\");" << endl;
4090 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4091 out << " printf(\"*******************************\\n\");" << endl;
4092 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4093 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4094 out << " gSystem->ChangeDirectory(cdir);" << endl;
4095 out << " return kFALSE;" << endl;
4096 out << " }" << endl;
4097 out << " } else {" << endl;
4098 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4099 out << " gSystem->ChangeDirectory(cdir);" << endl;
4100 out << " return kFALSE;" << endl;
4101 out << " }" << endl;
4102 out << " // Check for SETUP.C and execute" << endl;
4103 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4104 out << " printf(\"*******************************\\n\");" << endl;
4105 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4106 out << " printf(\"*******************************\\n\");" << endl;
4107 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4108 out << " } else {" << endl;
4109 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4110 out << " gSystem->ChangeDirectory(cdir);" << endl;
4111 out << " return kFALSE;" << endl;
4112 out << " }" << endl;
4113 out << " // Restore original workdir" << endl;
4114 out << " gSystem->ChangeDirectory(cdir);" << endl;
4115 out << " return kTRUE;" << endl;
4116 out << "}" << endl;
4117 }
c57f56b7 4118 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4119 }
4120 Bool_t copy = kTRUE;
d3b18c4c 4121 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4122 if (copy) {
4123 CdWork();
4124 TString workdir = gGrid->GetHomeDirectory();
4125 workdir += fGridWorkingDir;
4126 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4127 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4128// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4129 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4130 Form("alien://%s/%s", workdir.Data(),
4131 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4132 }
4133}
4134
0f389141 4135//______________________________________________________________________________
4136void AliAnalysisAlien::WriteMergingMacro()
4137{
4138// Write a macro to merge the outputs per master job.
4139 if (!fMergeViaJDL) return;
4140 if (!fOutputFiles.Length()) {
4141 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4142 return;
4143 }
ec5acdce 4144 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
0f389141 4145 TString mergingMacro = fExecutable;
4146 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4147 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4148 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4149 ofstream out;
4150 out.open(mergingMacro.Data(), ios::out);
4151 if (!out.good()) {
4152 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4153 return;
4154 }
5b273635 4155 Bool_t hasSTEERBase = kFALSE;
4156 Bool_t hasESD = kFALSE;
4157 Bool_t hasAOD = kFALSE;
4158 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4159 Bool_t hasOADB = kFALSE;
5b273635 4160 Bool_t hasANALYSISalice = kFALSE;
4161 Bool_t hasCORRFW = kFALSE;
0f389141 4162 TString func = mergingMacro;
4163 TString comment;
4164 func.ReplaceAll(".C", "");
f47d5cb4 4165 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4166 out << "{" << endl;
4167 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4168 out << " TStopwatch timer;" << endl;
4169 out << " timer.Start();" << endl << endl;
b3e07543 4170 // Reset existing include path
4171 out << "// Reset existing include path and add current directory first in the search" << endl;
4172 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4173 if (!fExecutableCommand.Contains("aliroot")) {
4174 out << "// load base root libraries" << endl;
4175 out << " gSystem->Load(\"libTree\");" << endl;
4176 out << " gSystem->Load(\"libGeom\");" << endl;
4177 out << " gSystem->Load(\"libVMC\");" << endl;
4178 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4179 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4180 }
0f389141 4181 if (fAdditionalRootLibs.Length()) {
4182 // in principle libtree /lib geom libvmc etc. can go into this list, too
4183 out << "// Add aditional libraries" << endl;
4184 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4185 TIter next(list);
4186 TObjString *str;
4187 while((str=(TObjString*)next())) {
4188 if (str->GetString().Contains(".so"))
4189 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4190 }
4191 if (list) delete list;
4192 }
0f389141 4193 out << "// Load analysis framework libraries" << endl;
4194 if (!fPackages) {
7c2cd90a 4195 if (!fExecutableCommand.Contains("aliroot")) {
4196 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4197 out << " gSystem->Load(\"libESD\");" << endl;
4198 out << " gSystem->Load(\"libAOD\");" << endl;
4199 }
0f389141 4200 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4201 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4202 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4203 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4204 } else {
4205 TIter next(fPackages);
4206 TObject *obj;
4207 TString pkgname;
5b273635 4208 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4209 while ((obj=next())) {
4210 pkgname = obj->GetName();
4211 if (pkgname == "STEERBase" ||
4212 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4213 if (pkgname == "ESD" ||
4214 pkgname == "ESD.par") hasESD = kTRUE;
4215 if (pkgname == "AOD" ||
4216 pkgname == "AOD.par") hasAOD = kTRUE;
4217 if (pkgname == "ANALYSIS" ||
4218 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4219 if (pkgname == "OADB" ||
4220 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4221 if (pkgname == "ANALYSISalice" ||
4222 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4223 if (pkgname == "CORRFW" ||
4224 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4225 }
5b273635 4226 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4227 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4228 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4229 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4230 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4231 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4232 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4233 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4234 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4235 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4236 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4237 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4238 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4239 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4240 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4241 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4242 out << "// Compile other par packages" << endl;
4243 next.Reset();
4244 while ((obj=next())) {
4245 pkgname = obj->GetName();
4246 if (pkgname == "STEERBase" ||
4247 pkgname == "STEERBase.par" ||
4248 pkgname == "ESD" ||
4249 pkgname == "ESD.par" ||
4250 pkgname == "AOD" ||
4251 pkgname == "AOD.par" ||
4252 pkgname == "ANALYSIS" ||
4253 pkgname == "ANALYSIS.par" ||
7ea40ac8 4254 pkgname == "OADB" ||
4255 pkgname == "OADB.par" ||
0f389141 4256 pkgname == "ANALYSISalice" ||
4257 pkgname == "ANALYSISalice.par" ||
4258 pkgname == "CORRFW" ||
4259 pkgname == "CORRFW.par") continue;
5b273635 4260 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4261 }
4262 }
b3e07543 4263 out << "// include path" << endl;
4264 // Get the include path from the interpreter and remove entries pointing to AliRoot
4265 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4266 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4267 out << " TIter nextpath(listpaths);" << endl;
4268 out << " TObjString *pname;" << endl;
4269 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4270 out << " TString current = pname->GetName();" << endl;
4271 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4272 out << " gSystem->AddIncludePath(current);" << endl;
4273 out << " }" << endl;
4274 out << " if (listpaths) delete listpaths;" << endl;
4275 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4276 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4277 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4278 if (fAdditionalLibs.Length()) {
4279 out << "// Add aditional AliRoot libraries" << endl;
4280 TObjArray *list = fAdditionalLibs.Tokenize(" ");
4281 TIter next(list);
4282 TObjString *str;
4283 while((str=(TObjString*)next())) {
4284 if (str->GetString().Contains(".so"))
4285 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4286 }
4287 if (list) delete list;
4288 }
4289 out << endl;
4290 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4291 if (fAnalysisSource.Length()) {
4292 TObjArray *list = fAnalysisSource.Tokenize(" ");
4293 TIter next(list);
4294 TObjString *str;
4295 while((str=(TObjString*)next())) {
4296 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4297 }
4298 if (list) delete list;
4299 }
149d288c 4300 out << endl;
4301
0f389141 4302 if (fFastReadOption) {
4303 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4304 out << "// fast xrootd reading enabled" << endl;
4305 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4306 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4307 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4308 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4309 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4310 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4311 }
4312 // Change temp directory to current one
fdee57f1 4313 out << "// Connect to AliEn" << endl;
4314 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
e8b839ab 4315 out << "// Set temporary merging directory to current one" << endl;
4316 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4317 out << "// Set temporary compilation directory to current one" << endl;
4318 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
7c2cd90a 4319 out << " TString outputDir = dir;" << endl;
f790bc1b 4320 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4321 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4322 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4323 out << " TIter *iter = new TIter(list);" << endl;
4324 out << " TObjString *str;" << endl;
a2f5fc01 4325 out << " TString outputFile;" << endl;
0f389141 4326 out << " Bool_t merged = kTRUE;" << endl;
ec5acdce 4327 TString analysisFile = fExecutable;
4328 analysisFile.ReplaceAll(".sh", ".root");
4329 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4330 << analysisFile << "\");" << endl;
4331 out << " if (!mgr) {" << endl;
4332 out << " printf(\"ERROR: Analysis manager could not be extracted from file \");" << endl;
4333 out << " return;" << endl;
4334 out << " }" << endl;
0f389141 4335 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4336 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4337 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4338 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4339 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4340 out << " // Skip already merged outputs" << endl;
a2f5fc01 4341 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
4342 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 4343 out << " continue;" << endl;
4344 out << " }" << endl;
a2f5fc01 4345 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4346 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4347 out << " if (!merged) {" << endl;
a2f5fc01 4348 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4349 out << " return;" << endl;
0f389141 4350 out << " }" << endl;
4351 out << " }" << endl;
ec5acdce 4352 if (mgr && mgr->IsCollectThroughput()) {
4353 out << " TString infolog = \"" << mgr->GetFileInfoLog() << "\";" << endl;
4354 out << " AliAnalysisAlien::MergeInfo(infolog, outputDir);" << endl;
4355 }
7c2cd90a 4356 out << " // all outputs merged, validate" << endl;
4357 out << " ofstream out;" << endl;
4358 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4359 out << " out.close();" << endl;
4360 out << " // read the analysis manager from file" << endl;
f47d5cb4 4361 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
5c4250fc 4362 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4363 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4364 out << " mgr->PrintStatus();" << endl;
ec5acdce 4365 if (mgr) {
4366 if (mgr->GetDebugLevel()>3) {
f866cba5 4367 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4368 } else {
9a8288ff 4369 if (TestBit(AliAnalysisGrid::kTest))
4370 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4371 else
4372 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4373 }
4374 }
9a8288ff 4375 out << " TTree *tree = NULL;" << endl;
4376 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4377 out << "}" << endl << endl;
5b273635 4378 if (hasANALYSISalice) {
4379 out <<"//________________________________________________________________________________" << endl;
4380 out << "Bool_t SetupPar(const char *package) {" << endl;
4381 out << "// Compile the package and set it up." << endl;
4382 out << " TString pkgdir = package;" << endl;
4383 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4384 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4385 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4386 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4387 out << " // Check for BUILD.sh and execute" << endl;
4388 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4389 out << " printf(\"*******************************\\n\");" << endl;
4390 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4391 out << " printf(\"*******************************\\n\");" << endl;
4392 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4393 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4394 out << " gSystem->ChangeDirectory(cdir);" << endl;
4395 out << " return kFALSE;" << endl;
4396 out << " }" << endl;
4397 out << " } else {" << endl;
4398 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4399 out << " gSystem->ChangeDirectory(cdir);" << endl;
4400 out << " return kFALSE;" << endl;
4401 out << " }" << endl;
4402 out << " // Check for SETUP.C and execute" << endl;
4403 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4404 out << " printf(\"*******************************\\n\");" << endl;
4405 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4406 out << " printf(\"*******************************\\n\");" << endl;
4407 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4408 out << " } else {" << endl;
4409 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4410 out << " gSystem->ChangeDirectory(cdir);" << endl;
4411 out << " return kFALSE;" << endl;
4412 out << " }" << endl;
4413 out << " // Restore original workdir" << endl;
4414 out << " gSystem->ChangeDirectory(cdir);" << endl;
4415 out << " return kTRUE;" << endl;
4416 out << "}" << endl;
4417 }
0f389141 4418 }
4419 Bool_t copy = kTRUE;
d3b18c4c 4420 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4421 if (copy) {
4422 CdWork();
4423 TString workdir = gGrid->GetHomeDirectory();
4424 workdir += fGridWorkingDir;
4425 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4426 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4427// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4428 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4429 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4430 }
4431}
4432
4433//______________________________________________________________________________
4434Bool_t AliAnalysisAlien::SetupPar(const char *package)
4435{
205b201f 4436// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4437// Note that for loading the compiled library. The current directory should have precedence in
4438// LD_LIBRARY_PATH
4439 TString pkgdir = package;
4440 pkgdir.ReplaceAll(".par","");
b50a41c3 4441 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4442 TString cdir = gSystem->WorkingDirectory();
4443 gSystem->ChangeDirectory(pkgdir);
4444 // Check for BUILD.sh and execute
4445 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4446 printf("**************************************************\n");
4447 printf("*** Building PAR archive %s\n", package);
4448 printf("**************************************************\n");
4449 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4450 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4451 gSystem->ChangeDirectory(cdir);
4452 return kFALSE;
4453 }
4454 } else {
4455 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4456 gSystem->ChangeDirectory(cdir);
4457 return kFALSE;
4458 }
4459 // Check for SETUP.C and execute
4460 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4461 printf("**************************************************\n");
4462 printf("*** Setup PAR archive %s\n", package);
4463 printf("**************************************************\n");
4464 gROOT->Macro("PROOF-INF/SETUP.C");
4465 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4466 } else {
4467 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4468 gSystem->ChangeDirectory(cdir);
4469 return kFALSE;
4470 }
4471 // Restore original workdir
4472 gSystem->ChangeDirectory(cdir);
4473 return kTRUE;
4474}
4475
c57f56b7 4476//______________________________________________________________________________
4477void AliAnalysisAlien::WriteExecutable()
4478{
4479// Generate the alien executable script.
4480 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4481 ofstream out;
4482 out.open(fExecutable.Data(), ios::out);
4483 if (out.bad()) {
5513444a 4484 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4485 return;
4486 }
4487 out << "#!/bin/bash" << endl;
2a4d5166 4488 // Make sure we can properly compile par files
4489 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4490 out << "echo \"=========================================\"" << endl;
4491 out << "echo \"############## PATH : ##############\"" << endl;
4492 out << "echo $PATH" << endl;
4493 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4494 out << "echo $LD_LIBRARY_PATH" << endl;
4495 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4496 out << "echo $ROOTSYS" << endl;
4497 out << "echo \"############## which root : ##############\"" << endl;
4498 out << "which root" << endl;
4499 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4500 out << "echo $ALICE_ROOT" << endl;
4501 out << "echo \"############## which aliroot : ##############\"" << endl;
4502 out << "which aliroot" << endl;
9c5ddadc 4503 out << "echo \"############## system limits : ##############\"" << endl;
4504 out << "ulimit -a" << endl;
4505 out << "echo \"############## memory : ##############\"" << endl;
4506 out << "free -m" << endl;
c57f56b7 4507 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4508 out << fExecutableCommand << " ";
370c9880 4509 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4510 out << "RET=$?" << endl;
4511 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4512 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4513 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4514 out << " let sig=\"$RET - 128\""<<endl;
4515 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4516 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4517 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4518 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4519 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4520 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4521 out << " fi"<<endl;
4522 out << " exit $RET"<< endl;
4523 out << "fi" << endl << endl ;
4524 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4525 out << "echo \"############## memory after: ##############\"" << endl;
4526 out << "free -m" << endl;
c57f56b7 4527 }
4528 Bool_t copy = kTRUE;
d3b18c4c 4529 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4530 if (copy) {
4531 CdWork();
4532 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4533 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4534 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4535 workdir += fGridWorkingDir;
4536 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4537 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4538 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4539// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4540 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4541 executable.Data())) Fatal("","Terminating");
c57f56b7 4542 }
4543}
4544
0f389141 4545//______________________________________________________________________________
4546void AliAnalysisAlien::WriteMergeExecutable()
4547{
4548// Generate the alien executable script for the merging job.
4549 if (!fMergeViaJDL) return;
4550 TString mergeExec = fExecutable;
4551 mergeExec.ReplaceAll(".sh", "_merge.sh");
4552 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4553 ofstream out;
4554 out.open(mergeExec.Data(), ios::out);
4555 if (out.bad()) {
4556 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4557 return;
4558 }
4559 out << "#!/bin/bash" << endl;
2a4d5166 4560 // Make sure we can properly compile par files
4561 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4562 out << "echo \"=========================================\"" << endl;
4563 out << "echo \"############## PATH : ##############\"" << endl;
4564 out << "echo $PATH" << endl;
4565 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4566 out << "echo $LD_LIBRARY_PATH" << endl;
4567 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4568 out << "echo $ROOTSYS" << endl;
4569 out << "echo \"############## which root : ##############\"" << endl;
4570 out << "which root" << endl;
4571 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4572 out << "echo $ALICE_ROOT" << endl;
4573 out << "echo \"############## which aliroot : ##############\"" << endl;
4574 out << "which aliroot" << endl;
4575 out << "echo \"############## system limits : ##############\"" << endl;
4576 out << "ulimit -a" << endl;
4577 out << "echo \"############## memory : ##############\"" << endl;
4578 out << "free -m" << endl;
4579 out << "echo \"=========================================\"" << endl << endl;
0f389141 4580 TString mergeMacro = fExecutable;
4581 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4582 if (IsOneStageMerging())
4583 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4584 else
f47d5cb4 4585 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4586 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4587 out << "RET=$?" << endl;
4588 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4589 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4590 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4591 out << " let sig=\"$RET - 128\""<<endl;
4592 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4593 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4594 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4595 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4596 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4597 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4598 out << " fi"<<endl;
4599 out << " exit $RET"<< endl;
4600 out << "fi" << endl << endl ;
0f389141 4601 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4602 out << "echo \"############## memory after: ##############\"" << endl;
4603 out << "free -m" << endl;
0f389141 4604 }
4605 Bool_t copy = kTRUE;
d3b18c4c 4606 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4607 if (copy) {
4608 CdWork();
4609 TString workdir = gGrid->GetHomeDirectory();
4610 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4611 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4612 workdir += fGridWorkingDir;
4613 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4614 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4615 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4616// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4617 if (!copyLocal2Alien("WriteMergeExecutable",
4618 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4619 }
4620}
4621
c57f56b7 4622//______________________________________________________________________________
5513444a 4623void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4624{
4625// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4626// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4627// Next lines: full_path_to_dataset XXX (XXX is a string)
4628// To submit, one has to: submit jdl XXX for all lines
4629 ofstream out;
4630 out.open(filename, ios::out);
4631 if (out.bad()) {
4632 Error("WriteProductionFile", "Bad file name: %s", filename);
4633 return;
4634 }
d3b18c4c 4635 TString workdir;
4636 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4637 workdir = gGrid->GetHomeDirectory();
5513444a 4638 workdir += fGridWorkingDir;
f5e8c702 4639 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4640 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4641 out << locjdl << " " << njobspermaster << endl;
5513444a 4642 Int_t nmasterjobs = fInputFiles->GetEntries();
4643 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4644 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4645 runOutDir.ReplaceAll(".xml", "");
4646 if (fOutputToRunNo)
4647 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4648 else
4649 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4650 }
d3b18c4c 4651 if (gGrid) {
4652 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4653 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4654// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4655 if (!copyLocal2Alien("WriteProductionFile", filename,
4656 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4657 }
5513444a 4658}
4659
4660//______________________________________________________________________________
0f389141 4661void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4662{
4663// Generate the alien validation script.
4664 // Generate the validation script
4665 TObjString *os;
d3b18c4c 4666 if (fValidationScript.IsNull()) {
4667 fValidationScript = fExecutable;
4668 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4669 }
4670 TString validationScript = fValidationScript;
4671 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4672 if (!Connect()) {
4673 Error("WriteValidationScript", "Alien connection required");
4674 return;
4675 }
d3b18c4c 4676 if (!fTerminateFiles.IsNull()) {
4677 fTerminateFiles.Strip();
4678 fTerminateFiles.ReplaceAll(" ",",");
4679 }
a2f5fc01 4680 TString outStream = "";
4681 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4682 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4683 ofstream out;
0d5d317c 4684 out.open(validationScript, ios::out);
c57f56b7 4685 out << "#!/bin/bash" << endl;
4686 out << "##################################################" << endl;
4687 out << "validateout=`dirname $0`" << endl;
4688 out << "validatetime=`date`" << endl;
4689 out << "validated=\"0\";" << endl;
4690 out << "error=0" << endl;
4691 out << "if [ -z $validateout ]" << endl;
4692 out << "then" << endl;
4693 out << " validateout=\".\"" << endl;
4694 out << "fi" << endl << endl;
4695 out << "cd $validateout;" << endl;
4696 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4697 out << "echo \"*******************************************************\"" << outStream << endl;
4698 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4699 out << "" << endl;
a2f5fc01 4700 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4701 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4702 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4703 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4704 out << "ls -la ./" << outStream << endl;
4705 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4706 out << "##################################################" << endl;
ebec370a 4707 out << "" << endl;
4708
4709 out << "if [ ! -f stderr ] ; then" << endl;
4710 out << " error=1" << endl;
a2f5fc01 4711 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4712 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4713 out << "fi" << endl;
4714
b34c9f51 4715 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4716 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4717 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4718 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4719 out << "" << endl;
4720
ebec370a 4721 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4722 out << " error=1" << endl;
a2f5fc01 4723 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4724 out << " echo \"$parArch\" " << outStream << endl;
4725 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4726 out << "fi" << endl;
4727
4728 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4729 out << " error=1" << endl;
a2f5fc01 4730 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4731 out << " echo \"$segViol\" " << outStream << endl;
4732 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4733 out << "fi" << endl;
4734
4735 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4736 out << " error=1" << endl;
a2f5fc01 4737 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4738 out << " echo \"$segFault\" " << outStream << endl;
4739 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4740 out << "fi" << endl;
4741
b34c9f51 4742 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4743 out << " error=1" << endl;
a2f5fc01 4744 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4745 out << " echo \"$glibcErr\" " << outStream << endl;
4746 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4747 out << "fi" << endl;
4748
ebec370a 4749 // Part dedicated to the specific analyses running into the train
4750
d3b18c4c 4751 TString outputFiles = fOutputFiles;
4752 if (merge && !fTerminateFiles.IsNull()) {
4753 outputFiles += ",";
4754 outputFiles += fTerminateFiles;
4755 }
4756 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4757 TIter next1(arr);
a2f5fc01 4758 TString outputFile;
bbd88db8 4759 while (!merge && (os=(TObjString*)next1())) {
4760 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4761 outputFile = os->GetString();
4762 Int_t index = outputFile.Index("@");
4763 if (index > 0) outputFile.Remove(index);
bbd88db8 4764 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4765 if (outputFile.Contains("*")) continue;
4766 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4767 out << " error=1" << endl;
7c2cd90a 4768 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4769 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4770 out << "fi" << endl;
4771 }
4772 delete arr;
7c2cd90a 4773 out << "if ! [ -f outputs_valid ] ; then" << endl;
4774 out << " error=1" << endl;
4775 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4776 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4777 out << "fi" << endl;
923e2ca5 4778
c57f56b7 4779 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4780 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4781 if (!IsKeepLogs()) {
4782 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4783 outStream = "";
149d288c 4784 out << " rm -f std*" << endl;
4785 }
c57f56b7 4786 out << "fi" << endl;
4787
a2f5fc01 4788 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4789 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4790 out << "cd -" << endl;
4791 out << "exit $error" << endl;
4792 }
4793 Bool_t copy = kTRUE;
d3b18c4c 4794 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4795 if (copy) {
4796 CdWork();
4797 TString workdir = gGrid->GetHomeDirectory();
4798 workdir += fGridWorkingDir;
d3b18c4c 4799 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4800 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 4801// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4802 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
4803 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 4804 }
4805}