]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bugfix: correct range of DDL for specified detector
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
51ClassImp(AliAnalysisAlien)
6c395669 52#if 0
53;
54#endif
c57f56b7 55
6c395669 56namespace {
57 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
58 {
59 TString sl(Form("file:%s", loc));
60 TString sr(Form("alien://%s", rem));
61 Bool_t ret = TFile::Cp(sl, sr);
62 if (!ret) {
63 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
64 }
65 return ret;
66 }
67}
68
c57f56b7 69//______________________________________________________________________________
70AliAnalysisAlien::AliAnalysisAlien()
71 :AliAnalysisGrid(),
72 fGridJDL(NULL),
0f389141 73 fMergingJDL(NULL),
c57f56b7 74 fPrice(0),
75 fTTL(0),
76 fSplitMaxInputFileNumber(0),
77 fMaxInitFailed(0),
78 fMasterResubmitThreshold(0),
bb885a9e 79 fNtestFiles(0),
319593fb 80 fNrunsPerMaster(0),
16a4353c 81 fMaxMergeFiles(0),
b3e07543 82 fMaxMergeStages(0),
d2a409b2 83 fNsubmitted(0),
a3e84053 84 fProductionMode(0),
cd11251e 85 fOutputToRunNo(0),
0f389141 86 fMergeViaJDL(0),
87 fFastReadOption(0),
e1c22e21 88 fOverwriteMode(1),
149d288c 89 fNreplicas(2),
3bdcb562 90 fNproofWorkers(0),
27734f0e 91 fNproofWorkersPerSlave(0),
3bdcb562 92 fProofReset(0),
c57f56b7 93 fRunNumbers(),
94 fExecutable(),
0a1c1f7f 95 fExecutableCommand(),
c57f56b7 96 fArguments(),
631c0b05 97 fExecutableArgs(),
c57f56b7 98 fAnalysisMacro(),
99 fAnalysisSource(),
d3b18c4c 100 fValidationScript(),
d5c6455a 101 fAdditionalRootLibs(),
c57f56b7 102 fAdditionalLibs(),
103 fSplitMode(),
104 fAPIVersion(),
105 fROOTVersion(),
106 fAliROOTVersion(),
648174cf 107 fExternalPackages(),
c57f56b7 108 fUser(),
109 fGridWorkingDir(),
110 fGridDataDir(),
111 fDataPattern(),
112 fGridOutputDir(),
113 fOutputArchive(),
114 fOutputFiles(),
115 fInputFormat(),
e7c71df0 116 fDatasetName(),
c57f56b7 117 fJDLName(),
d3b18c4c 118 fTerminateFiles(),
bb885a9e 119 fMergeExcludes(),
37d21c01 120 fRegisterExcludes(),
f965131e 121 fIncludePath(),
bb885a9e 122 fCloseSE(),
0df6ccf2 123 fFriendChainName(),
c6cb3634 124 fJobTag(),
648174cf 125 fOutputSingle(),
5fce53f4 126 fRunPrefix(),
3bdcb562 127 fProofCluster(),
128 fProofDataSet(),
129 fFileForTestMode(),
130 fRootVersionForProof(),
131 fAliRootMode(),
f47d5cb4 132 fMergeDirName(),
4e5c5506 133 fInputFiles(0),
28ce6c9c 134 fPackages(0),
77f34eae 135 fModules(0),
28ce6c9c 136 fProofParam()
c57f56b7 137{
138// Dummy ctor.
139 SetDefaults();
140}
141
142//______________________________________________________________________________
143AliAnalysisAlien::AliAnalysisAlien(const char *name)
144 :AliAnalysisGrid(name),
145 fGridJDL(NULL),
0f389141 146 fMergingJDL(NULL),
c57f56b7 147 fPrice(0),
148 fTTL(0),
149 fSplitMaxInputFileNumber(0),
150 fMaxInitFailed(0),
151 fMasterResubmitThreshold(0),
bb885a9e 152 fNtestFiles(0),
319593fb 153 fNrunsPerMaster(0),
16a4353c 154 fMaxMergeFiles(0),
b3e07543 155 fMaxMergeStages(0),
d2a409b2 156 fNsubmitted(0),
a3e84053 157 fProductionMode(0),
cd11251e 158 fOutputToRunNo(0),
0f389141 159 fMergeViaJDL(0),
160 fFastReadOption(0),
e1c22e21 161 fOverwriteMode(1),
149d288c 162 fNreplicas(2),
3bdcb562 163 fNproofWorkers(0),
27734f0e 164 fNproofWorkersPerSlave(0),
3bdcb562 165 fProofReset(0),
c57f56b7 166 fRunNumbers(),
167 fExecutable(),
0a1c1f7f 168 fExecutableCommand(),
c57f56b7 169 fArguments(),
631c0b05 170 fExecutableArgs(),
c57f56b7 171 fAnalysisMacro(),
172 fAnalysisSource(),
d3b18c4c 173 fValidationScript(),
d5c6455a 174 fAdditionalRootLibs(),
c57f56b7 175 fAdditionalLibs(),
176 fSplitMode(),
177 fAPIVersion(),
178 fROOTVersion(),
179 fAliROOTVersion(),
648174cf 180 fExternalPackages(),
c57f56b7 181 fUser(),
182 fGridWorkingDir(),
183 fGridDataDir(),
184 fDataPattern(),
185 fGridOutputDir(),
186 fOutputArchive(),
187 fOutputFiles(),
188 fInputFormat(),
e7c71df0 189 fDatasetName(),
c57f56b7 190 fJDLName(),
d3b18c4c 191 fTerminateFiles(),
bb885a9e 192 fMergeExcludes(),
37d21c01 193 fRegisterExcludes(),
f965131e 194 fIncludePath(),
bb885a9e 195 fCloseSE(),
0df6ccf2 196 fFriendChainName(),
c6cb3634 197 fJobTag(),
648174cf 198 fOutputSingle(),
5fce53f4 199 fRunPrefix(),
3bdcb562 200 fProofCluster(),
201 fProofDataSet(),
202 fFileForTestMode(),
203 fRootVersionForProof(),
204 fAliRootMode(),
f47d5cb4 205 fMergeDirName(),
4e5c5506 206 fInputFiles(0),
28ce6c9c 207 fPackages(0),
77f34eae 208 fModules(0),
28ce6c9c 209 fProofParam()
c57f56b7 210{
211// Default ctor.
212 SetDefaults();
213}
214
215//______________________________________________________________________________
216AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
217 :AliAnalysisGrid(other),
218 fGridJDL(NULL),
0f389141 219 fMergingJDL(NULL),
c57f56b7 220 fPrice(other.fPrice),
221 fTTL(other.fTTL),
222 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
223 fMaxInitFailed(other.fMaxInitFailed),
224 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 225 fNtestFiles(other.fNtestFiles),
319593fb 226 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 227 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 228 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 229 fNsubmitted(other.fNsubmitted),
a3e84053 230 fProductionMode(other.fProductionMode),
cd11251e 231 fOutputToRunNo(other.fOutputToRunNo),
0f389141 232 fMergeViaJDL(other.fMergeViaJDL),
233 fFastReadOption(other.fFastReadOption),
234 fOverwriteMode(other.fOverwriteMode),
149d288c 235 fNreplicas(other.fNreplicas),
3bdcb562 236 fNproofWorkers(other.fNproofWorkers),
27734f0e 237 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 238 fProofReset(other.fProofReset),
c57f56b7 239 fRunNumbers(other.fRunNumbers),
240 fExecutable(other.fExecutable),
0a1c1f7f 241 fExecutableCommand(other.fExecutableCommand),
c57f56b7 242 fArguments(other.fArguments),
631c0b05 243 fExecutableArgs(other.fExecutableArgs),
c57f56b7 244 fAnalysisMacro(other.fAnalysisMacro),
245 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 246 fValidationScript(other.fValidationScript),
d5c6455a 247 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 248 fAdditionalLibs(other.fAdditionalLibs),
249 fSplitMode(other.fSplitMode),
250 fAPIVersion(other.fAPIVersion),
251 fROOTVersion(other.fROOTVersion),
252 fAliROOTVersion(other.fAliROOTVersion),
648174cf 253 fExternalPackages(other.fExternalPackages),
c57f56b7 254 fUser(other.fUser),
255 fGridWorkingDir(other.fGridWorkingDir),
256 fGridDataDir(other.fGridDataDir),
257 fDataPattern(other.fDataPattern),
258 fGridOutputDir(other.fGridOutputDir),
259 fOutputArchive(other.fOutputArchive),
260 fOutputFiles(other.fOutputFiles),
261 fInputFormat(other.fInputFormat),
e7c71df0 262 fDatasetName(other.fDatasetName),
c57f56b7 263 fJDLName(other.fJDLName),
d3b18c4c 264 fTerminateFiles(other.fTerminateFiles),
bb885a9e 265 fMergeExcludes(other.fMergeExcludes),
37d21c01 266 fRegisterExcludes(other.fRegisterExcludes),
f965131e 267 fIncludePath(other.fIncludePath),
bb885a9e 268 fCloseSE(other.fCloseSE),
0df6ccf2 269 fFriendChainName(other.fFriendChainName),
c6cb3634 270 fJobTag(other.fJobTag),
648174cf 271 fOutputSingle(other.fOutputSingle),
5fce53f4 272 fRunPrefix(other.fRunPrefix),
3bdcb562 273 fProofCluster(other.fProofCluster),
274 fProofDataSet(other.fProofDataSet),
275 fFileForTestMode(other.fFileForTestMode),
276 fRootVersionForProof(other.fRootVersionForProof),
277 fAliRootMode(other.fAliRootMode),
f47d5cb4 278 fMergeDirName(other.fMergeDirName),
4e5c5506 279 fInputFiles(0),
28ce6c9c 280 fPackages(0),
77f34eae 281 fModules(0),
28ce6c9c 282 fProofParam()
c57f56b7 283{
284// Copy ctor.
285 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 286 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 287 fRunRange[0] = other.fRunRange[0];
288 fRunRange[1] = other.fRunRange[1];
c57f56b7 289 if (other.fInputFiles) {
290 fInputFiles = new TObjArray();
291 TIter next(other.fInputFiles);
292 TObject *obj;
293 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
294 fInputFiles->SetOwner();
295 }
4e5c5506 296 if (other.fPackages) {
297 fPackages = new TObjArray();
298 TIter next(other.fPackages);
299 TObject *obj;
300 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
301 fPackages->SetOwner();
302 }
77f34eae 303 if (other.fModules) {
304 fModules = new TObjArray();
305 fModules->SetOwner();
306 TIter next(other.fModules);
307 AliAnalysisTaskCfg *mod, *crt;
308 while ((crt=(AliAnalysisTaskCfg*)next())) {
309 mod = new AliAnalysisTaskCfg(*crt);
310 fModules->Add(mod);
311 }
312 }
c57f56b7 313}
314
315//______________________________________________________________________________
316AliAnalysisAlien::~AliAnalysisAlien()
317{
318// Destructor.
77f34eae 319 delete fGridJDL;
320 delete fMergingJDL;
321 delete fInputFiles;
322 delete fPackages;
323 delete fModules;
28ce6c9c 324 fProofParam.DeleteAll();
c57f56b7 325}
326
327//______________________________________________________________________________
328AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
329{
330// Assignment.
331 if (this != &other) {
332 AliAnalysisGrid::operator=(other);
333 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 334 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 335 fPrice = other.fPrice;
336 fTTL = other.fTTL;
337 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
338 fMaxInitFailed = other.fMaxInitFailed;
339 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 340 fNtestFiles = other.fNtestFiles;
a3e84053 341 fNrunsPerMaster = other.fNrunsPerMaster;
342 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 343 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 344 fNsubmitted = other.fNsubmitted;
345 fProductionMode = other.fProductionMode;
cd11251e 346 fOutputToRunNo = other.fOutputToRunNo;
0f389141 347 fMergeViaJDL = other.fMergeViaJDL;
348 fFastReadOption = other.fFastReadOption;
349 fOverwriteMode = other.fOverwriteMode;
149d288c 350 fNreplicas = other.fNreplicas;
3bdcb562 351 fNproofWorkers = other.fNproofWorkers;
27734f0e 352 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 353 fProofReset = other.fProofReset;
c57f56b7 354 fRunNumbers = other.fRunNumbers;
355 fExecutable = other.fExecutable;
0a1c1f7f 356 fExecutableCommand = other.fExecutableCommand;
c57f56b7 357 fArguments = other.fArguments;
631c0b05 358 fExecutableArgs = other.fExecutableArgs;
c57f56b7 359 fAnalysisMacro = other.fAnalysisMacro;
360 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 361 fValidationScript = other.fValidationScript;
d5c6455a 362 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 363 fAdditionalLibs = other.fAdditionalLibs;
364 fSplitMode = other.fSplitMode;
365 fAPIVersion = other.fAPIVersion;
366 fROOTVersion = other.fROOTVersion;
367 fAliROOTVersion = other.fAliROOTVersion;
648174cf 368 fExternalPackages = other.fExternalPackages;
c57f56b7 369 fUser = other.fUser;
370 fGridWorkingDir = other.fGridWorkingDir;
371 fGridDataDir = other.fGridDataDir;
372 fDataPattern = other.fDataPattern;
373 fGridOutputDir = other.fGridOutputDir;
374 fOutputArchive = other.fOutputArchive;
375 fOutputFiles = other.fOutputFiles;
376 fInputFormat = other.fInputFormat;
e7c71df0 377 fDatasetName = other.fDatasetName;
c57f56b7 378 fJDLName = other.fJDLName;
d3b18c4c 379 fTerminateFiles = other.fTerminateFiles;
bb885a9e 380 fMergeExcludes = other.fMergeExcludes;
37d21c01 381 fRegisterExcludes = other.fRegisterExcludes;
f965131e 382 fIncludePath = other.fIncludePath;
bb885a9e 383 fCloseSE = other.fCloseSE;
0df6ccf2 384 fFriendChainName = other.fFriendChainName;
c6cb3634 385 fJobTag = other.fJobTag;
648174cf 386 fOutputSingle = other.fOutputSingle;
5fce53f4 387 fRunPrefix = other.fRunPrefix;
3bdcb562 388 fProofCluster = other.fProofCluster;
389 fProofDataSet = other.fProofDataSet;
390 fFileForTestMode = other.fFileForTestMode;
391 fRootVersionForProof = other.fRootVersionForProof;
392 fAliRootMode = other.fAliRootMode;
f47d5cb4 393 fMergeDirName = other.fMergeDirName;
c57f56b7 394 if (other.fInputFiles) {
395 fInputFiles = new TObjArray();
396 TIter next(other.fInputFiles);
397 TObject *obj;
398 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
399 fInputFiles->SetOwner();
400 }
4e5c5506 401 if (other.fPackages) {
402 fPackages = new TObjArray();
403 TIter next(other.fPackages);
404 TObject *obj;
405 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
406 fPackages->SetOwner();
407 }
77f34eae 408 if (other.fModules) {
409 fModules = new TObjArray();
410 fModules->SetOwner();
411 TIter next(other.fModules);
412 AliAnalysisTaskCfg *mod, *crt;
413 while ((crt=(AliAnalysisTaskCfg*)next())) {
414 mod = new AliAnalysisTaskCfg(*crt);
415 fModules->Add(mod);
416 }
417 }
c57f56b7 418 }
419 return *this;
420}
421
37d21c01 422//______________________________________________________________________________
423void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
424{
425// Add a single additional library to be loaded. Extension must be present.
426 TString lib(name);
427 if (!lib.Contains(".")) {
428 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
429 return;
430 }
431 if (fAdditionalLibs.Contains(name)) {
432 Warning("AddAdditionalLibrary", "Library %s already added.", name);
433 return;
434 }
435 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
436 fAdditionalLibs += lib;
437}
438
77f34eae 439//______________________________________________________________________________
440void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
441{
442// Adding a module. Checks if already existing. Becomes owned by this.
443 if (!module) return;
444 if (GetModule(module->GetName())) {
445 Error("AddModule", "A module having the same name %s already added", module->GetName());
446 return;
447 }
448 if (!fModules) {
449 fModules = new TObjArray();
450 fModules->SetOwner();
451 }
452 fModules->Add(module);
453}
454
455//______________________________________________________________________________
456void AliAnalysisAlien::AddModules(TObjArray *list)
457{
458// Adding a list of modules. Checks if already existing. Becomes owned by this.
459 TIter next(list);
460 AliAnalysisTaskCfg *module;
461 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
462}
463
464//______________________________________________________________________________
465Bool_t AliAnalysisAlien::CheckDependencies()
466{
467// Check if all dependencies are satisfied. Reorder modules if needed.
468 Int_t nmodules = GetNmodules();
469 if (!nmodules) {
470 Warning("CheckDependencies", "No modules added yet to check their dependencies");
471 return kTRUE;
472 }
473 AliAnalysisTaskCfg *mod = 0;
474 AliAnalysisTaskCfg *dep = 0;
475 TString depname;
476 Int_t i, j, k;
477 for (i=0; i<nmodules; i++) {
478 mod = (AliAnalysisTaskCfg*) fModules->At(i);
479 Int_t ndeps = mod->GetNdeps();
480 Int_t istart = i;
481 for (j=0; j<ndeps; j++) {
482 depname = mod->GetDependency(j);
483 dep = GetModule(depname);
484 if (!dep) {
485 Error("CheckDependencies","Dependency %s not added for module %s",
486 depname.Data(), mod->GetName());
487 return kFALSE;
488 }
489 if (dep->NeedsDependency(mod->GetName())) {
490 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
491 mod->GetName(), dep->GetName());
492 return kFALSE;
493 }
494 Int_t idep = fModules->IndexOf(dep);
495 // The dependency task must come first
496 if (idep>i) {
497 // Remove at idep and move all objects below up one slot
498 // down to index i included.
499 fModules->RemoveAt(idep);
3e40fd4c 500 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 501 fModules->AddAt(dep, i++);
502 }
503 //Redo from istart if dependencies were inserted
504 if (i>istart) i=istart-1;
505 }
506 }
507 return kTRUE;
508}
509
3e40fd4c 510//______________________________________________________________________________
511AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
512{
513// Create the analysis manager and optionally execute the macro in filename.
514 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
515 if (mgr) return mgr;
516 mgr = new AliAnalysisManager(name);
517 mgr->SetGridHandler((AliAnalysisGrid*)this);
518 if (strlen(filename)) {
519 TString line = gSystem->ExpandPathName(filename);
520 line.Prepend(".x ");
521 gROOT->ProcessLine(line.Data());
522 }
523 return mgr;
524}
525
77f34eae 526//______________________________________________________________________________
527Int_t AliAnalysisAlien::GetNmodules() const
528{
529// Get number of modules.
530 if (!fModules) return 0;
531 return fModules->GetEntries();
532}
533
534//______________________________________________________________________________
535AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
536{
537// Get a module by name.
538 if (!fModules) return 0;
539 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
540}
541
542//______________________________________________________________________________
543Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
544{
545// Load a given module.
546 if (mod->IsLoaded()) return kTRUE;
192ddca2 547 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
548 if (!mgr) {
549 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
550 return kFALSE;
551 }
77f34eae 552 Int_t ndeps = mod->GetNdeps();
553 TString depname;
554 for (Int_t j=0; j<ndeps; j++) {
555 depname = mod->GetDependency(j);
556 AliAnalysisTaskCfg *dep = GetModule(depname);
557 if (!dep) {
558 Error("LoadModule","Dependency %s not existing for module %s",
559 depname.Data(), mod->GetName());
560 return kFALSE;
561 }
562 if (!LoadModule(dep)) {
563 Error("LoadModule","Dependency %s for module %s could not be loaded",
564 depname.Data(), mod->GetName());
565 return kFALSE;
566 }
567 }
568 // Load libraries for the module
569 if (!mod->CheckLoadLibraries()) {
570 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
571 return kFALSE;
572 }
192ddca2 573 // Check if a custom file name was requested
574 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
575
576 // Check if a custom terminate file name was requested
577 if (strlen(mod->GetTerminateFileName())) {
578 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
579 fTerminateFiles += mod->GetTerminateFileName();
580 }
581
77f34eae 582 // Execute the macro
583 if (mod->ExecuteMacro()<0) {
584 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
585 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
586 return kFALSE;
587 }
588 // Configure dependencies
589 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
590 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
591 mod->GetConfigMacro()->GetTitle(), mod->GetName());
592 return kFALSE;
593 }
5e8c7c22 594 // Adjust extra libraries
595 Int_t nlibs = mod->GetNlibs();
596 TString lib;
597 for (Int_t i=0; i<nlibs; i++) {
598 lib = mod->GetLibrary(i);
5e8c7c22 599 lib = Form("lib%s.so", lib.Data());
c9d6bd12 600 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 601 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
602 fAdditionalLibs += lib;
603 }
77f34eae 604 return kTRUE;
605}
606
139fe6ba 607//______________________________________________________________________________
608Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
609{
610// Generate the full train.
611 fAdditionalLibs = "";
612 if (!LoadModules()) return kFALSE;
613 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
614 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 615 mgr->RunLocalInit();
139fe6ba 616 mgr->PrintStatus();
617 Int_t productionMode = fProductionMode;
618 SetProductionMode();
619 TString macro = fAnalysisMacro;
620 TString executable = fExecutable;
621 TString validation = fValidationScript;
622 TString execCommand = fExecutableCommand;
623 SetAnalysisMacro(Form("%s.C", name));
624 SetExecutable(Form("%s.sh", name));
a8e31fb0 625// SetExecutableCommand("aliroot -b -q ");
139fe6ba 626 SetValidationScript(Form("%s_validation.sh", name));
627 StartAnalysis();
628 SetProductionMode(productionMode);
629 fAnalysisMacro = macro;
630 fExecutable = executable;
631 fExecutableCommand = execCommand;
632 fValidationScript = validation;
633 return kTRUE;
634}
635
77f34eae 636//______________________________________________________________________________
21ca8e59 637Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 638{
639// Generate test macros for a single module or for the full train.
5e8c7c22 640 fAdditionalLibs = "";
77f34eae 641 if (strlen(modname)) {
21ca8e59 642 if (!CheckDependencies()) return kFALSE;
643 AliAnalysisTaskCfg *mod = GetModule(modname);
644 if (!mod) {
645 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
646 return kFALSE;
647 }
648 if (!LoadModule(mod)) return kFALSE;
649 } else if (!LoadModules()) return kFALSE;
650 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
651 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 652 mgr->RunLocalInit();
21ca8e59 653 mgr->PrintStatus();
654 SetLocalTest(kTRUE);
655 Int_t productionMode = fProductionMode;
656 SetProductionMode();
657 TString macro = fAnalysisMacro;
658 TString executable = fExecutable;
659 TString validation = fValidationScript;
660 TString execCommand = fExecutableCommand;
661 SetAnalysisMacro(Form("%s.C", name));
662 SetExecutable(Form("%s.sh", name));
a8e31fb0 663// SetExecutableCommand("aliroot -b -q ");
21ca8e59 664 SetValidationScript(Form("%s_validation.sh", name));
665 WriteAnalysisFile();
666 WriteAnalysisMacro();
667 WriteExecutable();
668 WriteValidationScript();
669 SetLocalTest(kFALSE);
670 SetProductionMode(productionMode);
671 fAnalysisMacro = macro;
672 fExecutable = executable;
673 fExecutableCommand = execCommand;
674 fValidationScript = validation;
77f34eae 675 return kTRUE;
676}
677
678//______________________________________________________________________________
679Bool_t AliAnalysisAlien::LoadModules()
680{
681// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 682 fAdditionalLibs = "";
21ca8e59 683 Int_t nmodules = GetNmodules();
684 if (!nmodules) {
685 Warning("LoadModules", "No module to be loaded");
686 return kTRUE;
687 }
3e40fd4c 688 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
689 if (!mgr) {
690 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
691 return kFALSE;
692 }
77f34eae 693 if (!CheckDependencies()) return kFALSE;
21ca8e59 694 nmodules = GetNmodules();
77f34eae 695 AliAnalysisTaskCfg *mod;
21ca8e59 696 for (Int_t imod=0; imod<nmodules; imod++) {
697 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 698 if (!LoadModule(mod)) return kFALSE;
699 }
77f34eae 700 return kTRUE;
701}
702
74013535 703//______________________________________________________________________________
704void AliAnalysisAlien::SetRunPrefix(const char *prefix)
705{
706// Set the run number format. Can be a prefix or a format like "%09d"
707 fRunPrefix = prefix;
708 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
709}
710
f965131e 711//______________________________________________________________________________
712void AliAnalysisAlien::AddIncludePath(const char *path)
713{
714// Add include path in the remote analysis macro.
715 TString p(path);
716 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
717 else fIncludePath += Form("-I%s ", path);
718}
719
c57f56b7 720//______________________________________________________________________________
721void AliAnalysisAlien::AddRunNumber(Int_t run)
722{
723// Add a run number to the list of runs to be processed.
724 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 725 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 726}
727
98690895 728//______________________________________________________________________________
729void AliAnalysisAlien::AddRunList(const char* runList)
730{
731// Add several runs into the list of runs; they are expected to be separated by a blank character.
732 TString sList = runList;
733 TObjArray *list = sList.Tokenize(" ");
734 Int_t n = list->GetEntries();
735 for (Int_t i = 0; i < n; i++) {
736 TObjString *os = (TObjString*)list->At(i);
737 AddRunNumber(os->GetString().Atoi());
738 }
739 delete list;
740}
741
ee75cfc3 742//______________________________________________________________________________
743void AliAnalysisAlien::AddRunNumber(const char* run)
744{
745// Add a run number to the list of runs to be processed.
7aa5d539 746 TString runs = run;
747 TObjString *os;
748 TObjArray *arr = runs.Tokenize(" ");
749 TIter next(arr);
750 TString prefix;
751 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
752 while ((os=(TObjString*)next())){
753 if (fRunNumbers.Length()) fRunNumbers += " ";
754 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
755 }
756 delete arr;
ee75cfc3 757}
758
c57f56b7 759//______________________________________________________________________________
760void AliAnalysisAlien::AddDataFile(const char *lfn)
761{
762// Adds a data file to the input to be analysed. The file should be a valid LFN
763// or point to an existing file in the alien workdir.
764 if (!fInputFiles) fInputFiles = new TObjArray();
765 fInputFiles->Add(new TObjString(lfn));
766}
648174cf 767
768//______________________________________________________________________________
769void AliAnalysisAlien::AddExternalPackage(const char *package)
770{
771// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
772 if (fExternalPackages) fExternalPackages += " ";
773 fExternalPackages += package;
774}
775
c57f56b7 776//______________________________________________________________________________
777Bool_t AliAnalysisAlien::Connect()
778{
779// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
780 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 781 if (fProductionMode) return kTRUE;
c57f56b7 782 if (!gGrid) {
783 Info("Connect", "Trying to connect to AliEn ...");
784 TGrid::Connect("alien://");
785 }
786 if (!gGrid || !gGrid->IsConnected()) {
787 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
788 return kFALSE;
789 }
790 fUser = gGrid->GetUser();
791 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
792 return kTRUE;
793}
794
795//______________________________________________________________________________
796void AliAnalysisAlien::CdWork()
797{
798// Check validity of alien workspace. Create directory if possible.
799 if (!Connect()) {
800 Error("CdWork", "Alien connection required");
801 return;
802 }
803 TString homedir = gGrid->GetHomeDirectory();
804 TString workdir = homedir + fGridWorkingDir;
923e2ca5 805 if (DirectoryExists(workdir)) {
806 gGrid->Cd(workdir);
807 return;
808 }
809 // Work directory not existing - create it
810 gGrid->Cd(homedir);
b93f8109 811 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 812 gGrid->Cd(fGridWorkingDir);
d3b18c4c 813 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 814 } else {
d3b18c4c 815 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 816 workdir.Data(), homedir.Data());
817 fGridWorkingDir = "";
818 }
c57f56b7 819}
820
348be253 821//______________________________________________________________________________
822Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
823{
824// Check if file copying is possible.
d3b18c4c 825 if (fProductionMode) return kTRUE;
b5ae4fe0 826 TString salienpath(alienpath);
827 if (salienpath.Contains(" ")) {
828 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
829 return kFALSE;
830 }
348be253 831 if (!Connect()) {
832 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
833 return kFALSE;
834 }
d3339be3 835 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
836 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 837 // Check if alien_CLOSE_SE is defined
838 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
839 if (!closeSE.IsNull()) {
840 Info("CheckFileCopy", "Your current close storage is pointing to: \
841 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
842 } else {
843 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
844 }
845 // Check if grid directory exists.
846 if (!DirectoryExists(alienpath)) {
847 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
848 return kFALSE;
849 }
b5ae4fe0 850 TString stest = "plugin_test_copy";
851 TFile f(stest, "RECREATE");
348be253 852 // User may not have write permissions to current directory
853 if (f.IsZombie()) {
854 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
855 gSystem->WorkingDirectory());
856 return kFALSE;
857 }
858 f.Close();
b5ae4fe0 859 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
860 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 861 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 862 \n# 1. Make sure you have write permissions there. If this is the case: \
863 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
864 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
865 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
866 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 867 gSystem->Unlink(stest.Data());
348be253 868 return kFALSE;
869 }
b5ae4fe0 870 gSystem->Unlink(stest.Data());
871 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 872 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 873 return kTRUE;
874}
875
c57f56b7 876//______________________________________________________________________________
877Bool_t AliAnalysisAlien::CheckInputData()
878{
879// Check validity of input data. If necessary, create xml files.
d3b18c4c 880 if (fProductionMode) return kTRUE;
a8739e8a 881 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 882 if (!fGridDataDir.Length()) {
883 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
884 return kFALSE;
885 }
9f020952 886 if (fMergeViaJDL) {
887 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
888 return kFALSE;
889 }
d2a409b2 890 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 891 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
892 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 893 return kTRUE;
c57f56b7 894 }
895 // Process declared files
a2f5fc01 896 Bool_t isCollection = kFALSE;
897 Bool_t isXml = kFALSE;
898 Bool_t useTags = kFALSE;
c57f56b7 899 Bool_t checked = kFALSE;
d3b18c4c 900 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 901 TString file;
902 TString workdir = gGrid->GetHomeDirectory();
903 workdir += fGridWorkingDir;
904 if (fInputFiles) {
905 TObjString *objstr;
906 TIter next(fInputFiles);
907 while ((objstr=(TObjString*)next())) {
908 file = workdir;
909 file += "/";
910 file += objstr->GetString();
911 // Store full lfn path
912 if (FileExists(file)) objstr->SetString(file);
913 else {
914 file = objstr->GetName();
915 if (!FileExists(objstr->GetName())) {
916 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
917 objstr->GetName(), workdir.Data());
918 return kFALSE;
919 }
920 }
921 Bool_t iscoll, isxml, usetags;
922 CheckDataType(file, iscoll, isxml, usetags);
923 if (!checked) {
924 checked = kTRUE;
a2f5fc01 925 isCollection = iscoll;
926 isXml = isxml;
927 useTags = usetags;
928 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 929 } else {
a2f5fc01 930 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 931 Error("CheckInputData", "Some conflict was found in the types of inputs");
932 return kFALSE;
933 }
934 }
935 }
936 }
937 // Process requested run numbers
a8739e8a 938 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 939 // Check validity of alien data directory
940 if (!fGridDataDir.Length()) {
941 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
942 return kFALSE;
943 }
923e2ca5 944 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 945 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
946 return kFALSE;
947 }
a2f5fc01 948 if (isCollection) {
c57f56b7 949 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
950 return kFALSE;
951 }
952
a2f5fc01 953 if (checked && !isXml) {
c57f56b7 954 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
955 return kFALSE;
956 }
957 // Check validity of run number(s)
958 TObjArray *arr;
959 TObjString *os;
74013535 960 TString format;
319593fb 961 Int_t nruns = 0;
904f9f5f 962 TString schunk, schunk2;
c57f56b7 963 TString path;
964 if (!checked) {
965 checked = kTRUE;
a2f5fc01 966 useTags = fDataPattern.Contains("tag");
967 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 968 }
a2f5fc01 969 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 970 Error("CheckInputData", "Cannot mix input files using/not using tags");
971 return kFALSE;
972 }
973 if (fRunNumbers.Length()) {
a8739e8a 974 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 975 arr = fRunNumbers.Tokenize(" ");
976 TIter next(arr);
977 while ((os=(TObjString*)next())) {
978 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 979 if (!DirectoryExists(path)) {
980 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 981 continue;
c57f56b7 982 }
983 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
984 TString msg = "\n##### file: ";
985 msg += path;
986 msg += " type: xml_collection;";
a2f5fc01 987 if (useTags) msg += " using_tags: Yes";
c57f56b7 988 else msg += " using_tags: No";
23329835 989 Info("CheckDataType", "%s", msg.Data());
319593fb 990 if (fNrunsPerMaster<2) {
d2a409b2 991 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 992 } else {
993 nruns++;
994 if (((nruns-1)%fNrunsPerMaster) == 0) {
995 schunk = os->GetString();
996 }
997 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
998 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 999 AddDataFile(schunk);
319593fb 1000 }
c57f56b7 1001 }
1002 delete arr;
a8739e8a 1003 } else {
1004 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1005 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1006 format = Form("%%s/%s ", fRunPrefix.Data());
1007 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1008 if (!DirectoryExists(path)) {
a8739e8a 1009 continue;
1010 }
74013535 1011 format = Form("%%s/%s.xml", fRunPrefix.Data());
1012 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1013 TString msg = "\n##### file: ";
1014 msg += path;
1015 msg += " type: xml_collection;";
a2f5fc01 1016 if (useTags) msg += " using_tags: Yes";
a8739e8a 1017 else msg += " using_tags: No";
23329835 1018 Info("CheckDataType", "%s", msg.Data());
319593fb 1019 if (fNrunsPerMaster<2) {
74013535 1020 format = Form("%s.xml", fRunPrefix.Data());
1021 AddDataFile(Form(format.Data(),irun));
319593fb 1022 } else {
1023 nruns++;
1024 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1025 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1026 }
74013535 1027 format = Form("_%s.xml", fRunPrefix.Data());
1028 schunk2 = Form(format.Data(), irun);
319593fb 1029 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1030 schunk += schunk2;
d2a409b2 1031 AddDataFile(schunk);
319593fb 1032 }
a8739e8a 1033 }
904f9f5f 1034 if (!fInputFiles) {
1035 schunk += schunk2;
1036 AddDataFile(schunk);
1037 }
c57f56b7 1038 }
1039 return kTRUE;
1040}
1041
21ca8e59 1042//______________________________________________________________________________
43b807f6 1043Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1044{
1045// Copy data from the given grid directory according a pattern and make a local
1046// dataset.
2ba4c287 1047// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1048 if (!Connect()) {
1049 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
1050 return kFALSE;
1051 }
1052 if (!DirectoryExists(griddir)) {
1053 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
1054 return kFALSE;
1055 }
1056 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1057 printf("Running command: %s\n", command.Data());
1058 TGridResult *res = gGrid->Command(command);
1059 Int_t nfound = res->GetEntries();
1060 if (!nfound) {
1061 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
1062 return kFALSE;
1063 }
1064 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1065
1066 // archives
1067 TObjArray* additionalArchives = 0;
1068 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1069 additionalArchives = TString(archivefile).Tokenize(";");
1070 archivefile = additionalArchives->At(0)->GetName();
1071 additionalArchives->RemoveAt(0);
1072 additionalArchives->Compress();
1073 }
1074
21ca8e59 1075 // Copy files locally
1076 ofstream out;
1077 out.open(output, ios::out);
1078 TMap *map;
1079 TString turl, dirname, filename, temp;
1080 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1081 gSystem->MakeDirectory(outputdir);
1082 gSystem->ChangeDirectory(outputdir);
21ca8e59 1083 for (Int_t i=0; i<nfound; i++) {
1084 map = (TMap*)res->At(i);
1085 turl = map->GetValue("turl")->GetName();
1086 filename = gSystem->BaseName(turl.Data());
1087 dirname = gSystem->DirName(turl.Data());
1088 dirname = gSystem->BaseName(dirname.Data());
1089 gSystem->MakeDirectory(dirname);
43b807f6 1090
1091 TString source(turl);
1092 TString targetFileName(filename);
1093
1094 if (strlen(archivefile) > 0) {
1095 // TODO here the archive in which the file resides should be determined
1096 // however whereis returns only a guid, and guid2lfn does not work
1097 // Therefore we use the one provided as argument for now
1098 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1099 targetFileName = archivefile;
1100 }
1101 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
2ba4c287 1102 Bool_t success = kTRUE;
1103 if (additionalArchives)
1104 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++)
1105 {
1106 TString target;
1107 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1108 gSystem->MakeDirectory(gSystem->DirName(target));
1109 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1110 }
1111
1112 if (success) {
1113 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1114 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1115 }
21ca8e59 1116 }
1117 }
1118 gSystem->ChangeDirectory(cdir);
1119 delete res;
2ba4c287 1120 if (additionalArchives)
1121 delete additionalArchives;
21ca8e59 1122 return kTRUE;
1123}
1124
c57f56b7 1125//______________________________________________________________________________
1126Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1127{
1128// Create dataset for the grid data directory + run number.
3af861ff 1129 const Int_t gMaxEntries = 15000;
d3b18c4c 1130 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1131 if (!Connect()) {
1132 Error("CreateDataset", "Cannot create dataset with no grid connection");
1133 return kFALSE;
1134 }
1135
1136 // Cd workspace
d3b18c4c 1137 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1138 TString workdir = gGrid->GetHomeDirectory();
1139 workdir += fGridWorkingDir;
1140
1141 // Compose the 'find' command arguments
74013535 1142 TString format;
c57f56b7 1143 TString command;
1144 TString options = "-x collection ";
bb885a9e 1145 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1146 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1147 TString conditions = "";
3af861ff 1148 Int_t nstart = 0;
1149 Int_t ncount = 0;
1150 Int_t stage = 0;
c57f56b7 1151 TString file;
1152 TString path;
319593fb 1153 Int_t nruns = 0;
904f9f5f 1154 TString schunk, schunk2;
ab254fd1 1155 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1156 if (!fRunNumbers.Length() && !fRunRange[0]) {
1157 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1158 // Make a single data collection from data directory.
1159 path = fGridDataDir;
923e2ca5 1160 if (!DirectoryExists(path)) {
d2a409b2 1161 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1162 return kFALSE;
1163 }
923e2ca5 1164// CdWork();
d2a409b2 1165 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1166 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1167 while (1) {
1168 ncount = 0;
1169 stage++;
1170 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1171 command = "find ";
1172 command += Form("%s -o %d ",options.Data(), nstart);
1173 command += path;
1174 command += " ";
1175 command += pattern;
1176 command += conditions;
1177 printf("command: %s\n", command.Data());
1178 TGridResult *res = gGrid->Command(command);
1179 if (res) delete res;
1180 // Write standard output to file
1181 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1182 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1183 Bool_t nullFile = kFALSE;
1184 if (!hasGrep) {
1185 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1186 } else {
1187 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1188 if (nullFile) {
1189 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1190 gSystem->Exec("rm -f __tmp*");
1191 return kFALSE;
1192 }
1193 TString line;
1194 ifstream in;
1195 in.open("__tmp__");
1196 in >> line;
1197 in.close();
1198 gSystem->Exec("rm -f __tmp__");
1199 ncount = line.Atoi();
1200 }
1201 }
1202 if (ncount == gMaxEntries) {
1203 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1204 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1205 if (!cbase) cbase = cadd;
1206 else {
1207 cbase->Add(cadd);
1208 delete cadd;
defd7a3a 1209 }
3af861ff 1210 nstart += ncount;
1211 } else {
1212 if (cbase) {
1213 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1214 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1215 cbase->Add(cadd);
1216 delete cadd;
1217 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1218 delete cbase; cbase = 0;
1219 } else {
1220 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1221 }
1222 gSystem->Exec("rm -f __tmp*");
1223 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1224 break;
1225 }
0f389141 1226 }
1227 Bool_t fileExists = FileExists(file);
1228 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1229 // Copy xml file to alien space
0f389141 1230 if (fileExists) gGrid->Rm(file);
d2a409b2 1231 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1232 if (!FileExists(file)) {
1233 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1234 return kFALSE;
1235 }
1236 // Update list of files to be processed.
1237 }
1238 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1239 return kTRUE;
1240 }
c57f56b7 1241 // Several runs
a2f5fc01 1242 Bool_t nullResult = kTRUE;
a8739e8a 1243 if (fRunNumbers.Length()) {
1244 TObjArray *arr = fRunNumbers.Tokenize(" ");
1245 TObjString *os;
1246 TIter next(arr);
1247 while ((os=(TObjString*)next())) {
3af861ff 1248 nstart = 0;
1249 stage = 0;
dc030a24 1250 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1251 if (!DirectoryExists(path)) continue;
1252// CdWork();
a8739e8a 1253 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1254 else file = Form("%s.xml", os->GetString().Data());
319593fb 1255 // If local collection file does not exist, create it via 'find' command.
3af861ff 1256 while (1) {
1257 ncount = 0;
1258 stage++;
1259 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1260 command = "find ";
1261 command += Form("%s -o %d ",options.Data(), nstart);
1262 command += path;
1263 command += pattern;
1264 command += conditions;
1265 TGridResult *res = gGrid->Command(command);
1266 if (res) delete res;
1267 // Write standard output to file
1268 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1269 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1270 Bool_t nullFile = kFALSE;
1271 if (!hasGrep) {
1272 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1273 } else {
1274 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1275 if (nullFile) {
1276 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1277 gSystem->Exec("rm -f __tmp*");
1278 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1279 break;
3af861ff 1280 }
1281 TString line;
1282 ifstream in;
1283 in.open("__tmp__");
1284 in >> line;
1285 in.close();
1286 gSystem->Exec("rm -f __tmp__");
1287 ncount = line.Atoi();
1288 }
1289 nullResult = kFALSE;
1290 }
1291 if (ncount == gMaxEntries) {
1292 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1293 if (fNrunsPerMaster > 1) {
1294 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1295 file.Data(),gMaxEntries);
1296 return kFALSE;
1297 }
1298 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1299 if (!cbase) cbase = cadd;
1300 else {
1301 cbase->Add(cadd);
1302 delete cadd;
defd7a3a 1303 }
3af861ff 1304 nstart += ncount;
1305 } else {
c89a84bc 1306 if (cbase && fNrunsPerMaster<2) {
3af861ff 1307 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1308 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1309 cbase->Add(cadd);
1310 delete cadd;
1311 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1312 delete cbase; cbase = 0;
1313 } else {
1314 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1315 }
1316 gSystem->Exec("rm -f __tmp*");
1317 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1318 break;
d3339be3 1319 }
3af861ff 1320 }
a8739e8a 1321 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1322 // Check if there is one run per master job.
1323 if (fNrunsPerMaster<2) {
1324 if (FileExists(file)) {
0f389141 1325 if (fOverwriteMode) gGrid->Rm(file);
1326 else {
1327 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1328 continue;
1329 }
319593fb 1330 }
1331 // Copy xml file to alien space
1332 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1333 if (!FileExists(file)) {
1334 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1335 delete arr;
1336 return kFALSE;
1337 }
1338 } else {
1339 nruns++;
1340 if (((nruns-1)%fNrunsPerMaster) == 0) {
1341 schunk = os->GetString();
1342 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1343 } else {
1344 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1345 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1346 cbase->Add(cadd);
1347 delete cadd;
1348 }
1349 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1350 continue;
1351 }
1352 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1353 if (FileExists(schunk)) {
1354 if (fOverwriteMode) gGrid->Rm(file);
1355 else {
1356 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1357 continue;
1358 }
319593fb 1359 }
84fcd93f 1360 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1361 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1362 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1363 if (!FileExists(schunk)) {
1364 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1365 delete arr;
1366 return kFALSE;
1367 }
d3339be3 1368 }
a8739e8a 1369 }
1370 delete arr;
a2f5fc01 1371 if (nullResult) {
d3339be3 1372 Error("CreateDataset", "No valid dataset corresponding to the query!");
1373 return kFALSE;
1374 }
a8739e8a 1375 } else {
1376 // Process a full run range.
1377 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1378 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1379 nstart = 0;
1380 stage = 0;
74013535 1381 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1382 if (!DirectoryExists(path)) continue;
1383// CdWork();
74013535 1384 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1385 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1386 else file = Form(format.Data(), irun);
0f389141 1387 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1388 if (fOverwriteMode) gGrid->Rm(file);
1389 else {
1390 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1391 continue;
1392 }
a8739e8a 1393 }
319593fb 1394 // If local collection file does not exist, create it via 'find' command.
3af861ff 1395 while (1) {
1396 ncount = 0;
1397 stage++;
1398 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1399 command = "find ";
1400 command += Form("%s -o %d ",options.Data(), nstart);
1401 command += path;
1402 command += pattern;
1403 command += conditions;
1404 TGridResult *res = gGrid->Command(command);
1405 if (res) delete res;
1406 // Write standard output to file
1407 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1408 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1409 Bool_t nullFile = kFALSE;
1410 if (!hasGrep) {
1411 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1412 } else {
1413 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1414 if (nullFile) {
1415 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1416 gSystem->Exec("rm -f __tmp*");
1604c9ed 1417 break;
3af861ff 1418 }
1419 TString line;
1420 ifstream in;
1421 in.open("__tmp__");
1422 in >> line;
1423 in.close();
1424 gSystem->Exec("rm -f __tmp__");
1425 ncount = line.Atoi();
1426 }
1427 nullResult = kFALSE;
1428 }
1429 if (ncount == gMaxEntries) {
1430 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1431 if (fNrunsPerMaster > 1) {
1432 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1433 file.Data(),gMaxEntries);
1434 return kFALSE;
1435 }
1436 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1437 if (!cbase) cbase = cadd;
1438 else {
1439 cbase->Add(cadd);
1440 delete cadd;
defd7a3a 1441 }
3af861ff 1442 nstart += ncount;
1443 } else {
c89a84bc 1444 if (cbase && fNrunsPerMaster<2) {
3af861ff 1445 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1446 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1447 cbase->Add(cadd);
1448 delete cadd;
1449 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1450 delete cbase; cbase = 0;
1451 } else {
1452 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1453 }
1454 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1455 break;
d3339be3 1456 }
319593fb 1457 }
a8739e8a 1458 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1459 // Check if there is one run per master job.
1460 if (fNrunsPerMaster<2) {
1461 if (FileExists(file)) {
0f389141 1462 if (fOverwriteMode) gGrid->Rm(file);
1463 else {
1464 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1465 continue;
1466 }
319593fb 1467 }
1468 // Copy xml file to alien space
1469 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1470 if (!FileExists(file)) {
1471 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1472 return kFALSE;
1473 }
1474 } else {
1475 nruns++;
95e5b448 1476 // Check if the collection for the chunk exist locally.
1477 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1478 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1479 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1480 else continue;
1481 }
84fcd93f 1482 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1483 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1484 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1485 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1486 } else {
1487 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1488 cbase->Add(cadd);
1489 delete cadd;
1490 }
74013535 1491 format = Form("%%s_%s.xml", fRunPrefix.Data());
1492 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1493 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1494 continue;
1495 }
904f9f5f 1496 schunk = schunk2;
319593fb 1497 if (FileExists(schunk)) {
0f389141 1498 if (fOverwriteMode) gGrid->Rm(schunk);
1499 else {
1500 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1501 continue;
1502 }
319593fb 1503 }
84fcd93f 1504 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1505 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1506 if (FileExists(schunk)) {
0f389141 1507 if (fOverwriteMode) gGrid->Rm(schunk);
1508 else {
1509 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1510 continue;
1511 }
95e5b448 1512 }
319593fb 1513 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1514 if (!FileExists(schunk)) {
1515 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1516 return kFALSE;
1517 }
1518 }
c57f56b7 1519 }
a2f5fc01 1520 if (nullResult) {
d3339be3 1521 Error("CreateDataset", "No valid dataset corresponding to the query!");
1522 return kFALSE;
1523 }
a8739e8a 1524 }
c57f56b7 1525 return kTRUE;
1526}
1527
1528//______________________________________________________________________________
1529Bool_t AliAnalysisAlien::CreateJDL()
1530{
1531// Generate a JDL file according to current settings. The name of the file is
1532// specified by fJDLName.
1533 Bool_t error = kFALSE;
1534 TObjArray *arr = 0;
1535 Bool_t copy = kTRUE;
d3b18c4c 1536 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1537 Bool_t generate = kTRUE;
1538 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1539 if (!Connect()) {
1540 Error("CreateJDL", "Alien connection required");
1541 return kFALSE;
1542 }
1543 // Check validity of alien workspace
d3b18c4c 1544 TString workdir;
1545 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1546 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1547 workdir += fGridWorkingDir;
1548 if (generate) {
1549 TObjString *os;
1550 if (!fInputFiles) {
1551 Error("CreateJDL()", "Define some input files for your analysis.");
1552 error = kTRUE;
1553 }
1554 // Compose list of input files
1555 // Check if output files were defined
1556 if (!fOutputFiles.Length()) {
1557 Error("CreateJDL", "You must define at least one output file");
1558 error = kTRUE;
1559 }
1560 // Check if an output directory was defined and valid
1561 if (!fGridOutputDir.Length()) {
1562 Error("CreateJDL", "You must define AliEn output directory");
1563 error = kTRUE;
1564 } else {
d3b18c4c 1565 if (!fProductionMode) {
1566 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1567 if (!DirectoryExists(fGridOutputDir)) {
1568 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1569 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1570 } else {
1571 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1572 // error = kTRUE;
1573 }
67a3b254 1574 } else {
1575 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1576 }
1577 gGrid->Cd(workdir);
1578 }
c57f56b7 1579 }
1580 // Exit if any error up to now
1581 if (error) return kFALSE;
1582 // Set JDL fields
0f389141 1583 if (!fUser.IsNull()) {
1584 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1585 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1586 }
1587 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1588 TString mergeExec = fExecutable;
1589 mergeExec.ReplaceAll(".sh", "_merge.sh");
1590 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1591 mergeExec.ReplaceAll(".sh", ".C");
1592 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1593 if (!fArguments.IsNull())
1594 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1595 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1596 else {
1597 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1598 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1599 }
b3e07543 1600
1f0d1ca2 1601 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1602 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1603 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1604 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1605
0f389141 1606 if (fMaxInitFailed > 0) {
c57f56b7 1607 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1608 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1609 }
1610 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1611 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1612 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1613 }
1614 if (!IsOneStageMerging()) {
f47d5cb4 1615 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1616 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1617 }
1618 if (fSplitMode.Length()) {
c57f56b7 1619 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1620 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1621 }
1622 fMergingJDL->SetValue("Split", "\"se\"");
1623 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1624 if (!fAliROOTVersion.IsNull()) {
1625 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1626 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1627 }
1628 if (!fROOTVersion.IsNull()) {
c57f56b7 1629 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1630 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1631 }
1632 if (!fAPIVersion.IsNull()) {
c57f56b7 1633 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1634 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1635 }
648174cf 1636 if (!fExternalPackages.IsNull()) {
1637 arr = fExternalPackages.Tokenize(" ");
1638 TIter next(arr);
1639 while ((os=(TObjString*)next())) {
1640 TString pkgname = os->GetString();
1641 Int_t index = pkgname.Index("::");
1642 TString pkgversion = pkgname(index+2, pkgname.Length());
1643 pkgname.Remove(index);
1644 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1645 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1646 }
1647 delete arr;
1648 }
0f389141 1649 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1650 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1651 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1652 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1653 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1654 TString analysisFile = fExecutable;
1655 analysisFile.ReplaceAll(".sh", ".root");
1656 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1657 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1658 if (fAdditionalLibs.Length()) {
1659 arr = fAdditionalLibs.Tokenize(" ");
1660 TIter next(arr);
1661 while ((os=(TObjString*)next())) {
1662 if (os->GetString().Contains(".so")) continue;
1663 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1664 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1665 }
1666 delete arr;
1667 }
4e5c5506 1668 if (fPackages) {
1669 TIter next(fPackages);
1670 TObject *obj;
0f389141 1671 while ((obj=next())) {
4e5c5506 1672 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1673 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1674 }
4e5c5506 1675 }
c57f56b7 1676 if (fOutputArchive.Length()) {
e4d9a069 1677 TString outputArchive = fOutputArchive;
1678 if (!fRegisterExcludes.IsNull()) {
1679 arr = fRegisterExcludes.Tokenize(" ");
1680 TIter next1(arr);
1681 while ((os=(TObjString*)next1())) {
1682 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1683 outputArchive.ReplaceAll(os->GetString(),"");
1684 }
1685 delete arr;
1686 }
1687 arr = outputArchive.Tokenize(" ");
c57f56b7 1688 TIter next(arr);
0f389141 1689 Bool_t first = kTRUE;
1690 const char *comment = "Files to be archived";
1691 const char *comment1 = comment;
1692 while ((os=(TObjString*)next())) {
1693 if (!first) comment = NULL;
1694 if (!os->GetString().Contains("@") && fCloseSE.Length())
1695 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1696 else
1697 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1698 first = kFALSE;
1699 }
c57f56b7 1700 delete arr;
f790bc1b 1701 // Output archive for the merging jdl
f790bc1b 1702 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1703 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1704 // Add normal output files, extra files + terminate files
1705 TString files = GetListOfFiles("outextter");
37d21c01 1706 // Do not register files in fRegisterExcludes
1707 if (!fRegisterExcludes.IsNull()) {
1708 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1709 TIter next1(arr);
1710 while ((os=(TObjString*)next1())) {
1711 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1712 files.ReplaceAll(os->GetString(),"");
1713 }
66d71516 1714 delete arr;
0f389141 1715 }
f790bc1b 1716 files.ReplaceAll(".root", "*.root");
b83f84f0 1717 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1718 } else {
95f84779 1719 TString files = fOutputArchive;
1720 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1721 outputArchive = files;
f790bc1b 1722 }
0f389141 1723 arr = outputArchive.Tokenize(" ");
705adb3e 1724 TIter next2(arr);
0f389141 1725 comment = comment1;
1726 first = kTRUE;
705adb3e 1727 while ((os=(TObjString*)next2())) {
0f389141 1728 if (!first) comment = NULL;
0cdf65a8 1729 TString currentfile = os->GetString();
0cdf65a8 1730 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1731 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1732 else
0cdf65a8 1733 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1734 first = kFALSE;
1735 }
1736 delete arr;
c57f56b7 1737 }
149d288c 1738 arr = fOutputFiles.Tokenize(",");
c57f56b7 1739 TIter next(arr);
0f389141 1740 Bool_t first = kTRUE;
f790bc1b 1741 const char *comment = "Files to be saved";
43da816a 1742 while ((os=(TObjString*)next())) {
e1eaf596 1743 // Ignore ouputs in jdl that are also in outputarchive
1744 TString sout = os->GetString();
f790bc1b 1745 sout.ReplaceAll("*", "");
1746 sout.ReplaceAll(".root", "");
e1eaf596 1747 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1748 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1749 // Ignore fRegisterExcludes
1750 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1751 if (!first) comment = NULL;
43da816a 1752 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1753 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1754 else
0f389141 1755 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1756 first = kFALSE;
f790bc1b 1757 if (fMergeExcludes.Contains(sout)) continue;
1758 if (!os->GetString().Contains("@") && fCloseSE.Length())
1759 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1760 else
1761 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1762 }
c57f56b7 1763 delete arr;
0f389141 1764 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1765 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1766 TString validationScript = fValidationScript;
0f389141 1767 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1768 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1769 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1770 if (fMasterResubmitThreshold) {
1771 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1772 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1773 }
d2a409b2 1774 // Write a jdl with 2 input parameters: collection name and output dir name.
1775 WriteJDL(copy);
c57f56b7 1776 }
1777 // Copy jdl to grid workspace
a8739e8a 1778 if (copy) {
b5e4aaa7 1779 // Check if an output directory was defined and valid
1780 if (!fGridOutputDir.Length()) {
1781 Error("CreateJDL", "You must define AliEn output directory");
1782 return kFALSE;
1783 } else {
1784 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1785 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1786 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1787 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1788 } else {
1789 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1790 return kFALSE;
1791 }
1792 }
1793 gGrid->Cd(workdir);
1794 }
648174cf 1795 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1796 TString mergeJDLName = fExecutable;
1797 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1798 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1799 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1800 if (fProductionMode) {
648174cf 1801 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1802 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1803 }
648174cf 1804 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1805 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1806 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1807 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1808 Fatal("","Terminating");
1809// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1810 if (fMergeViaJDL) {
1811 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1812// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1813 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1814 Fatal("","Terminating");
0f389141 1815 }
648174cf 1816 }
c57f56b7 1817 if (fAdditionalLibs.Length()) {
e7c71df0 1818 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1819 TObjString *os;
1820 TIter next(arr);
1821 while ((os=(TObjString*)next())) {
c57f56b7 1822 if (os->GetString().Contains(".so")) continue;
4e5c5506 1823 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1824 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1825// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1826 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1827 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1828 Fatal("","Terminating");
c57f56b7 1829 }
1830 delete arr;
1831 }
4e5c5506 1832 if (fPackages) {
1833 TIter next(fPackages);
1834 TObject *obj;
1835 while ((obj=next())) {
fdbbc7be 1836 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1837 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1838// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1839 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1840 Form("%s/%s", workdir.Data(), obj->GetName())))
1841 Fatal("","Terminating");
4e5c5506 1842 }
1843 }
c57f56b7 1844 }
1845 return kTRUE;
1846}
1847
a8739e8a 1848//______________________________________________________________________________
d2a409b2 1849Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1850{
1851// Writes one or more JDL's corresponding to findex. If findex is negative,
1852// all run numbers are considered in one go (jdl). For non-negative indices
1853// they correspond to the indices in the array fInputFiles.
1854 if (!fInputFiles) return kFALSE;
d3b18c4c 1855 TObject *os;
1856 TString workdir;
1857 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1858 workdir += fGridWorkingDir;
f47d5cb4 1859 TString stageName = "$2";
1860 if (fProductionMode) stageName = "$4";
1861 if (!fMergeDirName.IsNull()) {
1862 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1863 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1864 } else {
1865 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1866 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1867 }
d3b18c4c 1868 if (fProductionMode) {
a8739e8a 1869 TIter next(fInputFiles);
b3e07543 1870 while ((os=next())) {
d3b18c4c 1871 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1872 }
56754bf6 1873 if (!fOutputToRunNo)
1874 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1875 else
1876 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1877 } else {
1878 if (!fRunNumbers.Length() && !fRunRange[0]) {
1879 // One jdl with no parameters in case input data is specified by name.
1880 TIter next(fInputFiles);
1881 while ((os=next()))
1882 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1883 if (!fOutputSingle.IsNull())
1884 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1885 else {
1886 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1887 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1888 }
1889 } else {
1890 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1891 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1892 if (!fOutputSingle.IsNull()) {
1893 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1894 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1895 } else {
1896 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1897 }
1898 }
a8739e8a 1899 }
1900
a8739e8a 1901 // Generate the JDL as a string
1902 TString sjdl = fGridJDL->Generate();
0f389141 1903 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1904 // Final merge jdl
1905 if (!fMergeDirName.IsNull()) {
1906 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1907 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1908 } else {
1909 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1910 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1911 }
b3e07543 1912 TString sjdl2 = fMergingJDL->Generate();
1913 Int_t index, index1;
a8739e8a 1914 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1915 sjdl.ReplaceAll("(member", "\n (member");
1916 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1917 sjdl.ReplaceAll("{", "{\n ");
1918 sjdl.ReplaceAll("};", "\n};");
1919 sjdl.ReplaceAll("{\n \n", "{\n");
1920 sjdl.ReplaceAll("\n\n", "\n");
1921 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1922 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1923 sjdl1.ReplaceAll("(member", "\n (member");
1924 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1925 sjdl1.ReplaceAll("{", "{\n ");
1926 sjdl1.ReplaceAll("};", "\n};");
1927 sjdl1.ReplaceAll("{\n \n", "{\n");
1928 sjdl1.ReplaceAll("\n\n", "\n");
1929 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1930 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1931 sjdl2.ReplaceAll("(member", "\n (member");
1932 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1933 sjdl2.ReplaceAll("{", "{\n ");
1934 sjdl2.ReplaceAll("};", "\n};");
1935 sjdl2.ReplaceAll("{\n \n", "{\n");
1936 sjdl2.ReplaceAll("\n\n", "\n");
1937 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1938 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1939 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1940 index = sjdl.Index("JDLVariables");
1941 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1942 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1943 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1944 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1945 index = fJobTag.Index(":");
1946 if (index < 0) index = fJobTag.Length();
1947 TString jobTag = fJobTag;
f47d5cb4 1948 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1949 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1950 if (fProductionMode) {
1951 sjdl1.Prepend("# Generated merging jdl (production mode) \
1952 \n# $1 = full alien path to output directory to be merged \
1953 \n# $2 = train number \
1954 \n# $3 = production (like LHC10b) \
1955 \n# $4 = merging stage \
1956 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1957 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1958 sjdl2.Prepend("# Generated merging jdl \
1959 \n# $1 = full alien path to output directory to be merged \
1960 \n# $2 = train number \
1961 \n# $3 = production (like LHC10b) \
1962 \n# $4 = merging stage \
1963 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1964 } else {
1965 sjdl1.Prepend("# Generated merging jdl \
1966 \n# $1 = full alien path to output directory to be merged \
1967 \n# $2 = merging stage \
03335324 1968 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1969 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1970 sjdl2.Prepend("# Generated merging jdl \
1971 \n# $1 = full alien path to output directory to be merged \
1972 \n# $2 = merging stage \
03335324 1973 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1974 }
0f389141 1975 index = sjdl1.Index("JDLVariables");
1976 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1977 index = sjdl2.Index("JDLVariables");
1978 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1979 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1980 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1981 index = sjdl2.Index("Split =");
1982 if (index>=0) {
1983 index1 = sjdl2.Index("\n", index);
1984 sjdl2.Remove(index, index1-index+1);
1985 }
1986 index = sjdl2.Index("SplitMaxInputFileNumber");
1987 if (index>=0) {
1988 index1 = sjdl2.Index("\n", index);
1989 sjdl2.Remove(index, index1-index+1);
1990 }
1991 index = sjdl2.Index("InputDataCollection");
1992 if (index>=0) {
1993 index1 = sjdl2.Index(";", index);
1994 sjdl2.Remove(index, index1-index+1);
1995 }
1996 index = sjdl2.Index("InputDataListFormat");
1997 if (index>=0) {
1998 index1 = sjdl2.Index("\n", index);
1999 sjdl2.Remove(index, index1-index+1);
2000 }
2001 index = sjdl2.Index("InputDataList");
2002 if (index>=0) {
2003 index1 = sjdl2.Index("\n", index);
2004 sjdl2.Remove(index, index1-index+1);
2005 }
f47d5cb4 2006 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2007 // Write jdl to file
a8739e8a 2008 ofstream out;
d2a409b2 2009 out.open(fJDLName.Data(), ios::out);
a8739e8a 2010 if (out.bad()) {
d3b18c4c 2011 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2012 return kFALSE;
2013 }
2014 out << sjdl << endl;
b3e07543 2015 out.close();
0f389141 2016 TString mergeJDLName = fExecutable;
2017 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2018 if (fMergeViaJDL) {
2019 ofstream out1;
2020 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2021 if (out1.bad()) {
d3b18c4c 2022 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2023 return kFALSE;
2024 }
2025 out1 << sjdl1 << endl;
b3e07543 2026 out1.close();
2027 ofstream out2;
2028 TString finalJDL = mergeJDLName;
2029 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2030 out2.open(finalJDL.Data(), ios::out);
2031 if (out2.bad()) {
2032 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2033 return kFALSE;
2034 }
2035 out2 << sjdl2 << endl;
2036 out2.close();
0f389141 2037 }
a8739e8a 2038
2039 // Copy jdl to grid workspace
2040 if (!copy) {
d3b18c4c 2041 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2042 } else {
d2a409b2 2043 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2044 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2045 TString finalJDL = mergeJDLName;
2046 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2047 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2048 if (fProductionMode) {
b5fe9cba 2049 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2050 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2051 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2052 }
d2a409b2 2053 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2054 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2055 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2056 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2057// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2058 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2059 Fatal("","Terminating");
0f389141 2060 if (fMergeViaJDL) {
b3e07543 2061 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2062// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2063// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2064 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2065 Fatal("","Terminating");
2066 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2067 Fatal("","Terminating");
0f389141 2068 }
a8739e8a 2069 }
2070 return kTRUE;
2071}
2072
c57f56b7 2073//______________________________________________________________________________
5513444a 2074Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2075{
2076// Returns true if file exists.
5513444a 2077 if (!gGrid) return kFALSE;
2ea55496 2078 TString slfn = lfn;
2079 slfn.ReplaceAll("alien://","");
2080 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2081 if (!res) return kFALSE;
2082 TMap *map = dynamic_cast<TMap*>(res->At(0));
2083 if (!map) {
2084 delete res;
2085 return kFALSE;
2086 }
2087 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2088 if (!objs || !objs->GetString().Length()) {
2089 delete res;
2090 return kFALSE;
2091 }
2092 delete res;
2093 return kTRUE;
2094}
2095
923e2ca5 2096//______________________________________________________________________________
2097Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2098{
2099// Returns true if directory exists. Can be also a path.
2100 if (!gGrid) return kFALSE;
2101 // Check if dirname is a path
2102 TString dirstripped = dirname;
2103 dirstripped = dirstripped.Strip();
2104 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2105 TString dir = gSystem->BaseName(dirstripped);
2106 dir += "/";
2107 TString path = gSystem->DirName(dirstripped);
2108 TGridResult *res = gGrid->Ls(path, "-F");
2109 if (!res) return kFALSE;
2110 TIter next(res);
2111 TMap *map;
2112 TObject *obj;
2113 while ((map=dynamic_cast<TMap*>(next()))) {
2114 obj = map->GetValue("name");
2115 if (!obj) break;
2116 if (dir == obj->GetName()) {
2117 delete res;
2118 return kTRUE;
2119 }
2120 }
2121 delete res;
2122 return kFALSE;
2123}
2124
c57f56b7 2125//______________________________________________________________________________
a2f5fc01 2126void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2127{
2128// Check input data type.
a2f5fc01 2129 isCollection = kFALSE;
2130 isXml = kFALSE;
2131 useTags = kFALSE;
c57f56b7 2132 if (!gGrid) {
2133 Error("CheckDataType", "No connection to grid");
2134 return;
2135 }
a2f5fc01 2136 isCollection = IsCollection(lfn);
c57f56b7 2137 TString msg = "\n##### file: ";
2138 msg += lfn;
a2f5fc01 2139 if (isCollection) {
c57f56b7 2140 msg += " type: raw_collection;";
2141 // special treatment for collections
a2f5fc01 2142 isXml = kFALSE;
c57f56b7 2143 // check for tag files in the collection
2144 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2145 if (!res) {
2146 msg += " using_tags: No (unknown)";
23329835 2147 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2148 return;
2149 }
2150 const char* typeStr = res->GetKey(0, "origLFN");
2151 if (!typeStr || !strlen(typeStr)) {
2152 msg += " using_tags: No (unknown)";
23329835 2153 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2154 return;
2155 }
2156 TString file = typeStr;
a2f5fc01 2157 useTags = file.Contains(".tag");
2158 if (useTags) msg += " using_tags: Yes";
c57f56b7 2159 else msg += " using_tags: No";
23329835 2160 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2161 return;
2162 }
2163 TString slfn(lfn);
2164 slfn.ToLower();
a2f5fc01 2165 isXml = slfn.Contains(".xml");
2166 if (isXml) {
c57f56b7 2167 // Open xml collection and check if there are tag files inside
2168 msg += " type: xml_collection;";
2169 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2170 if (!coll) {
2171 msg += " using_tags: No (unknown)";
23329835 2172 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2173 return;
2174 }
2175 TMap *map = coll->Next();
2176 if (!map) {
2177 msg += " using_tags: No (unknown)";
23329835 2178 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2179 return;
2180 }
2181 map = (TMap*)map->GetValue("");
2182 TString file;
2183 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2184 useTags = file.Contains(".tag");
c57f56b7 2185 delete coll;
a2f5fc01 2186 if (useTags) msg += " using_tags: Yes";
c57f56b7 2187 else msg += " using_tags: No";
23329835 2188 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2189 return;
2190 }
a2f5fc01 2191 useTags = slfn.Contains(".tag");
c57f56b7 2192 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2193 else msg += " type: unknown file;";
a2f5fc01 2194 if (useTags) msg += " using_tags: Yes";
c57f56b7 2195 else msg += " using_tags: No";
23329835 2196 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2197}
2198
4e5c5506 2199//______________________________________________________________________________
2200void AliAnalysisAlien::EnablePackage(const char *package)
2201{
2202// Enables a par file supposed to exist in the current directory.
2203 TString pkg(package);
2204 pkg.ReplaceAll(".par", "");
2205 pkg += ".par";
2206 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2207 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2208 return;
2209 }
2210 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2211 Info("EnablePackage", "AliEn plugin will use .par packages");
2212 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2213 if (!fPackages) {
2214 fPackages = new TObjArray();
2215 fPackages->SetOwner();
2216 }
2217 fPackages->Add(new TObjString(pkg));
2218}
2219
3bdcb562 2220//______________________________________________________________________________
2221TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2222{
2223// Make a tree from files having the location specified in fFileForTestMode.
2224// Inspired from JF's CreateESDChain.
2225 if (fFileForTestMode.IsNull()) {
2226 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2227 return NULL;
2228 }
2229 if (gSystem->AccessPathName(fFileForTestMode)) {
2230 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2231 return NULL;
2232 }
2233 // Open the file
2234 ifstream in;
2235 in.open(fFileForTestMode);
2236 Int_t count = 0;
2237 // Read the input list of files and add them to the chain
b505f1f8 2238 TString line;
2239 TChain *chain = new TChain(treeName);
2240 TChain *chainFriend = 0;
2241 if (!fFriendChainName.IsNull()) chainFriend = new TChain(treeName);
2242 while (in.good())
2243 {
3bdcb562 2244 in >> line;
2245 if (line.IsNull()) continue;
2246 if (count++ == fNtestFiles) break;
2247 TString esdFile(line);
2248 TFile *file = TFile::Open(esdFile);
b505f1f8 2249 if (file && !file->IsZombie()) {
2250 chain->Add(esdFile);
3bdcb562 2251 file->Close();
b505f1f8 2252 if (!fFriendChainName.IsNull()) {
4b5d9e21 2253 if (esdFile.Index("#") > -1)
2254 esdFile.Remove(esdFile.Index("#"));
2255 esdFile = gSystem->DirName(esdFile);
2256 esdFile += "/" + fFriendChainName;
b505f1f8 2257 file = TFile::Open(esdFile);
2258 if (file && !file->IsZombie()) {
2259 file->Close();
2260 chainFriend->Add(esdFile);
2261 } else {
2262 Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data());
2263 return 0;
2264 }
4b5d9e21 2265 }
3bdcb562 2266 } else {
2267 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2268 }
b505f1f8 2269 }
2270 in.close();
2271 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2272 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2273 delete chain;
b505f1f8 2274 delete chainFriend;
3bdcb562 2275 return NULL;
b505f1f8 2276 }
3bdcb562 2277// chain->ls();
b505f1f8 2278 if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend);
2279 return chain;
3bdcb562 2280}
2281
d2a409b2 2282//______________________________________________________________________________
2283const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2284{
2285// Get job status for all jobs with jobid>jobidstart.
2286 static char mstatus[20];
2287 mstatus[0] = '\0';
2288 nrunning = 0;
2289 nwaiting = 0;
2290 nerror = 0;
2291 ndone = 0;
2292 TGridJobStatusList *list = gGrid->Ps("");
2293 if (!list) return mstatus;
2294 Int_t nentries = list->GetSize();
2295 TGridJobStatus *status;
2296 Int_t pid;
2297 for (Int_t ijob=0; ijob<nentries; ijob++) {
2298 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2299 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2300 if (pid<jobidstart) continue;
2301 if (pid == lastid) {
8ddad121 2302 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2303 }
2304 switch (status->GetStatus()) {
2305 case TGridJobStatus::kWAITING:
2306 nwaiting++; break;
2307 case TGridJobStatus::kRUNNING:
2308 nrunning++; break;
2309 case TGridJobStatus::kABORTED:
2310 case TGridJobStatus::kFAIL:
2311 case TGridJobStatus::kUNKNOWN:
2312 nerror++; break;
2313 case TGridJobStatus::kDONE:
2314 ndone++;
2315 }
2316 }
2317 list->Delete();
2318 delete list;
2319 return mstatus;
2320}
2321
c57f56b7 2322//______________________________________________________________________________
2323Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2324{
2325// Returns true if file is a collection. Functionality duplicated from
2326// TAlien::Type() because we don't want to directly depend on TAlien.
2327 if (!gGrid) {
2328 Error("IsCollection", "No connection to grid");
2329 return kFALSE;
2330 }
2331 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2332 if (!res) return kFALSE;
2333 const char* typeStr = res->GetKey(0, "type");
2334 if (!typeStr || !strlen(typeStr)) return kFALSE;
2335 if (!strcmp(typeStr, "collection")) return kTRUE;
2336 delete res;
2337 return kFALSE;
2338}
2339
fe2d7fc2 2340//______________________________________________________________________________
2341Bool_t AliAnalysisAlien::IsSingleOutput() const
2342{
2343// Check if single-ouput option is on.
2344 return (!fOutputSingle.IsNull());
2345}
2346
16a4353c 2347//______________________________________________________________________________
2348void AliAnalysisAlien::Print(Option_t *) const
2349{
2350// Print current plugin settings.
84fcd93f 2351 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2352 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2353 if (mgr && mgr->IsProofMode()) {
2354 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2355 if (TestBit(AliAnalysisGrid::kTest))
2356 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2357 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2358 if (!fProofDataSet.IsNull())
2359 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2360 if (fProofReset==1)
2361 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2362 if (fProofReset>1)
2363 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2364 if (!fRootVersionForProof.IsNull())
2365 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2366 else
2367 printf("= ROOT version requested________________________ default\n");
2368 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2369 if (!fAliRootMode.IsNull())
2370 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2371 if (fNproofWorkers)
2372 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2373 if (fNproofWorkersPerSlave)
2374 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2375 if (TestSpecialBit(kClearPackages))
2376 printf("= ClearPackages requested...\n");
2377 if (fIncludePath.Data())
2378 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2379 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2380 if (fPackages && fPackages->GetEntries()) {
2381 TIter next(fPackages);
2382 TObject *obj;
2383 TString list;
2384 while ((obj=next())) list += obj->GetName();
2385 printf("= Par files to be used: ________________________ %s\n", list.Data());
2386 }
2387 if (TestSpecialBit(kProofConnectGrid))
2388 printf("= Requested PROOF connection to grid\n");
2389 return;
2390 }
e1c22e21 2391 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2392 if (fOverwriteMode) {
2393 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2394 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2395 }
348be253 2396 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2397 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2398 printf("= Production mode:______________________________ %d\n", fProductionMode);
2399 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2400 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2401 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2402 if (fUser.Length())
84fcd93f 2403 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2404 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2405 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2406 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2407 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2408 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2409 if (fRunNumbers.Length())
84fcd93f 2410 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2411 if (fRunRange[0])
74013535 2412 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2413 if (!fRunRange[0] && !fRunNumbers.Length()) {
2414 TIter next(fInputFiles);
2415 TObject *obj;
2416 TString list;
2417 while ((obj=next())) list += obj->GetName();
84fcd93f 2418 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2419 }
2420 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2421 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2422 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2423 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2424 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2425 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2426 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2427 printf("=====================================================================\n");
2428 printf("= Job price: ___________________________________ %d\n", fPrice);
2429 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2430 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2431 if (fMaxInitFailed>0)
84fcd93f 2432 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2433 if (fMasterResubmitThreshold>0)
84fcd93f 2434 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2435 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2436 if (fNrunsPerMaster>0)
84fcd93f 2437 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2438 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2439 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2440 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2441 if (fArguments.Length())
84fcd93f 2442 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2443 if (fExecutableArgs.Length())
2444 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2445 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2446 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2447 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2448 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2449 if (fDatasetName)
84fcd93f 2450 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2451 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2452 if (fIncludePath.Data())
84fcd93f 2453 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2454 if (fCloseSE.Length())
84fcd93f 2455 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2456 if (fFriendChainName.Length())
84fcd93f 2457 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2458 if (fPackages && fPackages->GetEntries()) {
16a4353c 2459 TIter next(fPackages);
2460 TObject *obj;
2461 TString list;
2462 while ((obj=next())) list += obj->GetName();
84fcd93f 2463 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2464 }
2465}
2466
c57f56b7 2467//______________________________________________________________________________
2468void AliAnalysisAlien::SetDefaults()
2469{
2470// Set default values for everything. What cannot be filled will be left empty.
2471 if (fGridJDL) delete fGridJDL;
2472 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2473 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2474 fPrice = 1;
2475 fTTL = 30000;
2476 fSplitMaxInputFileNumber = 100;
2477 fMaxInitFailed = 0;
2478 fMasterResubmitThreshold = 0;
bb885a9e 2479 fNtestFiles = 10;
149d288c 2480 fNreplicas = 2;
a8739e8a 2481 fRunRange[0] = 0;
2482 fRunRange[1] = 0;
8e3bd06a 2483 fRunPrefix = "%d";
319593fb 2484 fNrunsPerMaster = 1;
16a4353c 2485 fMaxMergeFiles = 100;
c57f56b7 2486 fRunNumbers = "";
2487 fExecutable = "analysis.sh";
0a1c1f7f 2488 fExecutableCommand = "root -b -q";
c57f56b7 2489 fArguments = "";
631c0b05 2490 fExecutableArgs = "";
c57f56b7 2491 fAnalysisMacro = "myAnalysis.C";
2492 fAnalysisSource = "";
2493 fAdditionalLibs = "";
2494 fSplitMode = "se";
2495 fAPIVersion = "";
2496 fROOTVersion = "";
2497 fAliROOTVersion = "";
2498 fUser = ""; // Your alien user name
2499 fGridWorkingDir = "";
2500 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2501 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2502 fFriendChainName = "";
c57f56b7 2503 fGridOutputDir = "output";
b83f84f0 2504 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2505 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2506 fInputFormat = "xml-single";
2507 fJDLName = "analysis.jdl";
c6cb3634 2508 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2509 fMergeExcludes = "";
0f389141 2510 fMergeViaJDL = 0;
348be253 2511 SetUseCopy(kTRUE);
2512 SetCheckCopy(kTRUE);
149d288c 2513 SetDefaultOutputs(kTRUE);
e1c22e21 2514 fOverwriteMode = 1;
c57f56b7 2515}
2516
0f389141 2517//______________________________________________________________________________
b3e07543 2518Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2519{
b3e07543 2520// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2521 // First check if the result is already in the output directory.
2522 if (FileExists(Form("%s/%s",aliendir,filename))) {
2523 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2524 return kFALSE;
2525 }
b3e07543 2526 // Now check the last stage done.
2527 Int_t stage = 0;
2528 while (1) {
2529 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2530 stage++;
2531 }
2532 // Next stage of merging
2533 stage++;
2534 TString pattern = "*root_archive.zip";
2535 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2536 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2537 if (res) delete res;
2538 // Write standard output to file
2539 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2540 // Count the number of files inside
2541 ifstream ifile;
2542 ifile.open(Form("Stage_%d.xml",stage));
2543 if (!ifile.good()) {
2544 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2545 return kFALSE;
b3e07543 2546 }
2547 TString line;
2548 Int_t nfiles = 0;
2549 while (!ifile.eof()) {
2550 ifile >> line;
2551 if (line.Contains("/event")) nfiles++;
7c2cd90a 2552 }
b3e07543 2553 ifile.close();
2554 if (!nfiles) {
2555 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2556 return kFALSE;
2557 } else {
2558 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2559 }
b3e07543 2560 // Copy the file in the output directory
2561 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2562// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2563 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2564 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2565 // Check if this is the last stage to be done.
2566 Bool_t laststage = (nfiles<nperchunk);
2567 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2568 if (laststage) {
2569 printf("### Submiting final merging stage %d\n", stage);
2570 TString finalJDL = jdl;
2571 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2572 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2573 Int_t jobId = SubmitSingleJob(query);
2574 if (!jobId) return kFALSE;
2575 } else {
2576 printf("### Submiting merging stage %d\n", stage);
03335324 2577 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2578 Int_t jobId = SubmitSingleJob(query);
2579 if (!jobId) return kFALSE;
7c2cd90a 2580 }
b3e07543 2581 return kTRUE;
2582}
7c2cd90a 2583
105631b5 2584//______________________________________________________________________________
2585AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2586{
2587// Loat the analysis manager from a file.
2588 TFile *file = TFile::Open(fname);
2589 if (!file) {
2590 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2591 return 0;
2592 }
2593 TIter nextkey(file->GetListOfKeys());
2594 AliAnalysisManager *mgr = 0;
2595 TKey *key;
2596 while ((key=(TKey*)nextkey())) {
2597 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2598 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2599 }
2600 if (!mgr)
2601 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2602 return mgr;
2603}
2604
7c2cd90a 2605//______________________________________________________________________________
2606Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2607{
2608// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2609 if (!gGrid) return 0;
2610 printf("=> %s ------> ",query);
2611 TGridResult *res = gGrid->Command(query);
2612 if (!res) return 0;
2613 TString jobId = res->GetKey(0,"jobId");
2614 delete res;
2615 if (jobId.IsNull()) {
2616 printf("submission failed. Reason:\n");
2617 gGrid->Stdout();
2618 gGrid->Stderr();
2619 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2620 return 0;
2621 }
2622 printf(" Job id: %s\n", jobId.Data());
2623 return atoi(jobId);
2624}
2625
2626//______________________________________________________________________________
b3e07543 2627Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2628{
b3e07543 2629// Merge given output files from basedir. Basedir can be an alien output directory
2630// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2631// files in a group (ignored for xml input). Merging can be done in stages:
2632// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2633// stage=1 : works with an xml of all root_archive.zip in the output directory
2634// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2635 TString outputFile = output;
0f389141 2636 TString command;
a2f5fc01 2637 TString outputChunk;
2638 TString previousChunk = "";
b3e07543 2639 TObjArray *listoffiles = new TObjArray();
2640// listoffiles->SetOwner();
a2f5fc01 2641 Int_t countChunk = 0;
2642 Int_t countZero = nmaxmerge;
0f389141 2643 Bool_t merged = kTRUE;
a2f5fc01 2644 Int_t index = outputFile.Index("@");
2645 if (index > 0) outputFile.Remove(index);
7c2cd90a 2646 TString inputFile = outputFile;
b3e07543 2647 TString sbasedir = basedir;
2648 if (sbasedir.Contains(".xml")) {
2649 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2650 nmaxmerge = 9999999;
2651 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2652 if (!coll) {
2653 ::Error("MergeOutput", "Input XML collection empty.");
2654 return kFALSE;
2655 }
2656 // Iterate grid collection
2657 while (coll->Next()) {
2658 TString fname = gSystem->DirName(coll->GetTURL());
2659 fname += "/";
2660 fname += inputFile;
2661 listoffiles->Add(new TNamed(fname.Data(),""));
2662 }
2663 } else {
2664 command = Form("find %s/ *%s", basedir, inputFile.Data());
2665 printf("command: %s\n", command.Data());
2666 TGridResult *res = gGrid->Command(command);
2667 if (!res) {
2668 ::Error("MergeOutput","No result for the find command\n");
2669 delete listoffiles;
2670 return kFALSE;
2671 }
2672 TIter nextmap(res);
2673 TMap *map = 0;
2674 while ((map=(TMap*)nextmap())) {
2675 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2676 if (!objs || !objs->GetString().Length()) {
2677 // Nothing found - skip this output
2678 delete res;
2679 delete listoffiles;
2680 return kFALSE;
2681 }
2682 listoffiles->Add(new TNamed(objs->GetName(),""));
2683 }
2684 delete res;
2685 }
2686 if (!listoffiles->GetEntries()) {
7c2cd90a 2687 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2688 delete listoffiles;
0f389141 2689 return kFALSE;
2690 }
2691
2692 TFileMerger *fm = 0;
b3e07543 2693 TIter next0(listoffiles);
2694 TObjArray *listoffilestmp = new TObjArray();
2695 listoffilestmp->SetOwner();
2696 TObject *nextfile;
2697 TString snextfile;
2698 // Keep only the files at upper level
2699 Int_t countChar = 0;
2700 while ((nextfile=next0())) {
2701 snextfile = nextfile->GetName();
2702 Int_t crtCount = snextfile.CountChar('/');
2703 if (nextfile == listoffiles->First()) countChar = crtCount;
2704 if (crtCount < countChar) countChar = crtCount;
2705 }
2706 next0.Reset();
2707 while ((nextfile=next0())) {
2708 snextfile = nextfile->GetName();
2709 Int_t crtCount = snextfile.CountChar('/');
2710 if (crtCount > countChar) {
2711 delete nextfile;
2712 continue;
2713 }
2714 listoffilestmp->Add(nextfile);
2715 }
2716 delete listoffiles;
2717 listoffiles = listoffilestmp; // Now contains 'good' files
2718 listoffiles->Print();
2719 TIter next(listoffiles);
7c2cd90a 2720 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2721 outputChunk = outputFile;
2722 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2723 // Check for existent temporary merge files
e1c22e21 2724 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2725 // Preserve old merging functionality for stage 0.
2726 if (stage==0) {
2727 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2728 while (1) {
2729 // Skip as many input files as in a chunk
2ea55496 2730 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2731 nextfile = next();
2732 if (!nextfile) {
2ea55496 2733 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2734 delete listoffiles;
2ea55496 2735 return kFALSE;
2736 }
b3e07543 2737 snextfile = nextfile->GetName();
7c2cd90a 2738 }
2739 outputChunk = outputFile;
2740 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2741 countChunk++;
2742 if (gSystem->AccessPathName(outputChunk)) continue;
2743 // Merged file with chunks up to <countChunk> found
2744 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2745 previousChunk = outputChunk;
2746 break;
2747 }
2748 }
2749 countZero = nmaxmerge;
2750
b3e07543 2751 while ((nextfile=next())) {
2752 snextfile = nextfile->GetName();
2ea55496 2753 // Loop 'find' results and get next LFN
7c2cd90a 2754 if (countZero == nmaxmerge) {
2755 // First file in chunk - create file merger and add previous chunk if any.
2b895f34 2756 fm = new TFileMerger(kTRUE);
7c2cd90a 2757 fm->SetFastMethod(kTRUE);
2758 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2759 outputChunk = outputFile;
2760 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2761 }
2762 // If last file found, put merged results in the output file
b3e07543 2763 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2764 // Add file to be merged and decrement chunk counter.
b3e07543 2765 fm->AddFile(snextfile);
7c2cd90a 2766 countZero--;
b3e07543 2767 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2768 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2769 // Nothing found - skip this output
2770 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2771 merged = kFALSE;
2772 break;
7c2cd90a 2773 }
2774 fm->OutputFile(outputChunk);
2775 // Merge the outputs, then go to next chunk
2776 if (!fm->Merge()) {
2777 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2778 merged = kFALSE;
2779 break;
7c2cd90a 2780 } else {
2781 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2782 gSystem->Unlink(previousChunk);
2783 }
b3e07543 2784 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2785 countChunk++;
2786 countZero = nmaxmerge;
2787 previousChunk = outputChunk;
0f389141 2788 }
0f389141 2789 }
b3e07543 2790 delete listoffiles;
2791 delete fm;
7c2cd90a 2792 return merged;
2793 }
2794 // Merging stage different than 0.
2795 // Move to the begining of the requested chunk.
2b895f34 2796 fm = new TFileMerger(kTRUE);
7c2cd90a 2797 fm->SetFastMethod(kTRUE);
b3e07543 2798 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2799 delete listoffiles;
7c2cd90a 2800 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2801 // Nothing found - skip this output
2802 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2803 delete fm;
2804 return kFALSE;
2805 }
b3e07543 2806 fm->OutputFile(outputFile);
7c2cd90a 2807 // Merge the outputs
2808 if (!fm->Merge()) {
2809 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2810 delete fm;
2811 return kFALSE;
2812 } else {
b3e07543 2813 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2814 }
7c2cd90a 2815 delete fm;
2816 return kTRUE;
0f389141 2817}
2818
c57f56b7 2819//______________________________________________________________________________
2820Bool_t AliAnalysisAlien::MergeOutputs()
2821{
2822// Merge analysis outputs existing in the AliEn space.
2823 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2824 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2825 if (!Connect()) {
2826 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2827 return kFALSE;
0f389141 2828 }
c9e8f7fd 2829 if (fMergeViaJDL) {
2830 if (!TestBit(AliAnalysisGrid::kMerge)) {
2831 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2832 return kFALSE;
2833 }
2834 if (fProductionMode) {
2835 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2836 return kFALSE;
2837 }
0f389141 2838 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2839 if (!SubmitMerging()) return kFALSE;
0f389141 2840 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2841 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2842 return kFALSE;
c57f56b7 2843 }
2844 // Get the output path
dc030a24 2845 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2846 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2847 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2848 return kFALSE;
2849 }
2850 if (!fOutputFiles.Length()) {
2851 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2852 return kFALSE;
0f389141 2853 }
2854 // Check if fast read option was requested
e1c22e21 2855 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2856 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2857 if (fFastReadOption) {
d3339be3 2858 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2859 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2860 gEnv->SetValue("XNet.ConnectTimeout",50);
2861 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2862 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2863 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2864 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2865 }
e8b839ab 2866 // Make sure we change the temporary directory
2867 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2868 // Set temporary compilation directory to current one
2869 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2870 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2871 TIter next(list);
2872 TObjString *str;
a2f5fc01 2873 TString outputFile;
c57f56b7 2874 Bool_t merged = kTRUE;
2875 while((str=(TObjString*)next())) {
a2f5fc01 2876 outputFile = str->GetString();
2877 Int_t index = outputFile.Index("@");
2878 if (index > 0) outputFile.Remove(index);
2879 TString outputChunk = outputFile;
2880 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2881 // Skip already merged outputs
a2f5fc01 2882 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2883 if (fOverwriteMode) {
a2f5fc01 2884 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2885 gSystem->Unlink(outputFile);
2886 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2887 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2888 outputChunk.Data());
2889 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2890 }
2891 } else {
a2f5fc01 2892 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2893 continue;
2894 }
2895 } else {
a2f5fc01 2896 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2897 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2898 outputChunk.Data());
2899 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2900 }
2901 }
37d21c01 2902 if (fMergeExcludes.Contains(outputFile.Data()) ||
2903 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 2904 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2905 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2906 if (!merged) {
2907 Error("MergeOutputs", "Terminate() will NOT be executed");
2908 return kFALSE;
ff07ec61 2909 }
2910 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2911 if (fileOpened) fileOpened->Close();
c57f56b7 2912 }
0f389141 2913 return kTRUE;
c57f56b7 2914}
2915
bb885a9e 2916//______________________________________________________________________________
2917void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2918{
2919// Use the output files connected to output containers from the analysis manager
2920// rather than the files defined by SetOutputFiles
2921 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2922 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2923 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2924}
2925
149d288c 2926//______________________________________________________________________________
2927void AliAnalysisAlien::SetOutputFiles(const char *list)
2928{
2929// Manually set the output files list.
2930// Removes duplicates. Not allowed if default outputs are not disabled.
2931 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2932 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2933 return;
2934 }
2935 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2936 fOutputFiles = "";
2937 TString slist = list;
2938 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2939 TObjArray *arr = slist.Tokenize(" ");
2940 TObjString *os;
2941 TIter next(arr);
2942 TString sout;
2943 while ((os=(TObjString*)next())) {
2944 sout = os->GetString();
2945 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2946 if (fOutputFiles.Contains(sout)) continue;
2947 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2948 fOutputFiles += sout;
2949 }
2950 delete arr;
f790bc1b 2951}
149d288c 2952
2953//______________________________________________________________________________
2954void AliAnalysisAlien::SetOutputArchive(const char *list)
2955{
2956// Manually set the output archive list. Free text - you are on your own...
2957// Not allowed if default outputs are not disabled.
2958 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2959 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2960 return;
2961 }
2962 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2963 fOutputArchive = list;
2964}
2965
2966//______________________________________________________________________________
2967void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2968{
2969// Setting a prefered output SE is not allowed anymore.
2970 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2971}
2972
28ce6c9c 2973//______________________________________________________________________________
2974void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
2975{
2976// Set some PROOF special parameter.
2977 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2978 if (pair) {
2979 TObject *old = pair->Key();
2980 TObject *val = pair->Value();
2981 fProofParam.Remove(old);
2982 delete old;
2983 delete val;
2984 }
2985 fProofParam.Add(new TObjString(pname), new TObjString(value));
2986}
2987
2988//______________________________________________________________________________
2989const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
2990{
2991// Returns a special PROOF parameter.
2992 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2993 if (!pair) return 0;
2994 return pair->Value()->GetName();
2995}
2996
5513444a 2997//______________________________________________________________________________
2998Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2999{
3000// Start remote grid analysis.
3bdcb562 3001 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3002 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
3003 if (!mgr || !mgr->IsInitialized()) {
3004 Error("StartAnalysis", "You need an initialized analysis manager for this");
3005 return kFALSE;
3006 }
3007 // Are we in PROOF mode ?
3008 if (mgr->IsProofMode()) {
baa34407 3009 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3010 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3011 if (fProofCluster.IsNull()) {
3012 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3013 return kFALSE;
3014 }
3015 if (fProofDataSet.IsNull() && !testMode) {
3016 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3017 return kFALSE;
3018 }
3019 // Set the needed environment
3020 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3021 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3022 if (fProofReset && !testMode) {
3023 if (fProofReset==1) {
3024 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3025 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3026 } else {
3027 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3028 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3029 }
3030 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3031 return kFALSE;
3032 }
baa34407 3033
3034 if (!testMode) {
3035 // Check if there is an old active session
3036 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3037 if (nsessions) {
3038 Error("StartAnalysis","You have to reset your old session first\n");
3039 return kFALSE;
3040 }
53911ca6 3041 }
3bdcb562 3042 // Do we need to change the ROOT version ? The success of this cannot be checked.
3043 if (!fRootVersionForProof.IsNull() && !testMode) {
3044 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
3045 fProofCluster.Data(), fRootVersionForProof.Data()));
3046 }
3047 // Connect to PROOF and check the status
3048 Long_t proof = 0;
27734f0e 3049 TString sworkers;
3050 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3051 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3052 if (!testMode) {
27734f0e 3053 if (!sworkers.IsNull())
3054 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3055 else
3056 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3057 } else {
3058 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3059 if (!proof) {
3060 Error("StartAnalysis", "Could not start PROOF in test mode");
3061 return kFALSE;
3062 }
3063 }
3064 if (!proof) {
3065 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3066 return kFALSE;
3067 }
27734f0e 3068 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3069 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3070 // Set proof special parameters if any
3071 TIter nextpp(&fProofParam);
3072 TObject *proofparam;
3073 while ((proofparam=nextpp())) {
3074 TString svalue = GetProofParameter(proofparam->GetName());
3075 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3076 }
3bdcb562 3077 // Is dataset existing ?
3078 if (!testMode) {
3079 TString dataset = fProofDataSet;
3080 Int_t index = dataset.Index("#");
3081 if (index>=0) dataset.Remove(index);
830acc4c 3082// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3083// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3084// return kFALSE;
3085// }
3086// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3087 }
3088 // Is ClearPackages() needed ?
3089 if (TestSpecialBit(kClearPackages)) {
3090 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3091 gROOT->ProcessLine("gProof->ClearPackages();");
3092 }
3093 // Is a given aliroot mode requested ?
3094 TList optionsList;
1ac846e9 3095 TString parLibs;
830acc4c 3096 if (!fAliRootMode.IsNull()) {
3bdcb562 3097 TString alirootMode = fAliRootMode;
3098 if (alirootMode == "default") alirootMode = "";
3099 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3100 optionsList.SetOwner();
3101 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3102 // Check the additional libs to be loaded
3103 TString extraLibs;
1ac846e9 3104 Bool_t parMode = kFALSE;
dc1ffbcb 3105 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3106 // Parse the extra libs for .so
3107 if (fAdditionalLibs.Length()) {
3108 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3109 TIter next(list);
3110 TObjString *str;
1ac846e9 3111 while((str=(TObjString*)next())) {
3112 if (str->GetString().Contains(".so")) {
3113 if (parMode) {
3114 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3115 break;
3116 }
3117 TString stmp = str->GetName();
3118 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3119 stmp.ReplaceAll(".so","");
3120 if (!extraLibs.IsNull()) extraLibs += ":";
3121 extraLibs += stmp;
3122 continue;
3123 }
3124 if (str->GetString().Contains(".par")) {
3125 // The first par file found in the list will not allow any further .so
3126 parMode = kTRUE;
3127 if (!parLibs.IsNull()) parLibs += ":";
3128 parLibs += str->GetName();
3129 continue;
3130 }
3bdcb562 3131 }
3132 if (list) delete list;
3133 }
dc030a24 3134 if (!extraLibs.IsNull()) {
3135 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3136 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3137 }
830acc4c 3138 // Check extra includes
3139 if (!fIncludePath.IsNull()) {
3140 TString includePath = fIncludePath;
3141 includePath.ReplaceAll(" ",":");
dc030a24 3142 includePath.ReplaceAll("$ALICE_ROOT/","");
3143 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3144 includePath.ReplaceAll("-I","");
dc030a24 3145 includePath.Remove(TString::kTrailing, ':');
830acc4c 3146 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3147 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3148 }
27734f0e 3149 // Check if connection to grid is requested
3150 if (TestSpecialBit(kProofConnectGrid))
3151 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3152 // Enable AliRoot par
3153 if (testMode) {
3154 // Enable proof lite package
3155 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3156 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3157 TNamed *obj = (TNamed*)optionsList.At(i);
3158 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3159 }
3160 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3161 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3162 Info("StartAnalysis", "AliRootProofLite enabled");
3163 } else {
3164 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3165 return kFALSE;
3166 }
3167 } else {
dc030a24 3168 if ( ! fAliROOTVersion.IsNull() ) {
3169 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3170 fAliROOTVersion.Data(), &optionsList))) {
3171 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3172 return kFALSE;
3173 }
3174 }
27734f0e 3175 }
1ac846e9 3176 // Enable first par files from fAdditionalLibs
3177 if (!parLibs.IsNull()) {
3178 TObjArray *list = parLibs.Tokenize(":");
3179 TIter next(list);
3180 TObjString *package;
3181 while((package=(TObjString*)next())) {
40501bb7 3182 TString spkg = package->GetName();
3183 spkg.ReplaceAll(".par", "");
3184 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3185 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3186 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3187 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3188 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3189 return kFALSE;
3190 }
3191 } else {
3192 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3193 return kFALSE;
3194 }
3195 }
3196 if (list) delete list;
3197 }
3bdcb562 3198 } else {
3199 if (fAdditionalLibs.Contains(".so") && !testMode) {
3200 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3201 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3202 return kFALSE;
3203 }
3204 }
3205 // Enable par files if requested
3206 if (fPackages && fPackages->GetEntries()) {
3207 TIter next(fPackages);
3208 TObject *package;
3209 while ((package=next())) {
1ac846e9 3210 // Skip packages already enabled
3211 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3212 TString spkg = package->GetName();
3213 spkg.ReplaceAll(".par", "");
3214 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3215 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3216 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3217 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3218 return kFALSE;
3219 }
3220 } else {
3221 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3222 return kFALSE;
3223 }
3224 }
3225 }
3226 // Do we need to load analysis source files ?
3227 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3228 if (fAnalysisSource.Length()) {
3229 TObjArray *list = fAnalysisSource.Tokenize(" ");
3230 TIter next(list);
3231 TObjString *str;
3232 while((str=(TObjString*)next())) {
3233 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3234 }
3bdcb562 3235 if (list) delete list;
3236 }
830acc4c 3237 if (testMode) {
3238 // Register dataset to proof lite.
3239 if (fFileForTestMode.IsNull()) {
3240 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3241 return kFALSE;
3242 }
3243 if (gSystem->AccessPathName(fFileForTestMode)) {
3244 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3245 return kFALSE;
3246 }
3247 TFileCollection *coll = new TFileCollection();
3248 coll->AddFromFile(fFileForTestMode);
8ddad121 3249 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3250 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3251 }
3bdcb562 3252 return kTRUE;
3253 }
c57f56b7 3254
43da816a 3255 // Check if output files have to be taken from the analysis manager
3256 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3257 // Add output files and AOD files
3258 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3259 // Add extra files registered to the analysis manager
f790bc1b 3260 TString extra = GetListOfFiles("ext");
3261 if (!extra.IsNull()) {
149d288c 3262 extra.ReplaceAll(".root", "*.root");
f790bc1b 3263 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3264 fOutputFiles += extra;
84fcd93f 3265 }
149d288c 3266 // Compose the output archive.
b83f84f0 3267 fOutputArchive = "log_archive.zip:std*@disk=1 ";
3268 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3269 }
f7b1cbc2 3270// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3271 if (TestBit(AliAnalysisGrid::kOffline)) {
3272 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3273 \n there nor any job run. You can revise the JDL and analysis \
3274 \n macro then run the same in \"submit\" mode.");
3275 } else if (TestBit(AliAnalysisGrid::kTest)) {
3276 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3277 \n dataset.");
3278 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3279 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3280 \n space and job submitted.");
3281 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3282 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3283 if (fMergeViaJDL) CheckInputData();
5513444a 3284 return kTRUE;
c57f56b7 3285 } else {
3286 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3287 }
3288
348be253 3289 Print();
c57f56b7 3290 if (!Connect()) {
3291 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3292 return kFALSE;
16a4353c 3293 }
d3b18c4c 3294 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3295 if (!CheckInputData()) {
3296 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3297 return kFALSE;
c57f56b7 3298 }
d3339be3 3299 if (!CreateDataset(fDataPattern)) {
3300 TString serror;
3301 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3302 if (fRunNumbers.Length()) serror = "run numbers";
3303 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3304 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3305 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3306 return kFALSE;
3307 }
21ca8e59 3308 WriteAnalysisFile();
c57f56b7 3309 WriteAnalysisMacro();
3310 WriteExecutable();
3311 WriteValidationScript();
0f389141 3312 if (fMergeViaJDL) {
3313 WriteMergingMacro();
3314 WriteMergeExecutable();
3315 WriteValidationScript(kTRUE);
3316 }
5513444a 3317 if (!CreateJDL()) return kFALSE;
3318 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3319 if (testMode) {
c57f56b7 3320 // Locally testing the analysis
3321 Info("StartAnalysis", "\n_______________________________________________________________________ \
3322 \n Running analysis script in a daughter shell as on a worker node \
3323 \n_______________________________________________________________________");
149d288c 3324 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3325 TIter next(list);
3326 TObjString *str;
a2f5fc01 3327 TString outputFile;
c57f56b7 3328 while((str=(TObjString*)next())) {
a2f5fc01 3329 outputFile = str->GetString();
3330 Int_t index = outputFile.Index("@");
3331 if (index > 0) outputFile.Remove(index);
3332 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3333 }
3334 delete list;
3335 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3336 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3337// gSystem->Exec("cat stdout");
5513444a 3338 return kFALSE;
c57f56b7 3339 }
5513444a 3340 // Check if submitting is managed by LPM manager
a3e84053 3341 if (fProductionMode) {
7ac61826 3342 //TString prodfile = fJDLName;
3343 //prodfile.ReplaceAll(".jdl", ".prod");
3344 //WriteProductionFile(prodfile);
5513444a 3345 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3346 return kFALSE;
3347 }
a8739e8a 3348 // Submit AliEn job(s)
d2a409b2 3349 gGrid->Cd(fGridOutputDir);
a8739e8a 3350 TGridResult *res;
c57f56b7 3351 TString jobID = "";
d2a409b2 3352 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3353 // Submit a given xml or a set of runs
a8739e8a 3354 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3355 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3356 if (res) {
3357 const char *cjobId = res->GetKey(0,"jobId");
3358 if (!cjobId) {
a03be957 3359 gGrid->Stdout();
3360 gGrid->Stderr();
a8739e8a 3361 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3362 return kFALSE;
a8739e8a 3363 } else {
3364 Info("StartAnalysis", "\n_______________________________________________________________________ \
3365 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3366 \n_______________________________________________________________________",
3367 fJDLName.Data(), cjobId);
3368 jobID = cjobId;
3369 }
3370 delete res;
a03be957 3371 } else {
3372 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3373 return kFALSE;
a8739e8a 3374 }
3375 } else {
d2a409b2 3376 // Submit for a range of enumeration of runs.
a03be957 3377 if (!Submit()) return kFALSE;
c57f56b7 3378 }
a8739e8a 3379
c57f56b7 3380 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3381 \n You may exit at any time and terminate the job later using the option <terminate> \
3382 \n ##################################################################################", jobID.Data());
bb885a9e 3383 gSystem->Exec("aliensh");
5513444a 3384 return kTRUE;
c57f56b7 3385}
3386
f790bc1b 3387//______________________________________________________________________________
3388const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3389{
3390// Get a comma-separated list of output files of the requested type.
3391// Type can be (case unsensitive):
3392// aod - list of aod files (std, extensions and filters)
3393// out - list of output files connected to containers (but not aod's or extras)
3394// ext - list of extra files registered to the manager
3395// ter - list of files produced in terminate
3396 static TString files;
3397 files = "";
3398 TString stype = type;
3399 stype.ToLower();
3400 TString aodfiles, extra;
3401 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3402 if (!mgr) {
3403 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3404 return files.Data();
3405 }
3406 if (mgr->GetOutputEventHandler()) {
3407 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3408 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3409 if (!extraaod.IsNull()) {
3410 aodfiles += ",";
3411 aodfiles += extraaod;
3412 }
3413 }
3414 if (stype.Contains("aod")) {
3415 files = aodfiles;
3416 if (stype == "aod") return files.Data();
3417 }
3418 // Add output files that are not in the list of AOD files
3419 TString outputfiles = "";
3420 TIter next(mgr->GetOutputs());
3421 AliAnalysisDataContainer *output;
3422 const char *filename = 0;
3423 while ((output=(AliAnalysisDataContainer*)next())) {
3424 filename = output->GetFileName();
3425 if (!(strcmp(filename, "default"))) continue;
3426 if (outputfiles.Contains(filename)) continue;
3427 if (aodfiles.Contains(filename)) continue;
3428 if (!outputfiles.IsNull()) outputfiles += ",";
3429 outputfiles += filename;
3430 }
3431 if (stype.Contains("out")) {
3432 if (!files.IsNull()) files += ",";
3433 files += outputfiles;
3434 if (stype == "out") return files.Data();
3435 }
3436 // Add extra files registered to the analysis manager
3437 TString sextra;
3438 extra = mgr->GetExtraFiles();
3439 if (!extra.IsNull()) {
3440 extra.Strip();
3441 extra.ReplaceAll(" ", ",");
3442 TObjArray *fextra = extra.Tokenize(",");
3443 TIter nextx(fextra);
3444 TObject *obj;
3445 while ((obj=nextx())) {
3446 if (aodfiles.Contains(obj->GetName())) continue;
3447 if (outputfiles.Contains(obj->GetName())) continue;
3448 if (sextra.Contains(obj->GetName())) continue;
3449 if (!sextra.IsNull()) sextra += ",";
3450 sextra += obj->GetName();
3451 }
3452 delete fextra;
3453 if (stype.Contains("ext")) {
3454 if (!files.IsNull()) files += ",";
3455 files += sextra;
3456 }
3457 }
3458 if (stype == "ext") return files.Data();
3459 TString termfiles;
3460 if (!fTerminateFiles.IsNull()) {
3461 fTerminateFiles.Strip();
3462 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3463 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3464 TIter nextx(fextra);
3465 TObject *obj;
3466 while ((obj=nextx())) {
3467 if (aodfiles.Contains(obj->GetName())) continue;
3468 if (outputfiles.Contains(obj->GetName())) continue;
3469 if (termfiles.Contains(obj->GetName())) continue;
3470 if (sextra.Contains(obj->GetName())) continue;
3471 if (!termfiles.IsNull()) termfiles += ",";
3472 termfiles += obj->GetName();
3473 }
3474 delete fextra;
3475 }
3476 if (stype.Contains("ter")) {
3477 if (!files.IsNull() && !termfiles.IsNull()) {
3478 files += ",";
3479 files += termfiles;
3480 }
3481 }
3482 return files.Data();
3483}
3484
d2a409b2 3485//______________________________________________________________________________
a03be957 3486Bool_t AliAnalysisAlien::Submit()
d2a409b2 3487{
3488// Submit all master jobs.
3489 Int_t nmasterjobs = fInputFiles->GetEntries();
3490 Long_t tshoot = gSystem->Now();
a03be957 3491 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3492 while (fNsubmitted < nmasterjobs) {
3493 Long_t now = gSystem->Now();
3494 if ((now-tshoot)>30000) {
3495 tshoot = now;
a03be957 3496 if (!SubmitNext()) return kFALSE;
d2a409b2 3497 }
3498 }
a03be957 3499 return kTRUE;
d2a409b2 3500}
3501
0f389141 3502//______________________________________________________________________________
a03be957 3503Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3504{
3505// Submit all merging jobs.
dc030a24 3506 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3507 gGrid->Cd(fGridOutputDir);
3508 TString mergeJDLName = fExecutable;
3509 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3510 if (!fInputFiles) {
3511 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3512 return kFALSE;
3513 }
0f389141 3514 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3515 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3516 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3517 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3518 if (fOutputToRunNo) {
3519 // The output directory is the run number
3520 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3521 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3522 } else {
17b4e9a4 3523 if (!fRunNumbers.Length() && !fRunRange[0]) {
3524 // The output directory is the grid outdir
3525 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3526 runOutDir = fGridOutputDir;
3527 } else {
3528 // The output directory is the master number in 3 digits format
3529 printf("### Submitting merging job for master <%03d>\n", i);
3530 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3531 }
7c2cd90a 3532 }
3533 // Check now the number of merging stages.
2b222f7c 3534 TObjArray *list = fOutputFiles.Tokenize(",");
3535 TIter next(list);
3536 TObjString *str;
3537 TString outputFile;
3538 while((str=(TObjString*)next())) {
3539 outputFile = str->GetString();
3540 Int_t index = outputFile.Index("@");
3541 if (index > 0) outputFile.Remove(index);
37d21c01 3542 if (!fMergeExcludes.Contains(outputFile) &&
3543 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3544 }
3545 delete list;
b3e07543 3546 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3547 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3548 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3549 }
a03be957 3550 if (!ntosubmit) return kTRUE;
9f020952 3551 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3552 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3553 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3554 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3555 \n ################################################################################################################");
0f389141 3556 gSystem->Exec("aliensh");
a03be957 3557 return kTRUE;
0f389141 3558}
3559
d2a409b2 3560//______________________________________________________________________________
a03be957 3561Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3562{
08d5b699 3563// Submit next bunch of master jobs if the queue is free. The first master job is
3564// submitted right away, while the next will not be unless the previous was split.
3565// The plugin will not submit new master jobs if there are more that 500 jobs in
3566// waiting phase.
d2a409b2 3567 static Bool_t iscalled = kFALSE;
3568 static Int_t firstmaster = 0;
3569 static Int_t lastmaster = 0;
3570 static Int_t npermaster = 0;
a03be957 3571 if (iscalled) return kTRUE;
d2a409b2 3572 iscalled = kTRUE;
3573 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3574 Int_t ntosubmit = 0;
3575 TGridResult *res;
3576 TString jobID = "";
3bdcb562 3577 Int_t nmasterjobs = fInputFiles->GetEntries();
3578 if (!fNsubmitted) {
3579 ntosubmit = 1;
3580 if (!IsUseSubmitPolicy()) {
90d50a8c 3581 if (nmasterjobs>5)
3bdcb562 3582 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3583 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3584 ntosubmit = nmasterjobs;
3585 }
3586 } else {
d2a409b2 3587 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3588 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3589 // If last master not split, just return
a03be957 3590 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3591 // No more than 100 waiting jobs
08d5b699 3592 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3593 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3594 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3595 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3596 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3597 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3598 }
d2a409b2 3599 for (Int_t i=0; i<ntosubmit; i++) {
3600 // Submit for a range of enumeration of runs.
a03be957 3601 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3602 TString query;
cd11251e 3603 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3604 runOutDir.ReplaceAll(".xml", "");
3605 if (fOutputToRunNo)
3606 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3607 else
3608 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3609 printf("********* %s\n",query.Data());
d2a409b2 3610 res = gGrid->Command(query);
3611 if (res) {
98ca124f 3612 TString cjobId1 = res->GetKey(0,"jobId");
3613 if (!cjobId1.Length()) {
d2a409b2 3614 iscalled = kFALSE;
a03be957 3615 gGrid->Stdout();
3616 gGrid->Stderr();
3617 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3618 return kFALSE;
d2a409b2 3619 } else {
3620 Info("StartAnalysis", "\n_______________________________________________________________________ \
3621 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3622 \n_______________________________________________________________________",
98ca124f 3623 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 3624 jobID += cjobId1;
3625 jobID += " ";
98ca124f 3626 lastmaster = cjobId1.Atoi();
d2a409b2 3627 if (!firstmaster) firstmaster = lastmaster;
3628 fNsubmitted++;
3629 }
3630 delete res;
a03be957 3631 } else {
3632 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3633 return kFALSE;
d2a409b2 3634 }
3635 }
3636 iscalled = kFALSE;
a03be957 3637 return kTRUE;
d2a409b2 3638}
3639
c57f56b7 3640//______________________________________________________________________________
3641void AliAnalysisAlien::WriteAnalysisFile()
3642{
f10e8481 3643// Write current analysis manager into the file <analysisFile>
3644 TString analysisFile = fExecutable;
3645 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3646 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3647 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3648 if (!mgr || !mgr->IsInitialized()) {
3649 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3650 return;
3651 }
3652 // Check analysis type
3653 TObject *handler;
3654 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3655 handler = (TObject*)mgr->GetInputEventHandler();
3656 if (handler) {
65eb22f8 3657 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3658 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3659 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3660 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3661 } else {
3662 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3663 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3664 }
c57f56b7 3665 }
3666 TDirectory *cdir = gDirectory;
f10e8481 3667 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3668 if (file) {
e16a394c 3669 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3670 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3671 // Unless merging makes no sense
3672 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3673 mgr->Write();
3674 delete file;
fe2d7fc2 3675 // Enable termination for local jobs
3676 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3677 }
3678 if (cdir) cdir->cd();
f10e8481 3679 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3680 }
3681 Bool_t copy = kTRUE;
d3b18c4c 3682 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3683 if (copy) {
3684 CdWork();
3685 TString workdir = gGrid->GetHomeDirectory();
3686 workdir += fGridWorkingDir;
d3b18c4c 3687 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3688 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3689 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3690 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3691 }
3692}
3693
3694//______________________________________________________________________________
3695void AliAnalysisAlien::WriteAnalysisMacro()
3696{
3697// Write the analysis macro that will steer the analysis in grid mode.
3698 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3699 ofstream out;
3700 out.open(fAnalysisMacro.Data(), ios::out);
3701 if (!out.good()) {
3702 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3703 return;
3704 }
5b273635 3705 Bool_t hasSTEERBase = kFALSE;
3706 Bool_t hasESD = kFALSE;
3707 Bool_t hasAOD = kFALSE;
3708 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3709 Bool_t hasOADB = kFALSE;
5b273635 3710 Bool_t hasANALYSISalice = kFALSE;
3711 Bool_t hasCORRFW = kFALSE;
c57f56b7 3712 TString func = fAnalysisMacro;
3713 TString type = "ESD";
3714 TString comment = "// Analysis using ";
c85cfc0f 3715 if (IsUseMCchain()) {
3716 type = "MC";
3717 comment += "MC";
3718 } else {
3719 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3720 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3721 type = "AOD";
3722 comment += "AOD";
3723 }
3724 }
0df6ccf2 3725 if (type!="AOD" && fFriendChainName!="") {
3726 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3727 return;
3728 }
c57f56b7 3729 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3730 else comment += " data";
3731 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3732 func.ReplaceAll(".C", "");
3733 out << "void " << func.Data() << "()" << endl;
3734 out << "{" << endl;
3735 out << comment.Data() << endl;
3736 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3737 out << " TStopwatch timer;" << endl;
3738 out << " timer.Start();" << endl << endl;
13f0c00b 3739 // Change temp directory to current one
fdee57f1 3740 if (!IsLocalTest()) {
3741 out << "// connect to AliEn and make the chain" << endl;
3742 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
3743 }
13f0c00b 3744 out << "// Set temporary merging directory to current one" << endl;
3745 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3746 out << "// Set temporary compilation directory to current one" << endl;
3747 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3748 // Reset existing include path
3749 out << "// Reset existing include path and add current directory first in the search" << endl;
3750 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3751 if (!fExecutableCommand.Contains("aliroot")) {
3752 out << "// load base root libraries" << endl;
3753 out << " gSystem->Load(\"libTree\");" << endl;
3754 out << " gSystem->Load(\"libGeom\");" << endl;
3755 out << " gSystem->Load(\"libVMC\");" << endl;
3756 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3757 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3758 }
d5c6455a 3759 if (fAdditionalRootLibs.Length()) {
47a4137d 3760 // in principle libtree /lib geom libvmc etc. can go into this list, too
3761 out << "// Add aditional libraries" << endl;
3762 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3763 TIter next(list);
3764 TObjString *str;
3765 while((str=(TObjString*)next())) {
3766 if (str->GetString().Contains(".so"))
3767 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3768 }
47a4137d 3769 if (list) delete list;
d5c6455a 3770 }
57377eb5 3771 out << "// Load analysis framework libraries" << endl;
652561d5 3772 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3773 if (!fPackages) {
f790bc1b 3774 if (!fExecutableCommand.Contains("aliroot")) {
3775 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3776 out << " gSystem->Load(\"libESD\");" << endl;
3777 out << " gSystem->Load(\"libAOD\");" << endl;
3778 }
4e5c5506 3779 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3780 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3781 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3782 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3783 } else {
4e5c5506 3784 TIter next(fPackages);
3785 TObject *obj;
57377eb5 3786 TString pkgname;
57377eb5 3787 while ((obj=next())) {
3788 pkgname = obj->GetName();
4478e6f1 3789 if (pkgname == "STEERBase" ||
3790 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3791 if (pkgname == "ESD" ||
3792 pkgname == "ESD.par") hasESD = kTRUE;
3793 if (pkgname == "AOD" ||
3794 pkgname == "AOD.par") hasAOD = kTRUE;
3795 if (pkgname == "ANALYSIS" ||
3796 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 3797 if (pkgname == "OADB" ||
3798 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 3799 if (pkgname == "ANALYSISalice" ||
3800 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3801 if (pkgname == "CORRFW" ||
3802 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3803 }
3804 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3805 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3806 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3807 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3808 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3809 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3810 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3811 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3812 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 3813 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
3814 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 3815 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3816 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3817 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3818 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3819 out << "// Compile other par packages" << endl;
3820 next.Reset();
fcc9bb6f 3821 while ((obj=next())) {
3822 pkgname = obj->GetName();
4478e6f1 3823 if (pkgname == "STEERBase" ||
3824 pkgname == "STEERBase.par" ||
3825 pkgname == "ESD" ||
3826 pkgname == "ESD.par" ||
3827 pkgname == "AOD" ||
3828 pkgname == "AOD.par" ||
3829 pkgname == "ANALYSIS" ||
3830 pkgname == "ANALYSIS.par" ||
7ea40ac8 3831 pkgname == "OADB" ||
3832 pkgname == "OADB.par" ||
4478e6f1 3833 pkgname == "ANALYSISalice" ||
3834 pkgname == "ANALYSISalice.par" ||
3835 pkgname == "CORRFW" ||
3836 pkgname == "CORRFW.par") continue;
5b273635 3837 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3838 }
4e5c5506 3839 }
b3e07543 3840 out << "// include path" << endl;
3841 // Get the include path from the interpreter and remove entries pointing to AliRoot
3842 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3843 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3844 out << " TIter nextpath(listpaths);" << endl;
3845 out << " TObjString *pname;" << endl;
3846 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3847 out << " TString current = pname->GetName();" << endl;
3848 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3849 out << " gSystem->AddIncludePath(current);" << endl;
3850 out << " }" << endl;
3851 out << " if (listpaths) delete listpaths;" << endl;
3852 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3853 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3854 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3855 if (fAdditionalLibs.Length()) {
3856 out << "// Add aditional AliRoot libraries" << endl;
3857 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3858 TIter next(list);
3859 TObjString *str;
3860 while((str=(TObjString*)next())) {
3861 if (str->GetString().Contains(".so"))
3862 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3863 if (str->GetString().Contains(".par"))
3864 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3865 }
3866 if (list) delete list;
3867 }
3868 out << endl;
c57f56b7 3869 out << "// analysis source to be compiled at runtime (if any)" << endl;
3870 if (fAnalysisSource.Length()) {
3871 TObjArray *list = fAnalysisSource.Tokenize(" ");
3872 TIter next(list);
3873 TObjString *str;
3874 while((str=(TObjString*)next())) {
3875 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3876 }
3877 if (list) delete list;
3878 }
3879 out << endl;
b3e07543 3880// out << " printf(\"Currently load libraries:\\n\");" << endl;
3881// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3882 if (fFastReadOption) {
d3339be3 3883 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3884 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3885 out << "// fast xrootd reading enabled" << endl;
3886 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3887 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3888 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3889 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3890 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3891 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 3892 }
c57f56b7 3893 out << "// read the analysis manager from file" << endl;
f10e8481 3894 TString analysisFile = fExecutable;
3895 analysisFile.ReplaceAll(".sh", ".root");
105631b5 3896 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
3897 << analysisFile << "\");" << endl;
3898 out << " if (!mgr) return;" << endl;
21ca8e59 3899 if (IsLocalTest()) {
3900 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
3901 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 3902 if (fFileForTestMode.IsNull())
3903 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
3904 else
3905 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 3906 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4b5d9e21 3907 if (!fFriendChainName.IsNull())
3908 out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\");" << endl;
21ca8e59 3909 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 3910 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 3911 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 3912 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
3913 } else {
7a366e0b 3914 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 3915 out << " mgr->SetNSysInfo(100);" << endl;
3916 }
21ca8e59 3917 }
c57f56b7 3918 out << " mgr->PrintStatus();" << endl;
52b6a92b 3919 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3920 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3921 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3922 } else {
9a8288ff 3923 if (TestBit(AliAnalysisGrid::kTest))
3924 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3925 else
3926 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3927 }
3928 }
21ca8e59 3929 if (!IsLocalTest()) {
242accb2 3930 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 3931 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
3932 } else {
3933 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 3934 }
f7498086 3935 out << " timer.Stop();" << endl;
3936 out << " timer.Print();" << endl;
c57f56b7 3937 out << "}" << endl << endl;
21ca8e59 3938 if (!IsLocalTest()) {
fcc9bb6f 3939 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3940 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3941 out << "{" << endl;
3942 out << "// Create a chain using url's from xml file" << endl;
242accb2 3943 out << " TString filename;" << endl;
3944 out << " Int_t run = 0;" << endl;
c85cfc0f 3945 if (IsUseMCchain()) {
3946 out << " TString treename = \"TE\";" << endl;
3947 } else {
3948 out << " TString treename = type;" << endl;
3949 out << " treename.ToLower();" << endl;
3950 out << " treename += \"Tree\";" << endl;
3951 }
e02fee64 3952 out << " printf(\"***************************************\\n\");" << endl;
3953 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3954 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3955 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3956 out << " if (!coll) {" << endl;
3957 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3958 out << " return NULL;" << endl;
3959 out << " }" << endl;
242accb2 3960 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3961 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3962 if(fFriendChainName!="") {
3963 out << " TChain *chainFriend = new TChain(treename);" << endl;
3964 }
c57f56b7 3965 out << " coll->Reset();" << endl;
0df6ccf2 3966 out << " while (coll->Next()) {" << endl;
242accb2 3967 out << " filename = coll->GetTURL("");" << endl;
3968 out << " if (mgr) {" << endl;
3969 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3970 out << " if (nrun && nrun != run) {" << endl;
3971 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3972 out << " mgr->SetRunFromPath(nrun);" << endl;
3973 out << " run = nrun;" << endl;
3974 out << " }" << endl;
3975 out << " }" << endl;
3976 out << " chain->Add(filename);" << endl;
0df6ccf2 3977 if(fFriendChainName!="") {
3978 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
4b5d9e21 3979 out << " if (fileFriend.Index(\"#\") > -1) fileFriend.Remove(fileFriend.Index(\"#\"));" << endl;
3980 out << " fileFriend = gSystem->DirName(fileFriend);" << endl;
3981 out << " fileFriend += \"/\";" << endl;
3982 out << " fileFriend += \"" << fFriendChainName << "\";";
3983 out << " TFile *file = TFile::Open(fileFriend);" << endl;
3984 out << " if (file) {" << endl;
3985 out << " file->Close();" << endl;
3986 out << " chainFriend->Add(fileFriend.Data());" << endl;
3987 out << " } else {" << endl;
3988 out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
3989 out << " return 0;" << endl;
3990 out << " }" << endl;
0df6ccf2 3991 }
3992 out << " }" << endl;
c57f56b7 3993 out << " if (!chain->GetNtrees()) {" << endl;
3994 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3995 out << " return NULL;" << endl;
3996 out << " }" << endl;
0df6ccf2 3997 if(fFriendChainName!="") {
3998 out << " chain->AddFriend(chainFriend);" << endl;
3999 }
c57f56b7 4000 out << " return chain;" << endl;
fcc9bb6f 4001 out << "}" << endl << endl;
c57f56b7 4002 }
5b273635 4003 if (hasANALYSISalice) {
4004 out <<"//________________________________________________________________________________" << endl;
4005 out << "Bool_t SetupPar(const char *package) {" << endl;
4006 out << "// Compile the package and set it up." << endl;
4007 out << " TString pkgdir = package;" << endl;
4008 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4009 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4010 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4011 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4012 out << " // Check for BUILD.sh and execute" << endl;
4013 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4014 out << " printf(\"*******************************\\n\");" << endl;
4015 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4016 out << " printf(\"*******************************\\n\");" << endl;
4017 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4018 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4019 out << " gSystem->ChangeDirectory(cdir);" << endl;
4020 out << " return kFALSE;" << endl;
4021 out << " }" << endl;
4022 out << " } else {" << endl;
4023 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4024 out << " gSystem->ChangeDirectory(cdir);" << endl;
4025 out << " return kFALSE;" << endl;
4026 out << " }" << endl;
4027 out << " // Check for SETUP.C and execute" << endl;
4028 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4029 out << " printf(\"*******************************\\n\");" << endl;
4030 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4031 out << " printf(\"*******************************\\n\");" << endl;
4032 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4033 out << " } else {" << endl;
4034 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4035 out << " gSystem->ChangeDirectory(cdir);" << endl;
4036 out << " return kFALSE;" << endl;
4037 out << " }" << endl;
4038 out << " // Restore original workdir" << endl;
4039 out << " gSystem->ChangeDirectory(cdir);" << endl;
4040 out << " return kTRUE;" << endl;
4041 out << "}" << endl;
4042 }
c57f56b7 4043 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4044 }
4045 Bool_t copy = kTRUE;
d3b18c4c 4046 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4047 if (copy) {
4048 CdWork();
4049 TString workdir = gGrid->GetHomeDirectory();
4050 workdir += fGridWorkingDir;
4051 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4052 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4053// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4054 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4055 Form("alien://%s/%s", workdir.Data(),
4056 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4057 }
4058}
4059
0f389141 4060//______________________________________________________________________________
4061void AliAnalysisAlien::WriteMergingMacro()
4062{
4063// Write a macro to merge the outputs per master job.
4064 if (!fMergeViaJDL) return;
4065 if (!fOutputFiles.Length()) {
4066 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4067 return;
4068 }
4069 TString mergingMacro = fExecutable;
4070 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4071 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4072 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4073 ofstream out;
4074 out.open(mergingMacro.Data(), ios::out);
4075 if (!out.good()) {
4076 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4077 return;
4078 }
5b273635 4079 Bool_t hasSTEERBase = kFALSE;
4080 Bool_t hasESD = kFALSE;
4081 Bool_t hasAOD = kFALSE;
4082 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4083 Bool_t hasOADB = kFALSE;
5b273635 4084 Bool_t hasANALYSISalice = kFALSE;
4085 Bool_t hasCORRFW = kFALSE;
0f389141 4086 TString func = mergingMacro;
4087 TString comment;
4088 func.ReplaceAll(".C", "");
f47d5cb4 4089 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4090 out << "{" << endl;
4091 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4092 out << " TStopwatch timer;" << endl;
4093 out << " timer.Start();" << endl << endl;
b3e07543 4094 // Reset existing include path
4095 out << "// Reset existing include path and add current directory first in the search" << endl;
4096 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4097 if (!fExecutableCommand.Contains("aliroot")) {
4098 out << "// load base root libraries" << endl;
4099 out << " gSystem->Load(\"libTree\");" << endl;
4100 out << " gSystem->Load(\"libGeom\");" << endl;
4101 out << " gSystem->Load(\"libVMC\");" << endl;
4102 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4103 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4104 }
0f389141 4105 if (fAdditionalRootLibs.Length()) {
4106 // in principle libtree /lib geom libvmc etc. can go into this list, too
4107 out << "// Add aditional libraries" << endl;
4108 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4109 TIter next(list);
4110 TObjString *str;
4111 while((str=(TObjString*)next())) {
4112 if (str->GetString().Contains(".so"))
4113 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4114 }
4115 if (list) delete list;
4116 }
0f389141 4117 out << "// Load analysis framework libraries" << endl;
4118 if (!fPackages) {
7c2cd90a 4119 if (!fExecutableCommand.Contains("aliroot")) {
4120 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4121 out << " gSystem->Load(\"libESD\");" << endl;
4122 out << " gSystem->Load(\"libAOD\");" << endl;
4123 }
0f389141 4124 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4125 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4126 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4127 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4128 } else {
4129 TIter next(fPackages);
4130 TObject *obj;
4131 TString pkgname;
5b273635 4132 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4133 while ((obj=next())) {
4134 pkgname = obj->GetName();
4135 if (pkgname == "STEERBase" ||
4136 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4137 if (pkgname == "ESD" ||
4138 pkgname == "ESD.par") hasESD = kTRUE;
4139 if (pkgname == "AOD" ||
4140 pkgname == "AOD.par") hasAOD = kTRUE;
4141 if (pkgname == "ANALYSIS" ||
4142 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4143 if (pkgname == "OADB" ||
4144 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4145 if (pkgname == "ANALYSISalice" ||
4146 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4147 if (pkgname == "CORRFW" ||
4148 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4149 }
5b273635 4150 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4151 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4152 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4153 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4154 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4155 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4156 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4157 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4158 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4159 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4160 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4161 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4162 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4163 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4164 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4165 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4166 out << "// Compile other par packages" << endl;
4167 next.Reset();
4168 while ((obj=next())) {
4169 pkgname = obj->GetName();
4170 if (pkgname == "STEERBase" ||
4171 pkgname == "STEERBase.par" ||
4172 pkgname == "ESD" ||
4173 pkgname == "ESD.par" ||
4174 pkgname == "AOD" ||
4175 pkgname == "AOD.par" ||
4176 pkgname == "ANALYSIS" ||
4177 pkgname == "ANALYSIS.par" ||
7ea40ac8 4178 pkgname == "OADB" ||
4179 pkgname == "OADB.par" ||
0f389141 4180 pkgname == "ANALYSISalice" ||
4181 pkgname == "ANALYSISalice.par" ||
4182 pkgname == "CORRFW" ||
4183 pkgname == "CORRFW.par") continue;
5b273635 4184 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4185 }
4186 }
b3e07543 4187 out << "// include path" << endl;
4188 // Get the include path from the interpreter and remove entries pointing to AliRoot
4189 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4190 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4191 out << " TIter nextpath(listpaths);" << endl;
4192 out << " TObjString *pname;" << endl;
4193 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4194 out << " TString current = pname->GetName();" << endl;
4195 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4196 out << " gSystem->AddIncludePath(current);" << endl;
4197 out << " }" << endl;
4198 out << " if (listpaths) delete listpaths;" << endl;
4199 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4200 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4201 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4202 if (fAdditionalLibs.Length()) {
4203 out << "// Add aditional AliRoot libraries" << endl;
4204 TObjArray *list = fAdditionalLibs.Tokenize(" ");
4205 TIter next(list);
4206 TObjString *str;
4207 while((str=(TObjString*)next())) {
4208 if (str->GetString().Contains(".so"))
4209 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4210 }
4211 if (list) delete list;
4212 }
4213 out << endl;
4214 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4215 if (fAnalysisSource.Length()) {
4216 TObjArray *list = fAnalysisSource.Tokenize(" ");
4217 TIter next(list);
4218 TObjString *str;
4219 while((str=(TObjString*)next())) {
4220 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4221 }
4222 if (list) delete list;
4223 }
149d288c 4224 out << endl;
4225
0f389141 4226 if (fFastReadOption) {
4227 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4228 out << "// fast xrootd reading enabled" << endl;
4229 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4230 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4231 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4232 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4233 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4234 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4235 }
4236 // Change temp directory to current one
fdee57f1 4237 out << "// Connect to AliEn" << endl;
4238 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
e8b839ab 4239 out << "// Set temporary merging directory to current one" << endl;
4240 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4241 out << "// Set temporary compilation directory to current one" << endl;
4242 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
7c2cd90a 4243 out << " TString outputDir = dir;" << endl;
f790bc1b 4244 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4245 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4246 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4247 out << " TIter *iter = new TIter(list);" << endl;
4248 out << " TObjString *str;" << endl;
a2f5fc01 4249 out << " TString outputFile;" << endl;
0f389141 4250 out << " Bool_t merged = kTRUE;" << endl;
4251 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4252 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4253 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4254 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4255 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4256 out << " // Skip already merged outputs" << endl;
a2f5fc01 4257 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
4258 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 4259 out << " continue;" << endl;
4260 out << " }" << endl;
a2f5fc01 4261 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4262 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4263 out << " if (!merged) {" << endl;
a2f5fc01 4264 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4265 out << " return;" << endl;
0f389141 4266 out << " }" << endl;
4267 out << " }" << endl;
7c2cd90a 4268 out << " // all outputs merged, validate" << endl;
4269 out << " ofstream out;" << endl;
4270 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4271 out << " out.close();" << endl;
4272 out << " // read the analysis manager from file" << endl;
f866cba5 4273 TString analysisFile = fExecutable;
4274 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 4275 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
105631b5 4276 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4277 << analysisFile << "\");" << endl;
4278 out << " if (!mgr) return;" << endl;
5c4250fc 4279 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4280 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4281 out << " mgr->PrintStatus();" << endl;
4282 if (AliAnalysisManager::GetAnalysisManager()) {
4283 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
4284 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4285 } else {
9a8288ff 4286 if (TestBit(AliAnalysisGrid::kTest))
4287 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4288 else
4289 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4290 }
4291 }
9a8288ff 4292 out << " TTree *tree = NULL;" << endl;
4293 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4294 out << "}" << endl << endl;
5b273635 4295 if (hasANALYSISalice) {
4296 out <<"//________________________________________________________________________________" << endl;
4297 out << "Bool_t SetupPar(const char *package) {" << endl;
4298 out << "// Compile the package and set it up." << endl;
4299 out << " TString pkgdir = package;" << endl;
4300 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4301 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4302 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4303 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4304 out << " // Check for BUILD.sh and execute" << endl;
4305 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4306 out << " printf(\"*******************************\\n\");" << endl;
4307 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4308 out << " printf(\"*******************************\\n\");" << endl;
4309 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4310 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4311 out << " gSystem->ChangeDirectory(cdir);" << endl;
4312 out << " return kFALSE;" << endl;
4313 out << " }" << endl;
4314 out << " } else {" << endl;
4315 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4316 out << " gSystem->ChangeDirectory(cdir);" << endl;
4317 out << " return kFALSE;" << endl;
4318 out << " }" << endl;
4319 out << " // Check for SETUP.C and execute" << endl;
4320 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4321 out << " printf(\"*******************************\\n\");" << endl;
4322 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4323 out << " printf(\"*******************************\\n\");" << endl;
4324 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4325 out << " } else {" << endl;
4326 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4327 out << " gSystem->ChangeDirectory(cdir);" << endl;
4328 out << " return kFALSE;" << endl;
4329 out << " }" << endl;
4330 out << " // Restore original workdir" << endl;
4331 out << " gSystem->ChangeDirectory(cdir);" << endl;
4332 out << " return kTRUE;" << endl;
4333 out << "}" << endl;
4334 }
0f389141 4335 }
4336 Bool_t copy = kTRUE;
d3b18c4c 4337 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4338 if (copy) {
4339 CdWork();
4340 TString workdir = gGrid->GetHomeDirectory();
4341 workdir += fGridWorkingDir;
4342 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4343 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4344// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4345 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4346 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4347 }
4348}
4349
4350//______________________________________________________________________________
4351Bool_t AliAnalysisAlien::SetupPar(const char *package)
4352{
205b201f 4353// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4354// Note that for loading the compiled library. The current directory should have precedence in
4355// LD_LIBRARY_PATH
4356 TString pkgdir = package;
4357 pkgdir.ReplaceAll(".par","");
b50a41c3 4358 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4359 TString cdir = gSystem->WorkingDirectory();
4360 gSystem->ChangeDirectory(pkgdir);
4361 // Check for BUILD.sh and execute
4362 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4363 printf("**************************************************\n");
4364 printf("*** Building PAR archive %s\n", package);
4365 printf("**************************************************\n");
4366 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4367 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4368 gSystem->ChangeDirectory(cdir);
4369 return kFALSE;
4370 }
4371 } else {
4372 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4373 gSystem->ChangeDirectory(cdir);
4374 return kFALSE;
4375 }
4376 // Check for SETUP.C and execute
4377 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4378 printf("**************************************************\n");
4379 printf("*** Setup PAR archive %s\n", package);
4380 printf("**************************************************\n");
4381 gROOT->Macro("PROOF-INF/SETUP.C");
4382 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4383 } else {
4384 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4385 gSystem->ChangeDirectory(cdir);
4386 return kFALSE;
4387 }
4388 // Restore original workdir
4389 gSystem->ChangeDirectory(cdir);
4390 return kTRUE;
4391}
4392
c57f56b7 4393//______________________________________________________________________________
4394void AliAnalysisAlien::WriteExecutable()
4395{
4396// Generate the alien executable script.
4397 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4398 ofstream out;
4399 out.open(fExecutable.Data(), ios::out);
4400 if (out.bad()) {
5513444a 4401 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4402 return;
4403 }
4404 out << "#!/bin/bash" << endl;
2a4d5166 4405 // Make sure we can properly compile par files
4406 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4407 out << "echo \"=========================================\"" << endl;
4408 out << "echo \"############## PATH : ##############\"" << endl;
4409 out << "echo $PATH" << endl;
4410 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4411 out << "echo $LD_LIBRARY_PATH" << endl;
4412 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4413 out << "echo $ROOTSYS" << endl;
4414 out << "echo \"############## which root : ##############\"" << endl;
4415 out << "which root" << endl;
4416 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4417 out << "echo $ALICE_ROOT" << endl;
4418 out << "echo \"############## which aliroot : ##############\"" << endl;
4419 out << "which aliroot" << endl;
9c5ddadc 4420 out << "echo \"############## system limits : ##############\"" << endl;
4421 out << "ulimit -a" << endl;
4422 out << "echo \"############## memory : ##############\"" << endl;
4423 out << "free -m" << endl;
c57f56b7 4424 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4425 out << fExecutableCommand << " ";
370c9880 4426 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4427 out << "RET=$?" << endl;
4428 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4429 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4430 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4431 out << " let sig=\"$RET - 128\""<<endl;
4432 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4433 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4434 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4435 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4436 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4437 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4438 out << " fi"<<endl;
4439 out << " exit $RET"<< endl;
4440 out << "fi" << endl << endl ;
4441 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4442 out << "echo \"############## memory after: ##############\"" << endl;
4443 out << "free -m" << endl;
c57f56b7 4444 }
4445 Bool_t copy = kTRUE;
d3b18c4c 4446 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4447 if (copy) {
4448 CdWork();
4449 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4450 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4451 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4452 workdir += fGridWorkingDir;
4453 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4454 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4455 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4456// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4457 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4458 executable.Data())) Fatal("","Terminating");
c57f56b7 4459 }
4460}
4461
0f389141 4462//______________________________________________________________________________
4463void AliAnalysisAlien::WriteMergeExecutable()
4464{
4465// Generate the alien executable script for the merging job.
4466 if (!fMergeViaJDL) return;
4467 TString mergeExec = fExecutable;
4468 mergeExec.ReplaceAll(".sh", "_merge.sh");
4469 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4470 ofstream out;
4471 out.open(mergeExec.Data(), ios::out);
4472 if (out.bad()) {
4473 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4474 return;
4475 }
4476 out << "#!/bin/bash" << endl;
2a4d5166 4477 // Make sure we can properly compile par files
4478 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4479 out << "echo \"=========================================\"" << endl;
4480 out << "echo \"############## PATH : ##############\"" << endl;
4481 out << "echo $PATH" << endl;
4482 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4483 out << "echo $LD_LIBRARY_PATH" << endl;
4484 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4485 out << "echo $ROOTSYS" << endl;
4486 out << "echo \"############## which root : ##############\"" << endl;
4487 out << "which root" << endl;
4488 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4489 out << "echo $ALICE_ROOT" << endl;
4490 out << "echo \"############## which aliroot : ##############\"" << endl;
4491 out << "which aliroot" << endl;
4492 out << "echo \"############## system limits : ##############\"" << endl;
4493 out << "ulimit -a" << endl;
4494 out << "echo \"############## memory : ##############\"" << endl;
4495 out << "free -m" << endl;
4496 out << "echo \"=========================================\"" << endl << endl;
0f389141 4497 TString mergeMacro = fExecutable;
4498 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4499 if (IsOneStageMerging())
4500 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4501 else
f47d5cb4 4502 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4503 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4504 out << "RET=$?" << endl;
4505 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4506 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4507 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4508 out << " let sig=\"$RET - 128\""<<endl;
4509 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4510 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4511 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4512 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4513 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4514 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4515 out << " fi"<<endl;
4516 out << " exit $RET"<< endl;
4517 out << "fi" << endl << endl ;
0f389141 4518 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4519 out << "echo \"############## memory after: ##############\"" << endl;
4520 out << "free -m" << endl;
0f389141 4521 }
4522 Bool_t copy = kTRUE;
d3b18c4c 4523 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4524 if (copy) {
4525 CdWork();
4526 TString workdir = gGrid->GetHomeDirectory();
4527 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4528 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4529 workdir += fGridWorkingDir;
4530 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4531 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4532 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4533// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4534 if (!copyLocal2Alien("WriteMergeExecutable",
4535 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4536 }
4537}
4538
c57f56b7 4539//______________________________________________________________________________
5513444a 4540void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4541{
4542// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4543// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4544// Next lines: full_path_to_dataset XXX (XXX is a string)
4545// To submit, one has to: submit jdl XXX for all lines
4546 ofstream out;
4547 out.open(filename, ios::out);
4548 if (out.bad()) {
4549 Error("WriteProductionFile", "Bad file name: %s", filename);
4550 return;
4551 }
d3b18c4c 4552 TString workdir;
4553 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4554 workdir = gGrid->GetHomeDirectory();
5513444a 4555 workdir += fGridWorkingDir;
f5e8c702 4556 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4557 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4558 out << locjdl << " " << njobspermaster << endl;
5513444a 4559 Int_t nmasterjobs = fInputFiles->GetEntries();
4560 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4561 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4562 runOutDir.ReplaceAll(".xml", "");
4563 if (fOutputToRunNo)
4564 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4565 else
4566 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4567 }
d3b18c4c 4568 if (gGrid) {
4569 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4570 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4571// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4572 if (!copyLocal2Alien("WriteProductionFile", filename,
4573 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4574 }
5513444a 4575}
4576
4577//______________________________________________________________________________
0f389141 4578void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4579{
4580// Generate the alien validation script.
4581 // Generate the validation script
4582 TObjString *os;
d3b18c4c 4583 if (fValidationScript.IsNull()) {
4584 fValidationScript = fExecutable;
4585 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4586 }
4587 TString validationScript = fValidationScript;
4588 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4589 if (!Connect()) {
4590 Error("WriteValidationScript", "Alien connection required");
4591 return;
4592 }
d3b18c4c 4593 if (!fTerminateFiles.IsNull()) {
4594 fTerminateFiles.Strip();
4595 fTerminateFiles.ReplaceAll(" ",",");
4596 }
a2f5fc01 4597 TString outStream = "";
4598 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4599 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4600 ofstream out;
0d5d317c 4601 out.open(validationScript, ios::out);
c57f56b7 4602 out << "#!/bin/bash" << endl;
4603 out << "##################################################" << endl;
4604 out << "validateout=`dirname $0`" << endl;
4605 out << "validatetime=`date`" << endl;
4606 out << "validated=\"0\";" << endl;
4607 out << "error=0" << endl;
4608 out << "if [ -z $validateout ]" << endl;
4609 out << "then" << endl;
4610 out << " validateout=\".\"" << endl;
4611 out << "fi" << endl << endl;
4612 out << "cd $validateout;" << endl;
4613 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4614 out << "echo \"*******************************************************\"" << outStream << endl;
4615 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4616 out << "" << endl;
a2f5fc01 4617 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4618 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4619 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4620 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4621 out << "ls -la ./" << outStream << endl;
4622 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4623 out << "##################################################" << endl;
ebec370a 4624 out << "" << endl;
4625
4626 out << "if [ ! -f stderr ] ; then" << endl;
4627 out << " error=1" << endl;
a2f5fc01 4628 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4629 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4630 out << "fi" << endl;
4631
b34c9f51 4632 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4633 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4634 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4635 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4636 out << "" << endl;
4637
ebec370a 4638 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4639 out << " error=1" << endl;
a2f5fc01 4640 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4641 out << " echo \"$parArch\" " << outStream << endl;
4642 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4643 out << "fi" << endl;
4644
4645 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4646 out << " error=1" << endl;
a2f5fc01 4647 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4648 out << " echo \"$segViol\" " << outStream << endl;
4649 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4650 out << "fi" << endl;
4651
4652 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4653 out << " error=1" << endl;
a2f5fc01 4654 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4655 out << " echo \"$segFault\" " << outStream << endl;
4656 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4657 out << "fi" << endl;
4658
b34c9f51 4659 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4660 out << " error=1" << endl;
a2f5fc01 4661 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4662 out << " echo \"$glibcErr\" " << outStream << endl;
4663 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4664 out << "fi" << endl;
4665
ebec370a 4666 // Part dedicated to the specific analyses running into the train
4667
d3b18c4c 4668 TString outputFiles = fOutputFiles;
4669 if (merge && !fTerminateFiles.IsNull()) {
4670 outputFiles += ",";
4671 outputFiles += fTerminateFiles;
4672 }
4673 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4674 TIter next1(arr);
a2f5fc01 4675 TString outputFile;
bbd88db8 4676 while (!merge && (os=(TObjString*)next1())) {
4677 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4678 outputFile = os->GetString();
4679 Int_t index = outputFile.Index("@");
4680 if (index > 0) outputFile.Remove(index);
bbd88db8 4681 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4682 if (outputFile.Contains("*")) continue;
4683 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4684 out << " error=1" << endl;
7c2cd90a 4685 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4686 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4687 out << "fi" << endl;
4688 }
4689 delete arr;
7c2cd90a 4690 out << "if ! [ -f outputs_valid ] ; then" << endl;
4691 out << " error=1" << endl;
4692 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4693 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4694 out << "fi" << endl;
923e2ca5 4695
c57f56b7 4696 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4697 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4698 if (!IsKeepLogs()) {
4699 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4700 outStream = "";
149d288c 4701 out << " rm -f std*" << endl;
4702 }
c57f56b7 4703 out << "fi" << endl;
4704
a2f5fc01 4705 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4706 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4707 out << "cd -" << endl;
4708 out << "exit $error" << endl;
4709 }
4710 Bool_t copy = kTRUE;
d3b18c4c 4711 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4712 if (copy) {
4713 CdWork();
4714 TString workdir = gGrid->GetHomeDirectory();
4715 workdir += fGridWorkingDir;
d3b18c4c 4716 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4717 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 4718// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4719 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
4720 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 4721 }
4722}