]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
never really used and removed before restructuring
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
51ClassImp(AliAnalysisAlien)
6c395669 52#if 0
53;
54#endif
c57f56b7 55
6c395669 56namespace {
57 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
58 {
59 TString sl(Form("file:%s", loc));
60 TString sr(Form("alien://%s", rem));
61 Bool_t ret = TFile::Cp(sl, sr);
62 if (!ret) {
63 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
64 }
65 return ret;
66 }
67}
68
c57f56b7 69//______________________________________________________________________________
70AliAnalysisAlien::AliAnalysisAlien()
71 :AliAnalysisGrid(),
72 fGridJDL(NULL),
0f389141 73 fMergingJDL(NULL),
c57f56b7 74 fPrice(0),
75 fTTL(0),
76 fSplitMaxInputFileNumber(0),
77 fMaxInitFailed(0),
78 fMasterResubmitThreshold(0),
bb885a9e 79 fNtestFiles(0),
319593fb 80 fNrunsPerMaster(0),
16a4353c 81 fMaxMergeFiles(0),
b3e07543 82 fMaxMergeStages(0),
d2a409b2 83 fNsubmitted(0),
a3e84053 84 fProductionMode(0),
cd11251e 85 fOutputToRunNo(0),
0f389141 86 fMergeViaJDL(0),
87 fFastReadOption(0),
e1c22e21 88 fOverwriteMode(1),
149d288c 89 fNreplicas(2),
3bdcb562 90 fNproofWorkers(0),
27734f0e 91 fNproofWorkersPerSlave(0),
3bdcb562 92 fProofReset(0),
c57f56b7 93 fRunNumbers(),
94 fExecutable(),
0a1c1f7f 95 fExecutableCommand(),
c57f56b7 96 fArguments(),
631c0b05 97 fExecutableArgs(),
c57f56b7 98 fAnalysisMacro(),
99 fAnalysisSource(),
d3b18c4c 100 fValidationScript(),
d5c6455a 101 fAdditionalRootLibs(),
c57f56b7 102 fAdditionalLibs(),
103 fSplitMode(),
104 fAPIVersion(),
105 fROOTVersion(),
106 fAliROOTVersion(),
648174cf 107 fExternalPackages(),
c57f56b7 108 fUser(),
109 fGridWorkingDir(),
110 fGridDataDir(),
111 fDataPattern(),
112 fGridOutputDir(),
113 fOutputArchive(),
114 fOutputFiles(),
115 fInputFormat(),
e7c71df0 116 fDatasetName(),
c57f56b7 117 fJDLName(),
d3b18c4c 118 fTerminateFiles(),
bb885a9e 119 fMergeExcludes(),
37d21c01 120 fRegisterExcludes(),
f965131e 121 fIncludePath(),
bb885a9e 122 fCloseSE(),
0df6ccf2 123 fFriendChainName(),
c6cb3634 124 fJobTag(),
648174cf 125 fOutputSingle(),
5fce53f4 126 fRunPrefix(),
3bdcb562 127 fProofCluster(),
128 fProofDataSet(),
129 fFileForTestMode(),
130 fRootVersionForProof(),
131 fAliRootMode(),
f47d5cb4 132 fMergeDirName(),
4e5c5506 133 fInputFiles(0),
28ce6c9c 134 fPackages(0),
77f34eae 135 fModules(0),
28ce6c9c 136 fProofParam()
c57f56b7 137{
138// Dummy ctor.
139 SetDefaults();
140}
141
142//______________________________________________________________________________
143AliAnalysisAlien::AliAnalysisAlien(const char *name)
144 :AliAnalysisGrid(name),
145 fGridJDL(NULL),
0f389141 146 fMergingJDL(NULL),
c57f56b7 147 fPrice(0),
148 fTTL(0),
149 fSplitMaxInputFileNumber(0),
150 fMaxInitFailed(0),
151 fMasterResubmitThreshold(0),
bb885a9e 152 fNtestFiles(0),
319593fb 153 fNrunsPerMaster(0),
16a4353c 154 fMaxMergeFiles(0),
b3e07543 155 fMaxMergeStages(0),
d2a409b2 156 fNsubmitted(0),
a3e84053 157 fProductionMode(0),
cd11251e 158 fOutputToRunNo(0),
0f389141 159 fMergeViaJDL(0),
160 fFastReadOption(0),
e1c22e21 161 fOverwriteMode(1),
149d288c 162 fNreplicas(2),
3bdcb562 163 fNproofWorkers(0),
27734f0e 164 fNproofWorkersPerSlave(0),
3bdcb562 165 fProofReset(0),
c57f56b7 166 fRunNumbers(),
167 fExecutable(),
0a1c1f7f 168 fExecutableCommand(),
c57f56b7 169 fArguments(),
631c0b05 170 fExecutableArgs(),
c57f56b7 171 fAnalysisMacro(),
172 fAnalysisSource(),
d3b18c4c 173 fValidationScript(),
d5c6455a 174 fAdditionalRootLibs(),
c57f56b7 175 fAdditionalLibs(),
176 fSplitMode(),
177 fAPIVersion(),
178 fROOTVersion(),
179 fAliROOTVersion(),
648174cf 180 fExternalPackages(),
c57f56b7 181 fUser(),
182 fGridWorkingDir(),
183 fGridDataDir(),
184 fDataPattern(),
185 fGridOutputDir(),
186 fOutputArchive(),
187 fOutputFiles(),
188 fInputFormat(),
e7c71df0 189 fDatasetName(),
c57f56b7 190 fJDLName(),
d3b18c4c 191 fTerminateFiles(),
bb885a9e 192 fMergeExcludes(),
37d21c01 193 fRegisterExcludes(),
f965131e 194 fIncludePath(),
bb885a9e 195 fCloseSE(),
0df6ccf2 196 fFriendChainName(),
c6cb3634 197 fJobTag(),
648174cf 198 fOutputSingle(),
5fce53f4 199 fRunPrefix(),
3bdcb562 200 fProofCluster(),
201 fProofDataSet(),
202 fFileForTestMode(),
203 fRootVersionForProof(),
204 fAliRootMode(),
f47d5cb4 205 fMergeDirName(),
4e5c5506 206 fInputFiles(0),
28ce6c9c 207 fPackages(0),
77f34eae 208 fModules(0),
28ce6c9c 209 fProofParam()
c57f56b7 210{
211// Default ctor.
212 SetDefaults();
213}
214
215//______________________________________________________________________________
216AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
217 :AliAnalysisGrid(other),
218 fGridJDL(NULL),
0f389141 219 fMergingJDL(NULL),
c57f56b7 220 fPrice(other.fPrice),
221 fTTL(other.fTTL),
222 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
223 fMaxInitFailed(other.fMaxInitFailed),
224 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 225 fNtestFiles(other.fNtestFiles),
319593fb 226 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 227 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 228 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 229 fNsubmitted(other.fNsubmitted),
a3e84053 230 fProductionMode(other.fProductionMode),
cd11251e 231 fOutputToRunNo(other.fOutputToRunNo),
0f389141 232 fMergeViaJDL(other.fMergeViaJDL),
233 fFastReadOption(other.fFastReadOption),
234 fOverwriteMode(other.fOverwriteMode),
149d288c 235 fNreplicas(other.fNreplicas),
3bdcb562 236 fNproofWorkers(other.fNproofWorkers),
27734f0e 237 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 238 fProofReset(other.fProofReset),
c57f56b7 239 fRunNumbers(other.fRunNumbers),
240 fExecutable(other.fExecutable),
0a1c1f7f 241 fExecutableCommand(other.fExecutableCommand),
c57f56b7 242 fArguments(other.fArguments),
631c0b05 243 fExecutableArgs(other.fExecutableArgs),
c57f56b7 244 fAnalysisMacro(other.fAnalysisMacro),
245 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 246 fValidationScript(other.fValidationScript),
d5c6455a 247 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 248 fAdditionalLibs(other.fAdditionalLibs),
249 fSplitMode(other.fSplitMode),
250 fAPIVersion(other.fAPIVersion),
251 fROOTVersion(other.fROOTVersion),
252 fAliROOTVersion(other.fAliROOTVersion),
648174cf 253 fExternalPackages(other.fExternalPackages),
c57f56b7 254 fUser(other.fUser),
255 fGridWorkingDir(other.fGridWorkingDir),
256 fGridDataDir(other.fGridDataDir),
257 fDataPattern(other.fDataPattern),
258 fGridOutputDir(other.fGridOutputDir),
259 fOutputArchive(other.fOutputArchive),
260 fOutputFiles(other.fOutputFiles),
261 fInputFormat(other.fInputFormat),
e7c71df0 262 fDatasetName(other.fDatasetName),
c57f56b7 263 fJDLName(other.fJDLName),
d3b18c4c 264 fTerminateFiles(other.fTerminateFiles),
bb885a9e 265 fMergeExcludes(other.fMergeExcludes),
37d21c01 266 fRegisterExcludes(other.fRegisterExcludes),
f965131e 267 fIncludePath(other.fIncludePath),
bb885a9e 268 fCloseSE(other.fCloseSE),
0df6ccf2 269 fFriendChainName(other.fFriendChainName),
c6cb3634 270 fJobTag(other.fJobTag),
648174cf 271 fOutputSingle(other.fOutputSingle),
5fce53f4 272 fRunPrefix(other.fRunPrefix),
3bdcb562 273 fProofCluster(other.fProofCluster),
274 fProofDataSet(other.fProofDataSet),
275 fFileForTestMode(other.fFileForTestMode),
276 fRootVersionForProof(other.fRootVersionForProof),
277 fAliRootMode(other.fAliRootMode),
f47d5cb4 278 fMergeDirName(other.fMergeDirName),
4e5c5506 279 fInputFiles(0),
28ce6c9c 280 fPackages(0),
77f34eae 281 fModules(0),
28ce6c9c 282 fProofParam()
c57f56b7 283{
284// Copy ctor.
285 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 286 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 287 fRunRange[0] = other.fRunRange[0];
288 fRunRange[1] = other.fRunRange[1];
c57f56b7 289 if (other.fInputFiles) {
290 fInputFiles = new TObjArray();
291 TIter next(other.fInputFiles);
292 TObject *obj;
293 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
294 fInputFiles->SetOwner();
295 }
4e5c5506 296 if (other.fPackages) {
297 fPackages = new TObjArray();
298 TIter next(other.fPackages);
299 TObject *obj;
300 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
301 fPackages->SetOwner();
302 }
77f34eae 303 if (other.fModules) {
304 fModules = new TObjArray();
305 fModules->SetOwner();
306 TIter next(other.fModules);
307 AliAnalysisTaskCfg *mod, *crt;
308 while ((crt=(AliAnalysisTaskCfg*)next())) {
309 mod = new AliAnalysisTaskCfg(*crt);
310 fModules->Add(mod);
311 }
312 }
c57f56b7 313}
314
315//______________________________________________________________________________
316AliAnalysisAlien::~AliAnalysisAlien()
317{
318// Destructor.
77f34eae 319 delete fGridJDL;
320 delete fMergingJDL;
321 delete fInputFiles;
322 delete fPackages;
323 delete fModules;
28ce6c9c 324 fProofParam.DeleteAll();
c57f56b7 325}
326
327//______________________________________________________________________________
328AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
329{
330// Assignment.
331 if (this != &other) {
332 AliAnalysisGrid::operator=(other);
333 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 334 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 335 fPrice = other.fPrice;
336 fTTL = other.fTTL;
337 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
338 fMaxInitFailed = other.fMaxInitFailed;
339 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 340 fNtestFiles = other.fNtestFiles;
a3e84053 341 fNrunsPerMaster = other.fNrunsPerMaster;
342 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 343 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 344 fNsubmitted = other.fNsubmitted;
345 fProductionMode = other.fProductionMode;
cd11251e 346 fOutputToRunNo = other.fOutputToRunNo;
0f389141 347 fMergeViaJDL = other.fMergeViaJDL;
348 fFastReadOption = other.fFastReadOption;
349 fOverwriteMode = other.fOverwriteMode;
149d288c 350 fNreplicas = other.fNreplicas;
3bdcb562 351 fNproofWorkers = other.fNproofWorkers;
27734f0e 352 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 353 fProofReset = other.fProofReset;
c57f56b7 354 fRunNumbers = other.fRunNumbers;
355 fExecutable = other.fExecutable;
0a1c1f7f 356 fExecutableCommand = other.fExecutableCommand;
c57f56b7 357 fArguments = other.fArguments;
631c0b05 358 fExecutableArgs = other.fExecutableArgs;
c57f56b7 359 fAnalysisMacro = other.fAnalysisMacro;
360 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 361 fValidationScript = other.fValidationScript;
d5c6455a 362 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 363 fAdditionalLibs = other.fAdditionalLibs;
364 fSplitMode = other.fSplitMode;
365 fAPIVersion = other.fAPIVersion;
366 fROOTVersion = other.fROOTVersion;
367 fAliROOTVersion = other.fAliROOTVersion;
648174cf 368 fExternalPackages = other.fExternalPackages;
c57f56b7 369 fUser = other.fUser;
370 fGridWorkingDir = other.fGridWorkingDir;
371 fGridDataDir = other.fGridDataDir;
372 fDataPattern = other.fDataPattern;
373 fGridOutputDir = other.fGridOutputDir;
374 fOutputArchive = other.fOutputArchive;
375 fOutputFiles = other.fOutputFiles;
376 fInputFormat = other.fInputFormat;
e7c71df0 377 fDatasetName = other.fDatasetName;
c57f56b7 378 fJDLName = other.fJDLName;
d3b18c4c 379 fTerminateFiles = other.fTerminateFiles;
bb885a9e 380 fMergeExcludes = other.fMergeExcludes;
37d21c01 381 fRegisterExcludes = other.fRegisterExcludes;
f965131e 382 fIncludePath = other.fIncludePath;
bb885a9e 383 fCloseSE = other.fCloseSE;
0df6ccf2 384 fFriendChainName = other.fFriendChainName;
c6cb3634 385 fJobTag = other.fJobTag;
648174cf 386 fOutputSingle = other.fOutputSingle;
5fce53f4 387 fRunPrefix = other.fRunPrefix;
3bdcb562 388 fProofCluster = other.fProofCluster;
389 fProofDataSet = other.fProofDataSet;
390 fFileForTestMode = other.fFileForTestMode;
391 fRootVersionForProof = other.fRootVersionForProof;
392 fAliRootMode = other.fAliRootMode;
f47d5cb4 393 fMergeDirName = other.fMergeDirName;
c57f56b7 394 if (other.fInputFiles) {
395 fInputFiles = new TObjArray();
396 TIter next(other.fInputFiles);
397 TObject *obj;
398 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
399 fInputFiles->SetOwner();
400 }
4e5c5506 401 if (other.fPackages) {
402 fPackages = new TObjArray();
403 TIter next(other.fPackages);
404 TObject *obj;
405 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
406 fPackages->SetOwner();
407 }
77f34eae 408 if (other.fModules) {
409 fModules = new TObjArray();
410 fModules->SetOwner();
411 TIter next(other.fModules);
412 AliAnalysisTaskCfg *mod, *crt;
413 while ((crt=(AliAnalysisTaskCfg*)next())) {
414 mod = new AliAnalysisTaskCfg(*crt);
415 fModules->Add(mod);
416 }
417 }
c57f56b7 418 }
419 return *this;
420}
421
37d21c01 422//______________________________________________________________________________
423void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
424{
425// Add a single additional library to be loaded. Extension must be present.
426 TString lib(name);
427 if (!lib.Contains(".")) {
428 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
429 return;
430 }
431 if (fAdditionalLibs.Contains(name)) {
432 Warning("AddAdditionalLibrary", "Library %s already added.", name);
433 return;
434 }
435 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
436 fAdditionalLibs += lib;
437}
438
77f34eae 439//______________________________________________________________________________
440void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
441{
442// Adding a module. Checks if already existing. Becomes owned by this.
443 if (!module) return;
444 if (GetModule(module->GetName())) {
445 Error("AddModule", "A module having the same name %s already added", module->GetName());
446 return;
447 }
448 if (!fModules) {
449 fModules = new TObjArray();
450 fModules->SetOwner();
451 }
452 fModules->Add(module);
453}
454
455//______________________________________________________________________________
456void AliAnalysisAlien::AddModules(TObjArray *list)
457{
458// Adding a list of modules. Checks if already existing. Becomes owned by this.
459 TIter next(list);
460 AliAnalysisTaskCfg *module;
461 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
462}
463
464//______________________________________________________________________________
465Bool_t AliAnalysisAlien::CheckDependencies()
466{
467// Check if all dependencies are satisfied. Reorder modules if needed.
468 Int_t nmodules = GetNmodules();
469 if (!nmodules) {
470 Warning("CheckDependencies", "No modules added yet to check their dependencies");
471 return kTRUE;
472 }
473 AliAnalysisTaskCfg *mod = 0;
474 AliAnalysisTaskCfg *dep = 0;
475 TString depname;
476 Int_t i, j, k;
477 for (i=0; i<nmodules; i++) {
478 mod = (AliAnalysisTaskCfg*) fModules->At(i);
479 Int_t ndeps = mod->GetNdeps();
480 Int_t istart = i;
481 for (j=0; j<ndeps; j++) {
482 depname = mod->GetDependency(j);
483 dep = GetModule(depname);
484 if (!dep) {
485 Error("CheckDependencies","Dependency %s not added for module %s",
486 depname.Data(), mod->GetName());
487 return kFALSE;
488 }
489 if (dep->NeedsDependency(mod->GetName())) {
490 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
491 mod->GetName(), dep->GetName());
492 return kFALSE;
493 }
494 Int_t idep = fModules->IndexOf(dep);
495 // The dependency task must come first
496 if (idep>i) {
497 // Remove at idep and move all objects below up one slot
498 // down to index i included.
499 fModules->RemoveAt(idep);
3e40fd4c 500 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 501 fModules->AddAt(dep, i++);
502 }
503 //Redo from istart if dependencies were inserted
504 if (i>istart) i=istart-1;
505 }
506 }
507 return kTRUE;
508}
509
3e40fd4c 510//______________________________________________________________________________
511AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
512{
513// Create the analysis manager and optionally execute the macro in filename.
514 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
515 if (mgr) return mgr;
516 mgr = new AliAnalysisManager(name);
517 mgr->SetGridHandler((AliAnalysisGrid*)this);
518 if (strlen(filename)) {
519 TString line = gSystem->ExpandPathName(filename);
520 line.Prepend(".x ");
521 gROOT->ProcessLine(line.Data());
522 }
523 return mgr;
524}
525
77f34eae 526//______________________________________________________________________________
527Int_t AliAnalysisAlien::GetNmodules() const
528{
529// Get number of modules.
530 if (!fModules) return 0;
531 return fModules->GetEntries();
532}
533
534//______________________________________________________________________________
535AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
536{
537// Get a module by name.
538 if (!fModules) return 0;
539 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
540}
541
542//______________________________________________________________________________
543Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
544{
545// Load a given module.
546 if (mod->IsLoaded()) return kTRUE;
192ddca2 547 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
548 if (!mgr) {
549 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
550 return kFALSE;
551 }
77f34eae 552 Int_t ndeps = mod->GetNdeps();
553 TString depname;
554 for (Int_t j=0; j<ndeps; j++) {
555 depname = mod->GetDependency(j);
556 AliAnalysisTaskCfg *dep = GetModule(depname);
557 if (!dep) {
558 Error("LoadModule","Dependency %s not existing for module %s",
559 depname.Data(), mod->GetName());
560 return kFALSE;
561 }
562 if (!LoadModule(dep)) {
563 Error("LoadModule","Dependency %s for module %s could not be loaded",
564 depname.Data(), mod->GetName());
565 return kFALSE;
566 }
567 }
568 // Load libraries for the module
569 if (!mod->CheckLoadLibraries()) {
570 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
571 return kFALSE;
572 }
192ddca2 573 // Check if a custom file name was requested
574 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
575
576 // Check if a custom terminate file name was requested
577 if (strlen(mod->GetTerminateFileName())) {
578 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
579 fTerminateFiles += mod->GetTerminateFileName();
580 }
581
77f34eae 582 // Execute the macro
583 if (mod->ExecuteMacro()<0) {
584 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
585 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
586 return kFALSE;
587 }
588 // Configure dependencies
589 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
590 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
591 mod->GetConfigMacro()->GetTitle(), mod->GetName());
592 return kFALSE;
593 }
5e8c7c22 594 // Adjust extra libraries
595 Int_t nlibs = mod->GetNlibs();
596 TString lib;
597 for (Int_t i=0; i<nlibs; i++) {
598 lib = mod->GetLibrary(i);
599 if (fAdditionalLibs.Contains(lib)) continue;
600 lib = Form("lib%s.so", lib.Data());
601 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
602 fAdditionalLibs += lib;
603 }
77f34eae 604 return kTRUE;
605}
606
139fe6ba 607//______________________________________________________________________________
608Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
609{
610// Generate the full train.
611 fAdditionalLibs = "";
612 if (!LoadModules()) return kFALSE;
613 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
614 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 615 mgr->RunLocalInit();
139fe6ba 616 mgr->PrintStatus();
617 Int_t productionMode = fProductionMode;
618 SetProductionMode();
619 TString macro = fAnalysisMacro;
620 TString executable = fExecutable;
621 TString validation = fValidationScript;
622 TString execCommand = fExecutableCommand;
623 SetAnalysisMacro(Form("%s.C", name));
624 SetExecutable(Form("%s.sh", name));
a8e31fb0 625// SetExecutableCommand("aliroot -b -q ");
139fe6ba 626 SetValidationScript(Form("%s_validation.sh", name));
627 StartAnalysis();
628 SetProductionMode(productionMode);
629 fAnalysisMacro = macro;
630 fExecutable = executable;
631 fExecutableCommand = execCommand;
632 fValidationScript = validation;
633 return kTRUE;
634}
635
77f34eae 636//______________________________________________________________________________
21ca8e59 637Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 638{
639// Generate test macros for a single module or for the full train.
5e8c7c22 640 fAdditionalLibs = "";
77f34eae 641 if (strlen(modname)) {
21ca8e59 642 if (!CheckDependencies()) return kFALSE;
643 AliAnalysisTaskCfg *mod = GetModule(modname);
644 if (!mod) {
645 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
646 return kFALSE;
647 }
648 if (!LoadModule(mod)) return kFALSE;
649 } else if (!LoadModules()) return kFALSE;
650 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
651 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 652 mgr->RunLocalInit();
21ca8e59 653 mgr->PrintStatus();
654 SetLocalTest(kTRUE);
655 Int_t productionMode = fProductionMode;
656 SetProductionMode();
657 TString macro = fAnalysisMacro;
658 TString executable = fExecutable;
659 TString validation = fValidationScript;
660 TString execCommand = fExecutableCommand;
661 SetAnalysisMacro(Form("%s.C", name));
662 SetExecutable(Form("%s.sh", name));
a8e31fb0 663// SetExecutableCommand("aliroot -b -q ");
21ca8e59 664 SetValidationScript(Form("%s_validation.sh", name));
665 WriteAnalysisFile();
666 WriteAnalysisMacro();
667 WriteExecutable();
668 WriteValidationScript();
669 SetLocalTest(kFALSE);
670 SetProductionMode(productionMode);
671 fAnalysisMacro = macro;
672 fExecutable = executable;
673 fExecutableCommand = execCommand;
674 fValidationScript = validation;
77f34eae 675 return kTRUE;
676}
677
678//______________________________________________________________________________
679Bool_t AliAnalysisAlien::LoadModules()
680{
681// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 682 fAdditionalLibs = "";
21ca8e59 683 Int_t nmodules = GetNmodules();
684 if (!nmodules) {
685 Warning("LoadModules", "No module to be loaded");
686 return kTRUE;
687 }
3e40fd4c 688 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
689 if (!mgr) {
690 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
691 return kFALSE;
692 }
77f34eae 693 if (!CheckDependencies()) return kFALSE;
21ca8e59 694 nmodules = GetNmodules();
77f34eae 695 AliAnalysisTaskCfg *mod;
21ca8e59 696 for (Int_t imod=0; imod<nmodules; imod++) {
697 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 698 if (!LoadModule(mod)) return kFALSE;
699 }
77f34eae 700 return kTRUE;
701}
702
74013535 703//______________________________________________________________________________
704void AliAnalysisAlien::SetRunPrefix(const char *prefix)
705{
706// Set the run number format. Can be a prefix or a format like "%09d"
707 fRunPrefix = prefix;
708 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
709}
710
f965131e 711//______________________________________________________________________________
712void AliAnalysisAlien::AddIncludePath(const char *path)
713{
714// Add include path in the remote analysis macro.
715 TString p(path);
716 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
717 else fIncludePath += Form("-I%s ", path);
718}
719
c57f56b7 720//______________________________________________________________________________
721void AliAnalysisAlien::AddRunNumber(Int_t run)
722{
723// Add a run number to the list of runs to be processed.
724 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 725 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 726}
727
98690895 728//______________________________________________________________________________
729void AliAnalysisAlien::AddRunList(const char* runList)
730{
731// Add several runs into the list of runs; they are expected to be separated by a blank character.
732 TString sList = runList;
733 TObjArray *list = sList.Tokenize(" ");
734 Int_t n = list->GetEntries();
735 for (Int_t i = 0; i < n; i++) {
736 TObjString *os = (TObjString*)list->At(i);
737 AddRunNumber(os->GetString().Atoi());
738 }
739 delete list;
740}
741
ee75cfc3 742//______________________________________________________________________________
743void AliAnalysisAlien::AddRunNumber(const char* run)
744{
745// Add a run number to the list of runs to be processed.
7aa5d539 746 TString runs = run;
747 TObjString *os;
748 TObjArray *arr = runs.Tokenize(" ");
749 TIter next(arr);
750 TString prefix;
751 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
752 while ((os=(TObjString*)next())){
753 if (fRunNumbers.Length()) fRunNumbers += " ";
754 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
755 }
756 delete arr;
ee75cfc3 757}
758
c57f56b7 759//______________________________________________________________________________
760void AliAnalysisAlien::AddDataFile(const char *lfn)
761{
762// Adds a data file to the input to be analysed. The file should be a valid LFN
763// or point to an existing file in the alien workdir.
764 if (!fInputFiles) fInputFiles = new TObjArray();
765 fInputFiles->Add(new TObjString(lfn));
766}
648174cf 767
768//______________________________________________________________________________
769void AliAnalysisAlien::AddExternalPackage(const char *package)
770{
771// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
772 if (fExternalPackages) fExternalPackages += " ";
773 fExternalPackages += package;
774}
775
c57f56b7 776//______________________________________________________________________________
777Bool_t AliAnalysisAlien::Connect()
778{
779// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
780 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 781 if (fProductionMode) return kTRUE;
c57f56b7 782 if (!gGrid) {
783 Info("Connect", "Trying to connect to AliEn ...");
784 TGrid::Connect("alien://");
785 }
786 if (!gGrid || !gGrid->IsConnected()) {
787 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
788 return kFALSE;
789 }
790 fUser = gGrid->GetUser();
791 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
792 return kTRUE;
793}
794
795//______________________________________________________________________________
796void AliAnalysisAlien::CdWork()
797{
798// Check validity of alien workspace. Create directory if possible.
799 if (!Connect()) {
800 Error("CdWork", "Alien connection required");
801 return;
802 }
803 TString homedir = gGrid->GetHomeDirectory();
804 TString workdir = homedir + fGridWorkingDir;
923e2ca5 805 if (DirectoryExists(workdir)) {
806 gGrid->Cd(workdir);
807 return;
808 }
809 // Work directory not existing - create it
810 gGrid->Cd(homedir);
b93f8109 811 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 812 gGrid->Cd(fGridWorkingDir);
d3b18c4c 813 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 814 } else {
d3b18c4c 815 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 816 workdir.Data(), homedir.Data());
817 fGridWorkingDir = "";
818 }
c57f56b7 819}
820
348be253 821//______________________________________________________________________________
822Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
823{
824// Check if file copying is possible.
d3b18c4c 825 if (fProductionMode) return kTRUE;
b5ae4fe0 826 TString salienpath(alienpath);
827 if (salienpath.Contains(" ")) {
828 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
829 return kFALSE;
830 }
348be253 831 if (!Connect()) {
832 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
833 return kFALSE;
834 }
d3339be3 835 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
836 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 837 // Check if alien_CLOSE_SE is defined
838 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
839 if (!closeSE.IsNull()) {
840 Info("CheckFileCopy", "Your current close storage is pointing to: \
841 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
842 } else {
843 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
844 }
845 // Check if grid directory exists.
846 if (!DirectoryExists(alienpath)) {
847 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
848 return kFALSE;
849 }
b5ae4fe0 850 TString stest = "plugin_test_copy";
851 TFile f(stest, "RECREATE");
348be253 852 // User may not have write permissions to current directory
853 if (f.IsZombie()) {
854 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
855 gSystem->WorkingDirectory());
856 return kFALSE;
857 }
858 f.Close();
b5ae4fe0 859 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
860 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 861 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 862 \n# 1. Make sure you have write permissions there. If this is the case: \
863 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
864 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
865 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
866 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 867 gSystem->Unlink(stest.Data());
348be253 868 return kFALSE;
869 }
b5ae4fe0 870 gSystem->Unlink(stest.Data());
871 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 872 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 873 return kTRUE;
874}
875
c57f56b7 876//______________________________________________________________________________
877Bool_t AliAnalysisAlien::CheckInputData()
878{
879// Check validity of input data. If necessary, create xml files.
d3b18c4c 880 if (fProductionMode) return kTRUE;
a8739e8a 881 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 882 if (!fGridDataDir.Length()) {
883 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
884 return kFALSE;
885 }
9f020952 886 if (fMergeViaJDL) {
887 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
888 return kFALSE;
889 }
d2a409b2 890 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 891 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
892 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 893 return kTRUE;
c57f56b7 894 }
895 // Process declared files
a2f5fc01 896 Bool_t isCollection = kFALSE;
897 Bool_t isXml = kFALSE;
898 Bool_t useTags = kFALSE;
c57f56b7 899 Bool_t checked = kFALSE;
d3b18c4c 900 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 901 TString file;
902 TString workdir = gGrid->GetHomeDirectory();
903 workdir += fGridWorkingDir;
904 if (fInputFiles) {
905 TObjString *objstr;
906 TIter next(fInputFiles);
907 while ((objstr=(TObjString*)next())) {
908 file = workdir;
909 file += "/";
910 file += objstr->GetString();
911 // Store full lfn path
912 if (FileExists(file)) objstr->SetString(file);
913 else {
914 file = objstr->GetName();
915 if (!FileExists(objstr->GetName())) {
916 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
917 objstr->GetName(), workdir.Data());
918 return kFALSE;
919 }
920 }
921 Bool_t iscoll, isxml, usetags;
922 CheckDataType(file, iscoll, isxml, usetags);
923 if (!checked) {
924 checked = kTRUE;
a2f5fc01 925 isCollection = iscoll;
926 isXml = isxml;
927 useTags = usetags;
928 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 929 } else {
a2f5fc01 930 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 931 Error("CheckInputData", "Some conflict was found in the types of inputs");
932 return kFALSE;
933 }
934 }
935 }
936 }
937 // Process requested run numbers
a8739e8a 938 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 939 // Check validity of alien data directory
940 if (!fGridDataDir.Length()) {
941 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
942 return kFALSE;
943 }
923e2ca5 944 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 945 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
946 return kFALSE;
947 }
a2f5fc01 948 if (isCollection) {
c57f56b7 949 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
950 return kFALSE;
951 }
952
a2f5fc01 953 if (checked && !isXml) {
c57f56b7 954 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
955 return kFALSE;
956 }
957 // Check validity of run number(s)
958 TObjArray *arr;
959 TObjString *os;
74013535 960 TString format;
319593fb 961 Int_t nruns = 0;
904f9f5f 962 TString schunk, schunk2;
c57f56b7 963 TString path;
964 if (!checked) {
965 checked = kTRUE;
a2f5fc01 966 useTags = fDataPattern.Contains("tag");
967 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 968 }
a2f5fc01 969 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 970 Error("CheckInputData", "Cannot mix input files using/not using tags");
971 return kFALSE;
972 }
973 if (fRunNumbers.Length()) {
a8739e8a 974 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 975 arr = fRunNumbers.Tokenize(" ");
976 TIter next(arr);
977 while ((os=(TObjString*)next())) {
978 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 979 if (!DirectoryExists(path)) {
980 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 981 continue;
c57f56b7 982 }
983 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
984 TString msg = "\n##### file: ";
985 msg += path;
986 msg += " type: xml_collection;";
a2f5fc01 987 if (useTags) msg += " using_tags: Yes";
c57f56b7 988 else msg += " using_tags: No";
23329835 989 Info("CheckDataType", "%s", msg.Data());
319593fb 990 if (fNrunsPerMaster<2) {
d2a409b2 991 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 992 } else {
993 nruns++;
994 if (((nruns-1)%fNrunsPerMaster) == 0) {
995 schunk = os->GetString();
996 }
997 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
998 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 999 AddDataFile(schunk);
319593fb 1000 }
c57f56b7 1001 }
1002 delete arr;
a8739e8a 1003 } else {
1004 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1005 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1006 format = Form("%%s/%s ", fRunPrefix.Data());
1007 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1008 if (!DirectoryExists(path)) {
a8739e8a 1009 continue;
1010 }
74013535 1011 format = Form("%%s/%s.xml", fRunPrefix.Data());
1012 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1013 TString msg = "\n##### file: ";
1014 msg += path;
1015 msg += " type: xml_collection;";
a2f5fc01 1016 if (useTags) msg += " using_tags: Yes";
a8739e8a 1017 else msg += " using_tags: No";
23329835 1018 Info("CheckDataType", "%s", msg.Data());
319593fb 1019 if (fNrunsPerMaster<2) {
74013535 1020 format = Form("%s.xml", fRunPrefix.Data());
1021 AddDataFile(Form(format.Data(),irun));
319593fb 1022 } else {
1023 nruns++;
1024 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1025 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1026 }
74013535 1027 format = Form("_%s.xml", fRunPrefix.Data());
1028 schunk2 = Form(format.Data(), irun);
319593fb 1029 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1030 schunk += schunk2;
d2a409b2 1031 AddDataFile(schunk);
319593fb 1032 }
a8739e8a 1033 }
904f9f5f 1034 if (!fInputFiles) {
1035 schunk += schunk2;
1036 AddDataFile(schunk);
1037 }
c57f56b7 1038 }
1039 return kTRUE;
1040}
1041
21ca8e59 1042//______________________________________________________________________________
43b807f6 1043Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1044{
1045// Copy data from the given grid directory according a pattern and make a local
1046// dataset.
2ba4c287 1047// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1048 if (!Connect()) {
1049 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
1050 return kFALSE;
1051 }
1052 if (!DirectoryExists(griddir)) {
1053 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
1054 return kFALSE;
1055 }
1056 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1057 printf("Running command: %s\n", command.Data());
1058 TGridResult *res = gGrid->Command(command);
1059 Int_t nfound = res->GetEntries();
1060 if (!nfound) {
1061 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
1062 return kFALSE;
1063 }
1064 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1065
1066 // archives
1067 TObjArray* additionalArchives = 0;
1068 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1069 additionalArchives = TString(archivefile).Tokenize(";");
1070 archivefile = additionalArchives->At(0)->GetName();
1071 additionalArchives->RemoveAt(0);
1072 additionalArchives->Compress();
1073 }
1074
21ca8e59 1075 // Copy files locally
1076 ofstream out;
1077 out.open(output, ios::out);
1078 TMap *map;
1079 TString turl, dirname, filename, temp;
1080 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1081 gSystem->MakeDirectory(outputdir);
1082 gSystem->ChangeDirectory(outputdir);
21ca8e59 1083 for (Int_t i=0; i<nfound; i++) {
1084 map = (TMap*)res->At(i);
1085 turl = map->GetValue("turl")->GetName();
1086 filename = gSystem->BaseName(turl.Data());
1087 dirname = gSystem->DirName(turl.Data());
1088 dirname = gSystem->BaseName(dirname.Data());
1089 gSystem->MakeDirectory(dirname);
43b807f6 1090
1091 TString source(turl);
1092 TString targetFileName(filename);
1093
1094 if (strlen(archivefile) > 0) {
1095 // TODO here the archive in which the file resides should be determined
1096 // however whereis returns only a guid, and guid2lfn does not work
1097 // Therefore we use the one provided as argument for now
1098 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1099 targetFileName = archivefile;
1100 }
1101 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
2ba4c287 1102 Bool_t success = kTRUE;
1103 if (additionalArchives)
1104 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++)
1105 {
1106 TString target;
1107 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1108 gSystem->MakeDirectory(gSystem->DirName(target));
1109 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1110 }
1111
1112 if (success) {
1113 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1114 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1115 }
21ca8e59 1116 }
1117 }
1118 gSystem->ChangeDirectory(cdir);
1119 delete res;
2ba4c287 1120 if (additionalArchives)
1121 delete additionalArchives;
21ca8e59 1122 return kTRUE;
1123}
1124
c57f56b7 1125//______________________________________________________________________________
1126Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1127{
1128// Create dataset for the grid data directory + run number.
3af861ff 1129 const Int_t gMaxEntries = 15000;
d3b18c4c 1130 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1131 if (!Connect()) {
1132 Error("CreateDataset", "Cannot create dataset with no grid connection");
1133 return kFALSE;
1134 }
1135
1136 // Cd workspace
d3b18c4c 1137 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1138 TString workdir = gGrid->GetHomeDirectory();
1139 workdir += fGridWorkingDir;
1140
1141 // Compose the 'find' command arguments
74013535 1142 TString format;
c57f56b7 1143 TString command;
1144 TString options = "-x collection ";
bb885a9e 1145 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1146 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1147 TString conditions = "";
3af861ff 1148 Int_t nstart = 0;
1149 Int_t ncount = 0;
1150 Int_t stage = 0;
c57f56b7 1151 TString file;
1152 TString path;
319593fb 1153 Int_t nruns = 0;
904f9f5f 1154 TString schunk, schunk2;
ab254fd1 1155 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1156 if (!fRunNumbers.Length() && !fRunRange[0]) {
1157 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1158 // Make a single data collection from data directory.
1159 path = fGridDataDir;
923e2ca5 1160 if (!DirectoryExists(path)) {
d2a409b2 1161 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1162 return kFALSE;
1163 }
923e2ca5 1164// CdWork();
d2a409b2 1165 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1166 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1167 while (1) {
1168 ncount = 0;
1169 stage++;
1170 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1171 command = "find ";
1172 command += Form("%s -o %d ",options.Data(), nstart);
1173 command += path;
1174 command += " ";
1175 command += pattern;
1176 command += conditions;
1177 printf("command: %s\n", command.Data());
1178 TGridResult *res = gGrid->Command(command);
1179 if (res) delete res;
1180 // Write standard output to file
1181 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1182 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1183 Bool_t nullFile = kFALSE;
1184 if (!hasGrep) {
1185 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1186 } else {
1187 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1188 if (nullFile) {
1189 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1190 gSystem->Exec("rm -f __tmp*");
1191 return kFALSE;
1192 }
1193 TString line;
1194 ifstream in;
1195 in.open("__tmp__");
1196 in >> line;
1197 in.close();
1198 gSystem->Exec("rm -f __tmp__");
1199 ncount = line.Atoi();
1200 }
1201 }
1202 if (ncount == gMaxEntries) {
1203 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1204 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1205 if (!cbase) cbase = cadd;
1206 else {
1207 cbase->Add(cadd);
1208 delete cadd;
defd7a3a 1209 }
3af861ff 1210 nstart += ncount;
1211 } else {
1212 if (cbase) {
1213 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1214 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1215 cbase->Add(cadd);
1216 delete cadd;
1217 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1218 delete cbase; cbase = 0;
1219 } else {
1220 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1221 }
1222 gSystem->Exec("rm -f __tmp*");
1223 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1224 break;
1225 }
0f389141 1226 }
1227 Bool_t fileExists = FileExists(file);
1228 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1229 // Copy xml file to alien space
0f389141 1230 if (fileExists) gGrid->Rm(file);
d2a409b2 1231 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1232 if (!FileExists(file)) {
1233 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1234 return kFALSE;
1235 }
1236 // Update list of files to be processed.
1237 }
1238 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1239 return kTRUE;
1240 }
c57f56b7 1241 // Several runs
a2f5fc01 1242 Bool_t nullResult = kTRUE;
a8739e8a 1243 if (fRunNumbers.Length()) {
1244 TObjArray *arr = fRunNumbers.Tokenize(" ");
1245 TObjString *os;
1246 TIter next(arr);
1247 while ((os=(TObjString*)next())) {
3af861ff 1248 nstart = 0;
1249 stage = 0;
dc030a24 1250 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1251 if (!DirectoryExists(path)) continue;
1252// CdWork();
a8739e8a 1253 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1254 else file = Form("%s.xml", os->GetString().Data());
319593fb 1255 // If local collection file does not exist, create it via 'find' command.
3af861ff 1256 while (1) {
1257 ncount = 0;
1258 stage++;
1259 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1260 command = "find ";
1261 command += Form("%s -o %d ",options.Data(), nstart);
1262 command += path;
1263 command += pattern;
1264 command += conditions;
1265 TGridResult *res = gGrid->Command(command);
1266 if (res) delete res;
1267 // Write standard output to file
1268 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1269 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1270 Bool_t nullFile = kFALSE;
1271 if (!hasGrep) {
1272 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1273 } else {
1274 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1275 if (nullFile) {
1276 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1277 gSystem->Exec("rm -f __tmp*");
1278 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1279 break;
3af861ff 1280 }
1281 TString line;
1282 ifstream in;
1283 in.open("__tmp__");
1284 in >> line;
1285 in.close();
1286 gSystem->Exec("rm -f __tmp__");
1287 ncount = line.Atoi();
1288 }
1289 nullResult = kFALSE;
1290 }
1291 if (ncount == gMaxEntries) {
1292 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1293 if (fNrunsPerMaster > 1) {
1294 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1295 file.Data(),gMaxEntries);
1296 return kFALSE;
1297 }
1298 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1299 if (!cbase) cbase = cadd;
1300 else {
1301 cbase->Add(cadd);
1302 delete cadd;
defd7a3a 1303 }
3af861ff 1304 nstart += ncount;
1305 } else {
c89a84bc 1306 if (cbase && fNrunsPerMaster<2) {
3af861ff 1307 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1308 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1309 cbase->Add(cadd);
1310 delete cadd;
1311 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1312 delete cbase; cbase = 0;
1313 } else {
1314 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1315 }
1316 gSystem->Exec("rm -f __tmp*");
1317 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1318 break;
d3339be3 1319 }
3af861ff 1320 }
a8739e8a 1321 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1322 // Check if there is one run per master job.
1323 if (fNrunsPerMaster<2) {
1324 if (FileExists(file)) {
0f389141 1325 if (fOverwriteMode) gGrid->Rm(file);
1326 else {
1327 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1328 continue;
1329 }
319593fb 1330 }
1331 // Copy xml file to alien space
1332 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1333 if (!FileExists(file)) {
1334 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1335 delete arr;
1336 return kFALSE;
1337 }
1338 } else {
1339 nruns++;
1340 if (((nruns-1)%fNrunsPerMaster) == 0) {
1341 schunk = os->GetString();
1342 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1343 } else {
1344 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1345 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1346 cbase->Add(cadd);
1347 delete cadd;
1348 }
1349 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1350 continue;
1351 }
1352 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1353 if (FileExists(schunk)) {
1354 if (fOverwriteMode) gGrid->Rm(file);
1355 else {
1356 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1357 continue;
1358 }
319593fb 1359 }
84fcd93f 1360 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1361 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1362 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1363 if (!FileExists(schunk)) {
1364 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1365 delete arr;
1366 return kFALSE;
1367 }
d3339be3 1368 }
a8739e8a 1369 }
1370 delete arr;
a2f5fc01 1371 if (nullResult) {
d3339be3 1372 Error("CreateDataset", "No valid dataset corresponding to the query!");
1373 return kFALSE;
1374 }
a8739e8a 1375 } else {
1376 // Process a full run range.
1377 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1378 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1379 nstart = 0;
1380 stage = 0;
74013535 1381 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1382 if (!DirectoryExists(path)) continue;
1383// CdWork();
74013535 1384 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1385 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1386 else file = Form(format.Data(), irun);
0f389141 1387 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1388 if (fOverwriteMode) gGrid->Rm(file);
1389 else {
1390 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1391 continue;
1392 }
a8739e8a 1393 }
319593fb 1394 // If local collection file does not exist, create it via 'find' command.
3af861ff 1395 while (1) {
1396 ncount = 0;
1397 stage++;
1398 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1399 command = "find ";
1400 command += Form("%s -o %d ",options.Data(), nstart);
1401 command += path;
1402 command += pattern;
1403 command += conditions;
1404 TGridResult *res = gGrid->Command(command);
1405 if (res) delete res;
1406 // Write standard output to file
1407 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1408 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1409 Bool_t nullFile = kFALSE;
1410 if (!hasGrep) {
1411 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1412 } else {
1413 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1414 if (nullFile) {
1415 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1416 gSystem->Exec("rm -f __tmp*");
1604c9ed 1417 break;
3af861ff 1418 }
1419 TString line;
1420 ifstream in;
1421 in.open("__tmp__");
1422 in >> line;
1423 in.close();
1424 gSystem->Exec("rm -f __tmp__");
1425 ncount = line.Atoi();
1426 }
1427 nullResult = kFALSE;
1428 }
1429 if (ncount == gMaxEntries) {
1430 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1431 if (fNrunsPerMaster > 1) {
1432 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1433 file.Data(),gMaxEntries);
1434 return kFALSE;
1435 }
1436 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1437 if (!cbase) cbase = cadd;
1438 else {
1439 cbase->Add(cadd);
1440 delete cadd;
defd7a3a 1441 }
3af861ff 1442 nstart += ncount;
1443 } else {
c89a84bc 1444 if (cbase && fNrunsPerMaster<2) {
3af861ff 1445 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1446 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1447 cbase->Add(cadd);
1448 delete cadd;
1449 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1450 delete cbase; cbase = 0;
1451 } else {
1452 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1453 }
1454 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1455 break;
d3339be3 1456 }
319593fb 1457 }
a8739e8a 1458 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1459 // Check if there is one run per master job.
1460 if (fNrunsPerMaster<2) {
1461 if (FileExists(file)) {
0f389141 1462 if (fOverwriteMode) gGrid->Rm(file);
1463 else {
1464 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1465 continue;
1466 }
319593fb 1467 }
1468 // Copy xml file to alien space
1469 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1470 if (!FileExists(file)) {
1471 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1472 return kFALSE;
1473 }
1474 } else {
1475 nruns++;
95e5b448 1476 // Check if the collection for the chunk exist locally.
1477 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1478 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1479 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1480 else continue;
1481 }
84fcd93f 1482 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1483 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1484 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1485 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1486 } else {
1487 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1488 cbase->Add(cadd);
1489 delete cadd;
1490 }
74013535 1491 format = Form("%%s_%s.xml", fRunPrefix.Data());
1492 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1493 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1494 continue;
1495 }
904f9f5f 1496 schunk = schunk2;
319593fb 1497 if (FileExists(schunk)) {
0f389141 1498 if (fOverwriteMode) gGrid->Rm(schunk);
1499 else {
1500 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1501 continue;
1502 }
319593fb 1503 }
84fcd93f 1504 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1505 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1506 if (FileExists(schunk)) {
0f389141 1507 if (fOverwriteMode) gGrid->Rm(schunk);
1508 else {
1509 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1510 continue;
1511 }
95e5b448 1512 }
319593fb 1513 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1514 if (!FileExists(schunk)) {
1515 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1516 return kFALSE;
1517 }
1518 }
c57f56b7 1519 }
a2f5fc01 1520 if (nullResult) {
d3339be3 1521 Error("CreateDataset", "No valid dataset corresponding to the query!");
1522 return kFALSE;
1523 }
a8739e8a 1524 }
c57f56b7 1525 return kTRUE;
1526}
1527
1528//______________________________________________________________________________
1529Bool_t AliAnalysisAlien::CreateJDL()
1530{
1531// Generate a JDL file according to current settings. The name of the file is
1532// specified by fJDLName.
1533 Bool_t error = kFALSE;
1534 TObjArray *arr = 0;
1535 Bool_t copy = kTRUE;
d3b18c4c 1536 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1537 Bool_t generate = kTRUE;
1538 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1539 if (!Connect()) {
1540 Error("CreateJDL", "Alien connection required");
1541 return kFALSE;
1542 }
1543 // Check validity of alien workspace
d3b18c4c 1544 TString workdir;
1545 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1546 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1547 workdir += fGridWorkingDir;
1548 if (generate) {
1549 TObjString *os;
1550 if (!fInputFiles) {
1551 Error("CreateJDL()", "Define some input files for your analysis.");
1552 error = kTRUE;
1553 }
1554 // Compose list of input files
1555 // Check if output files were defined
1556 if (!fOutputFiles.Length()) {
1557 Error("CreateJDL", "You must define at least one output file");
1558 error = kTRUE;
1559 }
1560 // Check if an output directory was defined and valid
1561 if (!fGridOutputDir.Length()) {
1562 Error("CreateJDL", "You must define AliEn output directory");
1563 error = kTRUE;
1564 } else {
d3b18c4c 1565 if (!fProductionMode) {
1566 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1567 if (!DirectoryExists(fGridOutputDir)) {
1568 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1569 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1570 } else {
1571 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1572 // error = kTRUE;
1573 }
67a3b254 1574 } else {
1575 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1576 }
1577 gGrid->Cd(workdir);
1578 }
c57f56b7 1579 }
1580 // Exit if any error up to now
1581 if (error) return kFALSE;
1582 // Set JDL fields
0f389141 1583 if (!fUser.IsNull()) {
1584 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1585 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1586 }
1587 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1588 TString mergeExec = fExecutable;
1589 mergeExec.ReplaceAll(".sh", "_merge.sh");
1590 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1591 mergeExec.ReplaceAll(".sh", ".C");
1592 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1593 if (!fArguments.IsNull())
1594 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1595 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1596 else {
1597 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1598 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1599 }
b3e07543 1600
1f0d1ca2 1601 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1602 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1603 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1604 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1605
0f389141 1606 if (fMaxInitFailed > 0) {
c57f56b7 1607 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1608 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1609 }
1610 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1611 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1612 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1613 }
1614 if (!IsOneStageMerging()) {
f47d5cb4 1615 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1616 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1617 }
1618 if (fSplitMode.Length()) {
c57f56b7 1619 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1620 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1621 }
1622 fMergingJDL->SetValue("Split", "\"se\"");
1623 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1624 if (!fAliROOTVersion.IsNull()) {
1625 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1626 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1627 }
1628 if (!fROOTVersion.IsNull()) {
c57f56b7 1629 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1630 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1631 }
1632 if (!fAPIVersion.IsNull()) {
c57f56b7 1633 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1634 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1635 }
648174cf 1636 if (!fExternalPackages.IsNull()) {
1637 arr = fExternalPackages.Tokenize(" ");
1638 TIter next(arr);
1639 while ((os=(TObjString*)next())) {
1640 TString pkgname = os->GetString();
1641 Int_t index = pkgname.Index("::");
1642 TString pkgversion = pkgname(index+2, pkgname.Length());
1643 pkgname.Remove(index);
1644 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1645 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1646 }
1647 delete arr;
1648 }
0f389141 1649 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1650 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1651 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1652 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1653 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1654 TString analysisFile = fExecutable;
1655 analysisFile.ReplaceAll(".sh", ".root");
1656 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1657 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1658 if (fAdditionalLibs.Length()) {
1659 arr = fAdditionalLibs.Tokenize(" ");
1660 TIter next(arr);
1661 while ((os=(TObjString*)next())) {
1662 if (os->GetString().Contains(".so")) continue;
1663 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1664 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1665 }
1666 delete arr;
1667 }
4e5c5506 1668 if (fPackages) {
1669 TIter next(fPackages);
1670 TObject *obj;
0f389141 1671 while ((obj=next())) {
4e5c5506 1672 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1673 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1674 }
4e5c5506 1675 }
c57f56b7 1676 if (fOutputArchive.Length()) {
1677 arr = fOutputArchive.Tokenize(" ");
1678 TIter next(arr);
0f389141 1679 Bool_t first = kTRUE;
1680 const char *comment = "Files to be archived";
1681 const char *comment1 = comment;
1682 while ((os=(TObjString*)next())) {
1683 if (!first) comment = NULL;
1684 if (!os->GetString().Contains("@") && fCloseSE.Length())
1685 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1686 else
1687 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1688 first = kFALSE;
1689 }
c57f56b7 1690 delete arr;
f790bc1b 1691 // Output archive for the merging jdl
1692 TString outputArchive;
1693 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1694 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1695 // Add normal output files, extra files + terminate files
1696 TString files = GetListOfFiles("outextter");
37d21c01 1697 // Do not register files in fRegisterExcludes
1698 if (!fRegisterExcludes.IsNull()) {
1699 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1700 TIter next1(arr);
1701 while ((os=(TObjString*)next1())) {
1702 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1703 files.ReplaceAll(os->GetString(),"");
1704 }
66d71516 1705 delete arr;
0f389141 1706 }
f790bc1b 1707 files.ReplaceAll(".root", "*.root");
b83f84f0 1708 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1709 } else {
95f84779 1710 TString files = fOutputArchive;
1711 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1712 outputArchive = files;
f790bc1b 1713 }
0f389141 1714 arr = outputArchive.Tokenize(" ");
705adb3e 1715 TIter next2(arr);
0f389141 1716 comment = comment1;
1717 first = kTRUE;
705adb3e 1718 while ((os=(TObjString*)next2())) {
0f389141 1719 if (!first) comment = NULL;
0cdf65a8 1720 TString currentfile = os->GetString();
0cdf65a8 1721 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1722 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1723 else
0cdf65a8 1724 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1725 first = kFALSE;
1726 }
1727 delete arr;
c57f56b7 1728 }
149d288c 1729 arr = fOutputFiles.Tokenize(",");
c57f56b7 1730 TIter next(arr);
0f389141 1731 Bool_t first = kTRUE;
f790bc1b 1732 const char *comment = "Files to be saved";
43da816a 1733 while ((os=(TObjString*)next())) {
e1eaf596 1734 // Ignore ouputs in jdl that are also in outputarchive
1735 TString sout = os->GetString();
f790bc1b 1736 sout.ReplaceAll("*", "");
1737 sout.ReplaceAll(".root", "");
e1eaf596 1738 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1739 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1740 // Ignore fRegisterExcludes
1741 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1742 if (!first) comment = NULL;
43da816a 1743 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1744 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1745 else
0f389141 1746 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1747 first = kFALSE;
f790bc1b 1748 if (fMergeExcludes.Contains(sout)) continue;
1749 if (!os->GetString().Contains("@") && fCloseSE.Length())
1750 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1751 else
1752 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1753 }
c57f56b7 1754 delete arr;
0f389141 1755 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1756 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1757 TString validationScript = fValidationScript;
0f389141 1758 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1759 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1760 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1761 if (fMasterResubmitThreshold) {
1762 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1763 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1764 }
d2a409b2 1765 // Write a jdl with 2 input parameters: collection name and output dir name.
1766 WriteJDL(copy);
c57f56b7 1767 }
1768 // Copy jdl to grid workspace
a8739e8a 1769 if (copy) {
b5e4aaa7 1770 // Check if an output directory was defined and valid
1771 if (!fGridOutputDir.Length()) {
1772 Error("CreateJDL", "You must define AliEn output directory");
1773 return kFALSE;
1774 } else {
1775 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1776 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1777 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1778 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1779 } else {
1780 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1781 return kFALSE;
1782 }
1783 }
1784 gGrid->Cd(workdir);
1785 }
648174cf 1786 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1787 TString mergeJDLName = fExecutable;
1788 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1789 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1790 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1791 if (fProductionMode) {
648174cf 1792 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1793 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1794 }
648174cf 1795 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1796 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1797 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1798 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1799 Fatal("","Terminating");
1800// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1801 if (fMergeViaJDL) {
1802 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1803// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1804 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1805 Fatal("","Terminating");
0f389141 1806 }
648174cf 1807 }
c57f56b7 1808 if (fAdditionalLibs.Length()) {
e7c71df0 1809 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1810 TObjString *os;
1811 TIter next(arr);
1812 while ((os=(TObjString*)next())) {
c57f56b7 1813 if (os->GetString().Contains(".so")) continue;
4e5c5506 1814 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1815 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1816// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1817 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1818 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1819 Fatal("","Terminating");
c57f56b7 1820 }
1821 delete arr;
1822 }
4e5c5506 1823 if (fPackages) {
1824 TIter next(fPackages);
1825 TObject *obj;
1826 while ((obj=next())) {
fdbbc7be 1827 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1828 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1829// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1830 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1831 Form("%s/%s", workdir.Data(), obj->GetName())))
1832 Fatal("","Terminating");
4e5c5506 1833 }
1834 }
c57f56b7 1835 }
1836 return kTRUE;
1837}
1838
a8739e8a 1839//______________________________________________________________________________
d2a409b2 1840Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1841{
1842// Writes one or more JDL's corresponding to findex. If findex is negative,
1843// all run numbers are considered in one go (jdl). For non-negative indices
1844// they correspond to the indices in the array fInputFiles.
1845 if (!fInputFiles) return kFALSE;
d3b18c4c 1846 TObject *os;
1847 TString workdir;
1848 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1849 workdir += fGridWorkingDir;
f47d5cb4 1850 TString stageName = "$2";
1851 if (fProductionMode) stageName = "$4";
1852 if (!fMergeDirName.IsNull()) {
1853 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1854 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1855 } else {
1856 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1857 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1858 }
d3b18c4c 1859 if (fProductionMode) {
a8739e8a 1860 TIter next(fInputFiles);
b3e07543 1861 while ((os=next())) {
d3b18c4c 1862 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1863 }
56754bf6 1864 if (!fOutputToRunNo)
1865 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1866 else
1867 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1868 } else {
1869 if (!fRunNumbers.Length() && !fRunRange[0]) {
1870 // One jdl with no parameters in case input data is specified by name.
1871 TIter next(fInputFiles);
1872 while ((os=next()))
1873 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1874 if (!fOutputSingle.IsNull())
1875 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1876 else {
1877 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1878 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1879 }
1880 } else {
1881 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1882 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1883 if (!fOutputSingle.IsNull()) {
1884 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1885 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1886 } else {
1887 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1888 }
1889 }
a8739e8a 1890 }
1891
a8739e8a 1892 // Generate the JDL as a string
1893 TString sjdl = fGridJDL->Generate();
0f389141 1894 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1895 // Final merge jdl
1896 if (!fMergeDirName.IsNull()) {
1897 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1898 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1899 } else {
1900 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1901 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1902 }
b3e07543 1903 TString sjdl2 = fMergingJDL->Generate();
1904 Int_t index, index1;
a8739e8a 1905 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1906 sjdl.ReplaceAll("(member", "\n (member");
1907 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1908 sjdl.ReplaceAll("{", "{\n ");
1909 sjdl.ReplaceAll("};", "\n};");
1910 sjdl.ReplaceAll("{\n \n", "{\n");
1911 sjdl.ReplaceAll("\n\n", "\n");
1912 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1913 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1914 sjdl1.ReplaceAll("(member", "\n (member");
1915 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1916 sjdl1.ReplaceAll("{", "{\n ");
1917 sjdl1.ReplaceAll("};", "\n};");
1918 sjdl1.ReplaceAll("{\n \n", "{\n");
1919 sjdl1.ReplaceAll("\n\n", "\n");
1920 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1921 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1922 sjdl2.ReplaceAll("(member", "\n (member");
1923 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1924 sjdl2.ReplaceAll("{", "{\n ");
1925 sjdl2.ReplaceAll("};", "\n};");
1926 sjdl2.ReplaceAll("{\n \n", "{\n");
1927 sjdl2.ReplaceAll("\n\n", "\n");
1928 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1929 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1930 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1931 index = sjdl.Index("JDLVariables");
1932 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1933 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1934 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1935 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1936 index = fJobTag.Index(":");
1937 if (index < 0) index = fJobTag.Length();
1938 TString jobTag = fJobTag;
f47d5cb4 1939 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1940 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1941 if (fProductionMode) {
1942 sjdl1.Prepend("# Generated merging jdl (production mode) \
1943 \n# $1 = full alien path to output directory to be merged \
1944 \n# $2 = train number \
1945 \n# $3 = production (like LHC10b) \
1946 \n# $4 = merging stage \
1947 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1948 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1949 sjdl2.Prepend("# Generated merging jdl \
1950 \n# $1 = full alien path to output directory to be merged \
1951 \n# $2 = train number \
1952 \n# $3 = production (like LHC10b) \
1953 \n# $4 = merging stage \
1954 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1955 } else {
1956 sjdl1.Prepend("# Generated merging jdl \
1957 \n# $1 = full alien path to output directory to be merged \
1958 \n# $2 = merging stage \
03335324 1959 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1960 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1961 sjdl2.Prepend("# Generated merging jdl \
1962 \n# $1 = full alien path to output directory to be merged \
1963 \n# $2 = merging stage \
03335324 1964 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1965 }
0f389141 1966 index = sjdl1.Index("JDLVariables");
1967 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1968 index = sjdl2.Index("JDLVariables");
1969 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1970 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1971 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1972 index = sjdl2.Index("Split =");
1973 if (index>=0) {
1974 index1 = sjdl2.Index("\n", index);
1975 sjdl2.Remove(index, index1-index+1);
1976 }
1977 index = sjdl2.Index("SplitMaxInputFileNumber");
1978 if (index>=0) {
1979 index1 = sjdl2.Index("\n", index);
1980 sjdl2.Remove(index, index1-index+1);
1981 }
1982 index = sjdl2.Index("InputDataCollection");
1983 if (index>=0) {
1984 index1 = sjdl2.Index(";", index);
1985 sjdl2.Remove(index, index1-index+1);
1986 }
1987 index = sjdl2.Index("InputDataListFormat");
1988 if (index>=0) {
1989 index1 = sjdl2.Index("\n", index);
1990 sjdl2.Remove(index, index1-index+1);
1991 }
1992 index = sjdl2.Index("InputDataList");
1993 if (index>=0) {
1994 index1 = sjdl2.Index("\n", index);
1995 sjdl2.Remove(index, index1-index+1);
1996 }
f47d5cb4 1997 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1998 // Write jdl to file
a8739e8a 1999 ofstream out;
d2a409b2 2000 out.open(fJDLName.Data(), ios::out);
a8739e8a 2001 if (out.bad()) {
d3b18c4c 2002 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2003 return kFALSE;
2004 }
2005 out << sjdl << endl;
b3e07543 2006 out.close();
0f389141 2007 TString mergeJDLName = fExecutable;
2008 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2009 if (fMergeViaJDL) {
2010 ofstream out1;
2011 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2012 if (out1.bad()) {
d3b18c4c 2013 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2014 return kFALSE;
2015 }
2016 out1 << sjdl1 << endl;
b3e07543 2017 out1.close();
2018 ofstream out2;
2019 TString finalJDL = mergeJDLName;
2020 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2021 out2.open(finalJDL.Data(), ios::out);
2022 if (out2.bad()) {
2023 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2024 return kFALSE;
2025 }
2026 out2 << sjdl2 << endl;
2027 out2.close();
0f389141 2028 }
a8739e8a 2029
2030 // Copy jdl to grid workspace
2031 if (!copy) {
d3b18c4c 2032 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2033 } else {
d2a409b2 2034 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2035 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2036 TString finalJDL = mergeJDLName;
2037 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2038 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2039 if (fProductionMode) {
b5fe9cba 2040 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2041 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2042 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2043 }
d2a409b2 2044 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2045 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2046 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2047 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2048// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2049 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2050 Fatal("","Terminating");
0f389141 2051 if (fMergeViaJDL) {
b3e07543 2052 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2053// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2054// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2055 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2056 Fatal("","Terminating");
2057 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2058 Fatal("","Terminating");
0f389141 2059 }
a8739e8a 2060 }
2061 return kTRUE;
2062}
2063
c57f56b7 2064//______________________________________________________________________________
5513444a 2065Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2066{
2067// Returns true if file exists.
5513444a 2068 if (!gGrid) return kFALSE;
2ea55496 2069 TString slfn = lfn;
2070 slfn.ReplaceAll("alien://","");
2071 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2072 if (!res) return kFALSE;
2073 TMap *map = dynamic_cast<TMap*>(res->At(0));
2074 if (!map) {
2075 delete res;
2076 return kFALSE;
2077 }
2078 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2079 if (!objs || !objs->GetString().Length()) {
2080 delete res;
2081 return kFALSE;
2082 }
2083 delete res;
2084 return kTRUE;
2085}
2086
923e2ca5 2087//______________________________________________________________________________
2088Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2089{
2090// Returns true if directory exists. Can be also a path.
2091 if (!gGrid) return kFALSE;
2092 // Check if dirname is a path
2093 TString dirstripped = dirname;
2094 dirstripped = dirstripped.Strip();
2095 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2096 TString dir = gSystem->BaseName(dirstripped);
2097 dir += "/";
2098 TString path = gSystem->DirName(dirstripped);
2099 TGridResult *res = gGrid->Ls(path, "-F");
2100 if (!res) return kFALSE;
2101 TIter next(res);
2102 TMap *map;
2103 TObject *obj;
2104 while ((map=dynamic_cast<TMap*>(next()))) {
2105 obj = map->GetValue("name");
2106 if (!obj) break;
2107 if (dir == obj->GetName()) {
2108 delete res;
2109 return kTRUE;
2110 }
2111 }
2112 delete res;
2113 return kFALSE;
2114}
2115
c57f56b7 2116//______________________________________________________________________________
a2f5fc01 2117void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2118{
2119// Check input data type.
a2f5fc01 2120 isCollection = kFALSE;
2121 isXml = kFALSE;
2122 useTags = kFALSE;
c57f56b7 2123 if (!gGrid) {
2124 Error("CheckDataType", "No connection to grid");
2125 return;
2126 }
a2f5fc01 2127 isCollection = IsCollection(lfn);
c57f56b7 2128 TString msg = "\n##### file: ";
2129 msg += lfn;
a2f5fc01 2130 if (isCollection) {
c57f56b7 2131 msg += " type: raw_collection;";
2132 // special treatment for collections
a2f5fc01 2133 isXml = kFALSE;
c57f56b7 2134 // check for tag files in the collection
2135 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2136 if (!res) {
2137 msg += " using_tags: No (unknown)";
23329835 2138 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2139 return;
2140 }
2141 const char* typeStr = res->GetKey(0, "origLFN");
2142 if (!typeStr || !strlen(typeStr)) {
2143 msg += " using_tags: No (unknown)";
23329835 2144 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2145 return;
2146 }
2147 TString file = typeStr;
a2f5fc01 2148 useTags = file.Contains(".tag");
2149 if (useTags) msg += " using_tags: Yes";
c57f56b7 2150 else msg += " using_tags: No";
23329835 2151 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2152 return;
2153 }
2154 TString slfn(lfn);
2155 slfn.ToLower();
a2f5fc01 2156 isXml = slfn.Contains(".xml");
2157 if (isXml) {
c57f56b7 2158 // Open xml collection and check if there are tag files inside
2159 msg += " type: xml_collection;";
2160 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2161 if (!coll) {
2162 msg += " using_tags: No (unknown)";
23329835 2163 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2164 return;
2165 }
2166 TMap *map = coll->Next();
2167 if (!map) {
2168 msg += " using_tags: No (unknown)";
23329835 2169 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2170 return;
2171 }
2172 map = (TMap*)map->GetValue("");
2173 TString file;
2174 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2175 useTags = file.Contains(".tag");
c57f56b7 2176 delete coll;
a2f5fc01 2177 if (useTags) msg += " using_tags: Yes";
c57f56b7 2178 else msg += " using_tags: No";
23329835 2179 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2180 return;
2181 }
a2f5fc01 2182 useTags = slfn.Contains(".tag");
c57f56b7 2183 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2184 else msg += " type: unknown file;";
a2f5fc01 2185 if (useTags) msg += " using_tags: Yes";
c57f56b7 2186 else msg += " using_tags: No";
23329835 2187 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2188}
2189
4e5c5506 2190//______________________________________________________________________________
2191void AliAnalysisAlien::EnablePackage(const char *package)
2192{
2193// Enables a par file supposed to exist in the current directory.
2194 TString pkg(package);
2195 pkg.ReplaceAll(".par", "");
2196 pkg += ".par";
2197 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2198 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2199 return;
2200 }
2201 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2202 Info("EnablePackage", "AliEn plugin will use .par packages");
2203 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2204 if (!fPackages) {
2205 fPackages = new TObjArray();
2206 fPackages->SetOwner();
2207 }
2208 fPackages->Add(new TObjString(pkg));
2209}
2210
3bdcb562 2211//______________________________________________________________________________
2212TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2213{
2214// Make a tree from files having the location specified in fFileForTestMode.
2215// Inspired from JF's CreateESDChain.
2216 if (fFileForTestMode.IsNull()) {
2217 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2218 return NULL;
2219 }
2220 if (gSystem->AccessPathName(fFileForTestMode)) {
2221 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2222 return NULL;
2223 }
2224 // Open the file
2225 ifstream in;
2226 in.open(fFileForTestMode);
2227 Int_t count = 0;
2228 // Read the input list of files and add them to the chain
b505f1f8 2229 TString line;
2230 TChain *chain = new TChain(treeName);
2231 TChain *chainFriend = 0;
2232 if (!fFriendChainName.IsNull()) chainFriend = new TChain(treeName);
2233 while (in.good())
2234 {
3bdcb562 2235 in >> line;
2236 if (line.IsNull()) continue;
2237 if (count++ == fNtestFiles) break;
2238 TString esdFile(line);
2239 TFile *file = TFile::Open(esdFile);
b505f1f8 2240 if (file && !file->IsZombie()) {
2241 chain->Add(esdFile);
3bdcb562 2242 file->Close();
b505f1f8 2243 if (!fFriendChainName.IsNull()) {
4b5d9e21 2244 if (esdFile.Index("#") > -1)
2245 esdFile.Remove(esdFile.Index("#"));
2246 esdFile = gSystem->DirName(esdFile);
2247 esdFile += "/" + fFriendChainName;
b505f1f8 2248 file = TFile::Open(esdFile);
2249 if (file && !file->IsZombie()) {
2250 file->Close();
2251 chainFriend->Add(esdFile);
2252 } else {
2253 Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data());
2254 return 0;
2255 }
4b5d9e21 2256 }
3bdcb562 2257 } else {
2258 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2259 }
b505f1f8 2260 }
2261 in.close();
2262 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2263 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2264 delete chain;
b505f1f8 2265 delete chainFriend;
3bdcb562 2266 return NULL;
b505f1f8 2267 }
3bdcb562 2268// chain->ls();
b505f1f8 2269 if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend);
2270 return chain;
3bdcb562 2271}
2272
d2a409b2 2273//______________________________________________________________________________
2274const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2275{
2276// Get job status for all jobs with jobid>jobidstart.
2277 static char mstatus[20];
2278 mstatus[0] = '\0';
2279 nrunning = 0;
2280 nwaiting = 0;
2281 nerror = 0;
2282 ndone = 0;
2283 TGridJobStatusList *list = gGrid->Ps("");
2284 if (!list) return mstatus;
2285 Int_t nentries = list->GetSize();
2286 TGridJobStatus *status;
2287 Int_t pid;
2288 for (Int_t ijob=0; ijob<nentries; ijob++) {
2289 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2290 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2291 if (pid<jobidstart) continue;
2292 if (pid == lastid) {
8ddad121 2293 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2294 }
2295 switch (status->GetStatus()) {
2296 case TGridJobStatus::kWAITING:
2297 nwaiting++; break;
2298 case TGridJobStatus::kRUNNING:
2299 nrunning++; break;
2300 case TGridJobStatus::kABORTED:
2301 case TGridJobStatus::kFAIL:
2302 case TGridJobStatus::kUNKNOWN:
2303 nerror++; break;
2304 case TGridJobStatus::kDONE:
2305 ndone++;
2306 }
2307 }
2308 list->Delete();
2309 delete list;
2310 return mstatus;
2311}
2312
c57f56b7 2313//______________________________________________________________________________
2314Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2315{
2316// Returns true if file is a collection. Functionality duplicated from
2317// TAlien::Type() because we don't want to directly depend on TAlien.
2318 if (!gGrid) {
2319 Error("IsCollection", "No connection to grid");
2320 return kFALSE;
2321 }
2322 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2323 if (!res) return kFALSE;
2324 const char* typeStr = res->GetKey(0, "type");
2325 if (!typeStr || !strlen(typeStr)) return kFALSE;
2326 if (!strcmp(typeStr, "collection")) return kTRUE;
2327 delete res;
2328 return kFALSE;
2329}
2330
fe2d7fc2 2331//______________________________________________________________________________
2332Bool_t AliAnalysisAlien::IsSingleOutput() const
2333{
2334// Check if single-ouput option is on.
2335 return (!fOutputSingle.IsNull());
2336}
2337
16a4353c 2338//______________________________________________________________________________
2339void AliAnalysisAlien::Print(Option_t *) const
2340{
2341// Print current plugin settings.
84fcd93f 2342 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2343 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2344 if (mgr && mgr->IsProofMode()) {
2345 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2346 if (TestBit(AliAnalysisGrid::kTest))
2347 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2348 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2349 if (!fProofDataSet.IsNull())
2350 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2351 if (fProofReset==1)
2352 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2353 if (fProofReset>1)
2354 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2355 if (!fRootVersionForProof.IsNull())
2356 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2357 else
2358 printf("= ROOT version requested________________________ default\n");
2359 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2360 if (!fAliRootMode.IsNull())
2361 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2362 if (fNproofWorkers)
2363 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2364 if (fNproofWorkersPerSlave)
2365 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2366 if (TestSpecialBit(kClearPackages))
2367 printf("= ClearPackages requested...\n");
2368 if (fIncludePath.Data())
2369 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2370 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2371 if (fPackages && fPackages->GetEntries()) {
2372 TIter next(fPackages);
2373 TObject *obj;
2374 TString list;
2375 while ((obj=next())) list += obj->GetName();
2376 printf("= Par files to be used: ________________________ %s\n", list.Data());
2377 }
2378 if (TestSpecialBit(kProofConnectGrid))
2379 printf("= Requested PROOF connection to grid\n");
2380 return;
2381 }
e1c22e21 2382 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2383 if (fOverwriteMode) {
2384 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2385 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2386 }
348be253 2387 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2388 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2389 printf("= Production mode:______________________________ %d\n", fProductionMode);
2390 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2391 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2392 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2393 if (fUser.Length())
84fcd93f 2394 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2395 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2396 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2397 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2398 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2399 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2400 if (fRunNumbers.Length())
84fcd93f 2401 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2402 if (fRunRange[0])
74013535 2403 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2404 if (!fRunRange[0] && !fRunNumbers.Length()) {
2405 TIter next(fInputFiles);
2406 TObject *obj;
2407 TString list;
2408 while ((obj=next())) list += obj->GetName();
84fcd93f 2409 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2410 }
2411 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2412 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2413 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2414 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2415 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2416 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2417 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2418 printf("=====================================================================\n");
2419 printf("= Job price: ___________________________________ %d\n", fPrice);
2420 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2421 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2422 if (fMaxInitFailed>0)
84fcd93f 2423 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2424 if (fMasterResubmitThreshold>0)
84fcd93f 2425 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2426 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2427 if (fNrunsPerMaster>0)
84fcd93f 2428 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2429 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2430 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2431 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2432 if (fArguments.Length())
84fcd93f 2433 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2434 if (fExecutableArgs.Length())
2435 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2436 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2437 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2438 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2439 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2440 if (fDatasetName)
84fcd93f 2441 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2442 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2443 if (fIncludePath.Data())
84fcd93f 2444 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2445 if (fCloseSE.Length())
84fcd93f 2446 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2447 if (fFriendChainName.Length())
84fcd93f 2448 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2449 if (fPackages && fPackages->GetEntries()) {
16a4353c 2450 TIter next(fPackages);
2451 TObject *obj;
2452 TString list;
2453 while ((obj=next())) list += obj->GetName();
84fcd93f 2454 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2455 }
2456}
2457
c57f56b7 2458//______________________________________________________________________________
2459void AliAnalysisAlien::SetDefaults()
2460{
2461// Set default values for everything. What cannot be filled will be left empty.
2462 if (fGridJDL) delete fGridJDL;
2463 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2464 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2465 fPrice = 1;
2466 fTTL = 30000;
2467 fSplitMaxInputFileNumber = 100;
2468 fMaxInitFailed = 0;
2469 fMasterResubmitThreshold = 0;
bb885a9e 2470 fNtestFiles = 10;
149d288c 2471 fNreplicas = 2;
a8739e8a 2472 fRunRange[0] = 0;
2473 fRunRange[1] = 0;
8e3bd06a 2474 fRunPrefix = "%d";
319593fb 2475 fNrunsPerMaster = 1;
16a4353c 2476 fMaxMergeFiles = 100;
c57f56b7 2477 fRunNumbers = "";
2478 fExecutable = "analysis.sh";
0a1c1f7f 2479 fExecutableCommand = "root -b -q";
c57f56b7 2480 fArguments = "";
631c0b05 2481 fExecutableArgs = "";
c57f56b7 2482 fAnalysisMacro = "myAnalysis.C";
2483 fAnalysisSource = "";
2484 fAdditionalLibs = "";
2485 fSplitMode = "se";
2486 fAPIVersion = "";
2487 fROOTVersion = "";
2488 fAliROOTVersion = "";
2489 fUser = ""; // Your alien user name
2490 fGridWorkingDir = "";
2491 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2492 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2493 fFriendChainName = "";
c57f56b7 2494 fGridOutputDir = "output";
b83f84f0 2495 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2496 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2497 fInputFormat = "xml-single";
2498 fJDLName = "analysis.jdl";
c6cb3634 2499 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2500 fMergeExcludes = "";
0f389141 2501 fMergeViaJDL = 0;
348be253 2502 SetUseCopy(kTRUE);
2503 SetCheckCopy(kTRUE);
149d288c 2504 SetDefaultOutputs(kTRUE);
e1c22e21 2505 fOverwriteMode = 1;
c57f56b7 2506}
2507
0f389141 2508//______________________________________________________________________________
b3e07543 2509Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2510{
b3e07543 2511// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2512 // First check if the result is already in the output directory.
2513 if (FileExists(Form("%s/%s",aliendir,filename))) {
2514 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2515 return kFALSE;
2516 }
b3e07543 2517 // Now check the last stage done.
2518 Int_t stage = 0;
2519 while (1) {
2520 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2521 stage++;
2522 }
2523 // Next stage of merging
2524 stage++;
2525 TString pattern = "*root_archive.zip";
2526 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2527 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2528 if (res) delete res;
2529 // Write standard output to file
2530 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2531 // Count the number of files inside
2532 ifstream ifile;
2533 ifile.open(Form("Stage_%d.xml",stage));
2534 if (!ifile.good()) {
2535 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2536 return kFALSE;
b3e07543 2537 }
2538 TString line;
2539 Int_t nfiles = 0;
2540 while (!ifile.eof()) {
2541 ifile >> line;
2542 if (line.Contains("/event")) nfiles++;
7c2cd90a 2543 }
b3e07543 2544 ifile.close();
2545 if (!nfiles) {
2546 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2547 return kFALSE;
2548 } else {
2549 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2550 }
b3e07543 2551 // Copy the file in the output directory
2552 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2553// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2554 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2555 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2556 // Check if this is the last stage to be done.
2557 Bool_t laststage = (nfiles<nperchunk);
2558 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2559 if (laststage) {
2560 printf("### Submiting final merging stage %d\n", stage);
2561 TString finalJDL = jdl;
2562 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2563 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2564 Int_t jobId = SubmitSingleJob(query);
2565 if (!jobId) return kFALSE;
2566 } else {
2567 printf("### Submiting merging stage %d\n", stage);
03335324 2568 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2569 Int_t jobId = SubmitSingleJob(query);
2570 if (!jobId) return kFALSE;
7c2cd90a 2571 }
b3e07543 2572 return kTRUE;
2573}
7c2cd90a 2574
105631b5 2575//______________________________________________________________________________
2576AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2577{
2578// Loat the analysis manager from a file.
2579 TFile *file = TFile::Open(fname);
2580 if (!file) {
2581 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2582 return 0;
2583 }
2584 TIter nextkey(file->GetListOfKeys());
2585 AliAnalysisManager *mgr = 0;
2586 TKey *key;
2587 while ((key=(TKey*)nextkey())) {
2588 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2589 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2590 }
2591 if (!mgr)
2592 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2593 return mgr;
2594}
2595
7c2cd90a 2596//______________________________________________________________________________
2597Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2598{
2599// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2600 if (!gGrid) return 0;
2601 printf("=> %s ------> ",query);
2602 TGridResult *res = gGrid->Command(query);
2603 if (!res) return 0;
2604 TString jobId = res->GetKey(0,"jobId");
2605 delete res;
2606 if (jobId.IsNull()) {
2607 printf("submission failed. Reason:\n");
2608 gGrid->Stdout();
2609 gGrid->Stderr();
2610 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2611 return 0;
2612 }
2613 printf(" Job id: %s\n", jobId.Data());
2614 return atoi(jobId);
2615}
2616
2617//______________________________________________________________________________
b3e07543 2618Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2619{
b3e07543 2620// Merge given output files from basedir. Basedir can be an alien output directory
2621// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2622// files in a group (ignored for xml input). Merging can be done in stages:
2623// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2624// stage=1 : works with an xml of all root_archive.zip in the output directory
2625// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2626 TString outputFile = output;
0f389141 2627 TString command;
a2f5fc01 2628 TString outputChunk;
2629 TString previousChunk = "";
b3e07543 2630 TObjArray *listoffiles = new TObjArray();
2631// listoffiles->SetOwner();
a2f5fc01 2632 Int_t countChunk = 0;
2633 Int_t countZero = nmaxmerge;
0f389141 2634 Bool_t merged = kTRUE;
a2f5fc01 2635 Int_t index = outputFile.Index("@");
2636 if (index > 0) outputFile.Remove(index);
7c2cd90a 2637 TString inputFile = outputFile;
b3e07543 2638 TString sbasedir = basedir;
2639 if (sbasedir.Contains(".xml")) {
2640 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2641 nmaxmerge = 9999999;
2642 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2643 if (!coll) {
2644 ::Error("MergeOutput", "Input XML collection empty.");
2645 return kFALSE;
2646 }
2647 // Iterate grid collection
2648 while (coll->Next()) {
2649 TString fname = gSystem->DirName(coll->GetTURL());
2650 fname += "/";
2651 fname += inputFile;
2652 listoffiles->Add(new TNamed(fname.Data(),""));
2653 }
2654 } else {
2655 command = Form("find %s/ *%s", basedir, inputFile.Data());
2656 printf("command: %s\n", command.Data());
2657 TGridResult *res = gGrid->Command(command);
2658 if (!res) {
2659 ::Error("MergeOutput","No result for the find command\n");
2660 delete listoffiles;
2661 return kFALSE;
2662 }
2663 TIter nextmap(res);
2664 TMap *map = 0;
2665 while ((map=(TMap*)nextmap())) {
2666 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2667 if (!objs || !objs->GetString().Length()) {
2668 // Nothing found - skip this output
2669 delete res;
2670 delete listoffiles;
2671 return kFALSE;
2672 }
2673 listoffiles->Add(new TNamed(objs->GetName(),""));
2674 }
2675 delete res;
2676 }
2677 if (!listoffiles->GetEntries()) {
7c2cd90a 2678 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2679 delete listoffiles;
0f389141 2680 return kFALSE;
2681 }
2682
2683 TFileMerger *fm = 0;
b3e07543 2684 TIter next0(listoffiles);
2685 TObjArray *listoffilestmp = new TObjArray();
2686 listoffilestmp->SetOwner();
2687 TObject *nextfile;
2688 TString snextfile;
2689 // Keep only the files at upper level
2690 Int_t countChar = 0;
2691 while ((nextfile=next0())) {
2692 snextfile = nextfile->GetName();
2693 Int_t crtCount = snextfile.CountChar('/');
2694 if (nextfile == listoffiles->First()) countChar = crtCount;
2695 if (crtCount < countChar) countChar = crtCount;
2696 }
2697 next0.Reset();
2698 while ((nextfile=next0())) {
2699 snextfile = nextfile->GetName();
2700 Int_t crtCount = snextfile.CountChar('/');
2701 if (crtCount > countChar) {
2702 delete nextfile;
2703 continue;
2704 }
2705 listoffilestmp->Add(nextfile);
2706 }
2707 delete listoffiles;
2708 listoffiles = listoffilestmp; // Now contains 'good' files
2709 listoffiles->Print();
2710 TIter next(listoffiles);
7c2cd90a 2711 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2712 outputChunk = outputFile;
2713 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2714 // Check for existent temporary merge files
e1c22e21 2715 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2716 // Preserve old merging functionality for stage 0.
2717 if (stage==0) {
2718 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2719 while (1) {
2720 // Skip as many input files as in a chunk
2ea55496 2721 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2722 nextfile = next();
2723 if (!nextfile) {
2ea55496 2724 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2725 delete listoffiles;
2ea55496 2726 return kFALSE;
2727 }
b3e07543 2728 snextfile = nextfile->GetName();
7c2cd90a 2729 }
2730 outputChunk = outputFile;
2731 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2732 countChunk++;
2733 if (gSystem->AccessPathName(outputChunk)) continue;
2734 // Merged file with chunks up to <countChunk> found
2735 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2736 previousChunk = outputChunk;
2737 break;
2738 }
2739 }
2740 countZero = nmaxmerge;
2741
b3e07543 2742 while ((nextfile=next())) {
2743 snextfile = nextfile->GetName();
2ea55496 2744 // Loop 'find' results and get next LFN
7c2cd90a 2745 if (countZero == nmaxmerge) {
2746 // First file in chunk - create file merger and add previous chunk if any.
2b895f34 2747 fm = new TFileMerger(kTRUE);
7c2cd90a 2748 fm->SetFastMethod(kTRUE);
2749 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2750 outputChunk = outputFile;
2751 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2752 }
2753 // If last file found, put merged results in the output file
b3e07543 2754 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2755 // Add file to be merged and decrement chunk counter.
b3e07543 2756 fm->AddFile(snextfile);
7c2cd90a 2757 countZero--;
b3e07543 2758 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2759 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2760 // Nothing found - skip this output
2761 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2762 merged = kFALSE;
2763 break;
7c2cd90a 2764 }
2765 fm->OutputFile(outputChunk);
2766 // Merge the outputs, then go to next chunk
2767 if (!fm->Merge()) {
2768 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2769 merged = kFALSE;
2770 break;
7c2cd90a 2771 } else {
2772 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2773 gSystem->Unlink(previousChunk);
2774 }
b3e07543 2775 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2776 countChunk++;
2777 countZero = nmaxmerge;
2778 previousChunk = outputChunk;
0f389141 2779 }
0f389141 2780 }
b3e07543 2781 delete listoffiles;
2782 delete fm;
7c2cd90a 2783 return merged;
2784 }
2785 // Merging stage different than 0.
2786 // Move to the begining of the requested chunk.
2b895f34 2787 fm = new TFileMerger(kTRUE);
7c2cd90a 2788 fm->SetFastMethod(kTRUE);
b3e07543 2789 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2790 delete listoffiles;
7c2cd90a 2791 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2792 // Nothing found - skip this output
2793 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2794 delete fm;
2795 return kFALSE;
2796 }
b3e07543 2797 fm->OutputFile(outputFile);
7c2cd90a 2798 // Merge the outputs
2799 if (!fm->Merge()) {
2800 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2801 delete fm;
2802 return kFALSE;
2803 } else {
b3e07543 2804 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2805 }
7c2cd90a 2806 delete fm;
2807 return kTRUE;
0f389141 2808}
2809
c57f56b7 2810//______________________________________________________________________________
2811Bool_t AliAnalysisAlien::MergeOutputs()
2812{
2813// Merge analysis outputs existing in the AliEn space.
2814 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2815 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2816 if (!Connect()) {
2817 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2818 return kFALSE;
0f389141 2819 }
c9e8f7fd 2820 if (fMergeViaJDL) {
2821 if (!TestBit(AliAnalysisGrid::kMerge)) {
2822 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2823 return kFALSE;
2824 }
2825 if (fProductionMode) {
2826 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2827 return kFALSE;
2828 }
0f389141 2829 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2830 if (!SubmitMerging()) return kFALSE;
0f389141 2831 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2832 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2833 return kFALSE;
c57f56b7 2834 }
2835 // Get the output path
dc030a24 2836 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2837 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2838 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2839 return kFALSE;
2840 }
2841 if (!fOutputFiles.Length()) {
2842 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2843 return kFALSE;
0f389141 2844 }
2845 // Check if fast read option was requested
e1c22e21 2846 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2847 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2848 if (fFastReadOption) {
d3339be3 2849 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2850 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2851 gEnv->SetValue("XNet.ConnectTimeout",50);
2852 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2853 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2854 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2855 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2856 }
e8b839ab 2857 // Make sure we change the temporary directory
2858 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2859 // Set temporary compilation directory to current one
2860 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2861 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2862 TIter next(list);
2863 TObjString *str;
a2f5fc01 2864 TString outputFile;
c57f56b7 2865 Bool_t merged = kTRUE;
2866 while((str=(TObjString*)next())) {
a2f5fc01 2867 outputFile = str->GetString();
2868 Int_t index = outputFile.Index("@");
2869 if (index > 0) outputFile.Remove(index);
2870 TString outputChunk = outputFile;
2871 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2872 // Skip already merged outputs
a2f5fc01 2873 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2874 if (fOverwriteMode) {
a2f5fc01 2875 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2876 gSystem->Unlink(outputFile);
2877 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2878 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2879 outputChunk.Data());
2880 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2881 }
2882 } else {
a2f5fc01 2883 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2884 continue;
2885 }
2886 } else {
a2f5fc01 2887 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2888 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2889 outputChunk.Data());
2890 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2891 }
2892 }
37d21c01 2893 if (fMergeExcludes.Contains(outputFile.Data()) ||
2894 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 2895 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2896 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2897 if (!merged) {
2898 Error("MergeOutputs", "Terminate() will NOT be executed");
2899 return kFALSE;
ff07ec61 2900 }
2901 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2902 if (fileOpened) fileOpened->Close();
c57f56b7 2903 }
0f389141 2904 return kTRUE;
c57f56b7 2905}
2906
bb885a9e 2907//______________________________________________________________________________
2908void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2909{
2910// Use the output files connected to output containers from the analysis manager
2911// rather than the files defined by SetOutputFiles
2912 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2913 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2914 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2915}
2916
149d288c 2917//______________________________________________________________________________
2918void AliAnalysisAlien::SetOutputFiles(const char *list)
2919{
2920// Manually set the output files list.
2921// Removes duplicates. Not allowed if default outputs are not disabled.
2922 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2923 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2924 return;
2925 }
2926 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2927 fOutputFiles = "";
2928 TString slist = list;
2929 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2930 TObjArray *arr = slist.Tokenize(" ");
2931 TObjString *os;
2932 TIter next(arr);
2933 TString sout;
2934 while ((os=(TObjString*)next())) {
2935 sout = os->GetString();
2936 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2937 if (fOutputFiles.Contains(sout)) continue;
2938 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2939 fOutputFiles += sout;
2940 }
2941 delete arr;
f790bc1b 2942}
149d288c 2943
2944//______________________________________________________________________________
2945void AliAnalysisAlien::SetOutputArchive(const char *list)
2946{
2947// Manually set the output archive list. Free text - you are on your own...
2948// Not allowed if default outputs are not disabled.
2949 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2950 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2951 return;
2952 }
2953 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2954 fOutputArchive = list;
2955}
2956
2957//______________________________________________________________________________
2958void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2959{
2960// Setting a prefered output SE is not allowed anymore.
2961 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2962}
2963
28ce6c9c 2964//______________________________________________________________________________
2965void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
2966{
2967// Set some PROOF special parameter.
2968 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2969 if (pair) {
2970 TObject *old = pair->Key();
2971 TObject *val = pair->Value();
2972 fProofParam.Remove(old);
2973 delete old;
2974 delete val;
2975 }
2976 fProofParam.Add(new TObjString(pname), new TObjString(value));
2977}
2978
2979//______________________________________________________________________________
2980const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
2981{
2982// Returns a special PROOF parameter.
2983 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2984 if (!pair) return 0;
2985 return pair->Value()->GetName();
2986}
2987
5513444a 2988//______________________________________________________________________________
2989Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2990{
2991// Start remote grid analysis.
3bdcb562 2992 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2993 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2994 if (!mgr || !mgr->IsInitialized()) {
2995 Error("StartAnalysis", "You need an initialized analysis manager for this");
2996 return kFALSE;
2997 }
2998 // Are we in PROOF mode ?
2999 if (mgr->IsProofMode()) {
baa34407 3000 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3001 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3002 if (fProofCluster.IsNull()) {
3003 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3004 return kFALSE;
3005 }
3006 if (fProofDataSet.IsNull() && !testMode) {
3007 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3008 return kFALSE;
3009 }
3010 // Set the needed environment
3011 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3012 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3013 if (fProofReset && !testMode) {
3014 if (fProofReset==1) {
3015 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3016 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3017 } else {
3018 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3019 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3020 }
3021 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3022 return kFALSE;
3023 }
baa34407 3024
3025 if (!testMode) {
3026 // Check if there is an old active session
3027 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3028 if (nsessions) {
3029 Error("StartAnalysis","You have to reset your old session first\n");
3030 return kFALSE;
3031 }
53911ca6 3032 }
3bdcb562 3033 // Do we need to change the ROOT version ? The success of this cannot be checked.
3034 if (!fRootVersionForProof.IsNull() && !testMode) {
3035 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
3036 fProofCluster.Data(), fRootVersionForProof.Data()));
3037 }
3038 // Connect to PROOF and check the status
3039 Long_t proof = 0;
27734f0e 3040 TString sworkers;
3041 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3042 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3043 if (!testMode) {
27734f0e 3044 if (!sworkers.IsNull())
3045 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3046 else
3047 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3048 } else {
3049 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3050 if (!proof) {
3051 Error("StartAnalysis", "Could not start PROOF in test mode");
3052 return kFALSE;
3053 }
3054 }
3055 if (!proof) {
3056 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3057 return kFALSE;
3058 }
27734f0e 3059 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3060 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3061 // Set proof special parameters if any
3062 TIter nextpp(&fProofParam);
3063 TObject *proofparam;
3064 while ((proofparam=nextpp())) {
3065 TString svalue = GetProofParameter(proofparam->GetName());
3066 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3067 }
3bdcb562 3068 // Is dataset existing ?
3069 if (!testMode) {
3070 TString dataset = fProofDataSet;
3071 Int_t index = dataset.Index("#");
3072 if (index>=0) dataset.Remove(index);
830acc4c 3073// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3074// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3075// return kFALSE;
3076// }
3077// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3078 }
3079 // Is ClearPackages() needed ?
3080 if (TestSpecialBit(kClearPackages)) {
3081 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3082 gROOT->ProcessLine("gProof->ClearPackages();");
3083 }
3084 // Is a given aliroot mode requested ?
3085 TList optionsList;
1ac846e9 3086 TString parLibs;
830acc4c 3087 if (!fAliRootMode.IsNull()) {
3bdcb562 3088 TString alirootMode = fAliRootMode;
3089 if (alirootMode == "default") alirootMode = "";
3090 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3091 optionsList.SetOwner();
3092 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3093 // Check the additional libs to be loaded
3094 TString extraLibs;
1ac846e9 3095 Bool_t parMode = kFALSE;
dc1ffbcb 3096 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3097 // Parse the extra libs for .so
3098 if (fAdditionalLibs.Length()) {
3099 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3100 TIter next(list);
3101 TObjString *str;
1ac846e9 3102 while((str=(TObjString*)next())) {
3103 if (str->GetString().Contains(".so")) {
3104 if (parMode) {
3105 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3106 break;
3107 }
3108 TString stmp = str->GetName();
3109 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3110 stmp.ReplaceAll(".so","");
3111 if (!extraLibs.IsNull()) extraLibs += ":";
3112 extraLibs += stmp;
3113 continue;
3114 }
3115 if (str->GetString().Contains(".par")) {
3116 // The first par file found in the list will not allow any further .so
3117 parMode = kTRUE;
3118 if (!parLibs.IsNull()) parLibs += ":";
3119 parLibs += str->GetName();
3120 continue;
3121 }
3bdcb562 3122 }
3123 if (list) delete list;
3124 }
dc030a24 3125 if (!extraLibs.IsNull()) {
3126 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3127 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3128 }
830acc4c 3129 // Check extra includes
3130 if (!fIncludePath.IsNull()) {
3131 TString includePath = fIncludePath;
3132 includePath.ReplaceAll(" ",":");
dc030a24 3133 includePath.ReplaceAll("$ALICE_ROOT/","");
3134 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3135 includePath.ReplaceAll("-I","");
dc030a24 3136 includePath.Remove(TString::kTrailing, ':');
830acc4c 3137 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3138 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3139 }
27734f0e 3140 // Check if connection to grid is requested
3141 if (TestSpecialBit(kProofConnectGrid))
3142 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3143 // Enable AliRoot par
3144 if (testMode) {
3145 // Enable proof lite package
3146 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3147 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3148 TNamed *obj = (TNamed*)optionsList.At(i);
3149 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3150 }
3151 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3152 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3153 Info("StartAnalysis", "AliRootProofLite enabled");
3154 } else {
3155 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3156 return kFALSE;
3157 }
3158 } else {
dc030a24 3159 if ( ! fAliROOTVersion.IsNull() ) {
3160 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3161 fAliROOTVersion.Data(), &optionsList))) {
3162 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3163 return kFALSE;
3164 }
3165 }
27734f0e 3166 }
1ac846e9 3167 // Enable first par files from fAdditionalLibs
3168 if (!parLibs.IsNull()) {
3169 TObjArray *list = parLibs.Tokenize(":");
3170 TIter next(list);
3171 TObjString *package;
3172 while((package=(TObjString*)next())) {
40501bb7 3173 TString spkg = package->GetName();
3174 spkg.ReplaceAll(".par", "");
3175 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3176 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3177 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3178 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3179 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3180 return kFALSE;
3181 }
3182 } else {
3183 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3184 return kFALSE;
3185 }
3186 }
3187 if (list) delete list;
3188 }
3bdcb562 3189 } else {
3190 if (fAdditionalLibs.Contains(".so") && !testMode) {
3191 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3192 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3193 return kFALSE;
3194 }
3195 }
3196 // Enable par files if requested
3197 if (fPackages && fPackages->GetEntries()) {
3198 TIter next(fPackages);
3199 TObject *package;
3200 while ((package=next())) {
1ac846e9 3201 // Skip packages already enabled
3202 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3203 TString spkg = package->GetName();
3204 spkg.ReplaceAll(".par", "");
3205 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3206 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3207 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3208 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3209 return kFALSE;
3210 }
3211 } else {
3212 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3213 return kFALSE;
3214 }
3215 }
3216 }
3217 // Do we need to load analysis source files ?
3218 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3219 if (fAnalysisSource.Length()) {
3220 TObjArray *list = fAnalysisSource.Tokenize(" ");
3221 TIter next(list);
3222 TObjString *str;
3223 while((str=(TObjString*)next())) {
3224 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3225 }
3bdcb562 3226 if (list) delete list;
3227 }
830acc4c 3228 if (testMode) {
3229 // Register dataset to proof lite.
3230 if (fFileForTestMode.IsNull()) {
3231 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3232 return kFALSE;
3233 }
3234 if (gSystem->AccessPathName(fFileForTestMode)) {
3235 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3236 return kFALSE;
3237 }
3238 TFileCollection *coll = new TFileCollection();
3239 coll->AddFromFile(fFileForTestMode);
8ddad121 3240 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3241 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3242 }
3bdcb562 3243 return kTRUE;
3244 }
c57f56b7 3245
43da816a 3246 // Check if output files have to be taken from the analysis manager
3247 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3248 // Add output files and AOD files
3249 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3250 // Add extra files registered to the analysis manager
f790bc1b 3251 TString extra = GetListOfFiles("ext");
3252 if (!extra.IsNull()) {
149d288c 3253 extra.ReplaceAll(".root", "*.root");
f790bc1b 3254 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3255 fOutputFiles += extra;
84fcd93f 3256 }
149d288c 3257 // Compose the output archive.
b83f84f0 3258 fOutputArchive = "log_archive.zip:std*@disk=1 ";
3259 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3260 }
f7b1cbc2 3261// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3262 if (TestBit(AliAnalysisGrid::kOffline)) {
3263 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3264 \n there nor any job run. You can revise the JDL and analysis \
3265 \n macro then run the same in \"submit\" mode.");
3266 } else if (TestBit(AliAnalysisGrid::kTest)) {
3267 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3268 \n dataset.");
3269 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3270 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3271 \n space and job submitted.");
3272 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3273 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3274 if (fMergeViaJDL) CheckInputData();
5513444a 3275 return kTRUE;
c57f56b7 3276 } else {
3277 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3278 }
3279
348be253 3280 Print();
c57f56b7 3281 if (!Connect()) {
3282 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3283 return kFALSE;
16a4353c 3284 }
d3b18c4c 3285 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3286 if (!CheckInputData()) {
3287 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3288 return kFALSE;
c57f56b7 3289 }
d3339be3 3290 if (!CreateDataset(fDataPattern)) {
3291 TString serror;
3292 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3293 if (fRunNumbers.Length()) serror = "run numbers";
3294 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3295 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3296 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3297 return kFALSE;
3298 }
21ca8e59 3299 WriteAnalysisFile();
c57f56b7 3300 WriteAnalysisMacro();
3301 WriteExecutable();
3302 WriteValidationScript();
0f389141 3303 if (fMergeViaJDL) {
3304 WriteMergingMacro();
3305 WriteMergeExecutable();
3306 WriteValidationScript(kTRUE);
3307 }
5513444a 3308 if (!CreateJDL()) return kFALSE;
3309 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3310 if (testMode) {
c57f56b7 3311 // Locally testing the analysis
3312 Info("StartAnalysis", "\n_______________________________________________________________________ \
3313 \n Running analysis script in a daughter shell as on a worker node \
3314 \n_______________________________________________________________________");
149d288c 3315 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3316 TIter next(list);
3317 TObjString *str;
a2f5fc01 3318 TString outputFile;
c57f56b7 3319 while((str=(TObjString*)next())) {
a2f5fc01 3320 outputFile = str->GetString();
3321 Int_t index = outputFile.Index("@");
3322 if (index > 0) outputFile.Remove(index);
3323 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3324 }
3325 delete list;
3326 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3327 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3328// gSystem->Exec("cat stdout");
5513444a 3329 return kFALSE;
c57f56b7 3330 }
5513444a 3331 // Check if submitting is managed by LPM manager
a3e84053 3332 if (fProductionMode) {
7ac61826 3333 //TString prodfile = fJDLName;
3334 //prodfile.ReplaceAll(".jdl", ".prod");
3335 //WriteProductionFile(prodfile);
5513444a 3336 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3337 return kFALSE;
3338 }
a8739e8a 3339 // Submit AliEn job(s)
d2a409b2 3340 gGrid->Cd(fGridOutputDir);
a8739e8a 3341 TGridResult *res;
c57f56b7 3342 TString jobID = "";
d2a409b2 3343 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3344 // Submit a given xml or a set of runs
a8739e8a 3345 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3346 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3347 if (res) {
3348 const char *cjobId = res->GetKey(0,"jobId");
3349 if (!cjobId) {
a03be957 3350 gGrid->Stdout();
3351 gGrid->Stderr();
a8739e8a 3352 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3353 return kFALSE;
a8739e8a 3354 } else {
3355 Info("StartAnalysis", "\n_______________________________________________________________________ \
3356 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3357 \n_______________________________________________________________________",
3358 fJDLName.Data(), cjobId);
3359 jobID = cjobId;
3360 }
3361 delete res;
a03be957 3362 } else {
3363 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3364 return kFALSE;
a8739e8a 3365 }
3366 } else {
d2a409b2 3367 // Submit for a range of enumeration of runs.
a03be957 3368 if (!Submit()) return kFALSE;
c57f56b7 3369 }
a8739e8a 3370
c57f56b7 3371 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3372 \n You may exit at any time and terminate the job later using the option <terminate> \
3373 \n ##################################################################################", jobID.Data());
bb885a9e 3374 gSystem->Exec("aliensh");
5513444a 3375 return kTRUE;
c57f56b7 3376}
3377
f790bc1b 3378//______________________________________________________________________________
3379const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3380{
3381// Get a comma-separated list of output files of the requested type.
3382// Type can be (case unsensitive):
3383// aod - list of aod files (std, extensions and filters)
3384// out - list of output files connected to containers (but not aod's or extras)
3385// ext - list of extra files registered to the manager
3386// ter - list of files produced in terminate
3387 static TString files;
3388 files = "";
3389 TString stype = type;
3390 stype.ToLower();
3391 TString aodfiles, extra;
3392 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3393 if (!mgr) {
3394 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3395 return files.Data();
3396 }
3397 if (mgr->GetOutputEventHandler()) {
3398 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3399 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3400 if (!extraaod.IsNull()) {
3401 aodfiles += ",";
3402 aodfiles += extraaod;
3403 }
3404 }
3405 if (stype.Contains("aod")) {
3406 files = aodfiles;
3407 if (stype == "aod") return files.Data();
3408 }
3409 // Add output files that are not in the list of AOD files
3410 TString outputfiles = "";
3411 TIter next(mgr->GetOutputs());
3412 AliAnalysisDataContainer *output;
3413 const char *filename = 0;
3414 while ((output=(AliAnalysisDataContainer*)next())) {
3415 filename = output->GetFileName();
3416 if (!(strcmp(filename, "default"))) continue;
3417 if (outputfiles.Contains(filename)) continue;
3418 if (aodfiles.Contains(filename)) continue;
3419 if (!outputfiles.IsNull()) outputfiles += ",";
3420 outputfiles += filename;
3421 }
3422 if (stype.Contains("out")) {
3423 if (!files.IsNull()) files += ",";
3424 files += outputfiles;
3425 if (stype == "out") return files.Data();
3426 }
3427 // Add extra files registered to the analysis manager
3428 TString sextra;
3429 extra = mgr->GetExtraFiles();
3430 if (!extra.IsNull()) {
3431 extra.Strip();
3432 extra.ReplaceAll(" ", ",");
3433 TObjArray *fextra = extra.Tokenize(",");
3434 TIter nextx(fextra);
3435 TObject *obj;
3436 while ((obj=nextx())) {
3437 if (aodfiles.Contains(obj->GetName())) continue;
3438 if (outputfiles.Contains(obj->GetName())) continue;
3439 if (sextra.Contains(obj->GetName())) continue;
3440 if (!sextra.IsNull()) sextra += ",";
3441 sextra += obj->GetName();
3442 }
3443 delete fextra;
3444 if (stype.Contains("ext")) {
3445 if (!files.IsNull()) files += ",";
3446 files += sextra;
3447 }
3448 }
3449 if (stype == "ext") return files.Data();
3450 TString termfiles;
3451 if (!fTerminateFiles.IsNull()) {
3452 fTerminateFiles.Strip();
3453 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3454 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3455 TIter nextx(fextra);
3456 TObject *obj;
3457 while ((obj=nextx())) {
3458 if (aodfiles.Contains(obj->GetName())) continue;
3459 if (outputfiles.Contains(obj->GetName())) continue;
3460 if (termfiles.Contains(obj->GetName())) continue;
3461 if (sextra.Contains(obj->GetName())) continue;
3462 if (!termfiles.IsNull()) termfiles += ",";
3463 termfiles += obj->GetName();
3464 }
3465 delete fextra;
3466 }
3467 if (stype.Contains("ter")) {
3468 if (!files.IsNull() && !termfiles.IsNull()) {
3469 files += ",";
3470 files += termfiles;
3471 }
3472 }
3473 return files.Data();
3474}
3475
d2a409b2 3476//______________________________________________________________________________
a03be957 3477Bool_t AliAnalysisAlien::Submit()
d2a409b2 3478{
3479// Submit all master jobs.
3480 Int_t nmasterjobs = fInputFiles->GetEntries();
3481 Long_t tshoot = gSystem->Now();
a03be957 3482 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3483 while (fNsubmitted < nmasterjobs) {
3484 Long_t now = gSystem->Now();
3485 if ((now-tshoot)>30000) {
3486 tshoot = now;
a03be957 3487 if (!SubmitNext()) return kFALSE;
d2a409b2 3488 }
3489 }
a03be957 3490 return kTRUE;
d2a409b2 3491}
3492
0f389141 3493//______________________________________________________________________________
a03be957 3494Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3495{
3496// Submit all merging jobs.
dc030a24 3497 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3498 gGrid->Cd(fGridOutputDir);
3499 TString mergeJDLName = fExecutable;
3500 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3501 if (!fInputFiles) {
3502 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3503 return kFALSE;
3504 }
0f389141 3505 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3506 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3507 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3508 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3509 if (fOutputToRunNo) {
3510 // The output directory is the run number
3511 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3512 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3513 } else {
17b4e9a4 3514 if (!fRunNumbers.Length() && !fRunRange[0]) {
3515 // The output directory is the grid outdir
3516 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3517 runOutDir = fGridOutputDir;
3518 } else {
3519 // The output directory is the master number in 3 digits format
3520 printf("### Submitting merging job for master <%03d>\n", i);
3521 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3522 }
7c2cd90a 3523 }
3524 // Check now the number of merging stages.
2b222f7c 3525 TObjArray *list = fOutputFiles.Tokenize(",");
3526 TIter next(list);
3527 TObjString *str;
3528 TString outputFile;
3529 while((str=(TObjString*)next())) {
3530 outputFile = str->GetString();
3531 Int_t index = outputFile.Index("@");
3532 if (index > 0) outputFile.Remove(index);
37d21c01 3533 if (!fMergeExcludes.Contains(outputFile) &&
3534 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3535 }
3536 delete list;
b3e07543 3537 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3538 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3539 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3540 }
a03be957 3541 if (!ntosubmit) return kTRUE;
9f020952 3542 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3543 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3544 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3545 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3546 \n ################################################################################################################");
0f389141 3547 gSystem->Exec("aliensh");
a03be957 3548 return kTRUE;
0f389141 3549}
3550
d2a409b2 3551//______________________________________________________________________________
a03be957 3552Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3553{
08d5b699 3554// Submit next bunch of master jobs if the queue is free. The first master job is
3555// submitted right away, while the next will not be unless the previous was split.
3556// The plugin will not submit new master jobs if there are more that 500 jobs in
3557// waiting phase.
d2a409b2 3558 static Bool_t iscalled = kFALSE;
3559 static Int_t firstmaster = 0;
3560 static Int_t lastmaster = 0;
3561 static Int_t npermaster = 0;
a03be957 3562 if (iscalled) return kTRUE;
d2a409b2 3563 iscalled = kTRUE;
3564 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3565 Int_t ntosubmit = 0;
3566 TGridResult *res;
3567 TString jobID = "";
3bdcb562 3568 Int_t nmasterjobs = fInputFiles->GetEntries();
3569 if (!fNsubmitted) {
3570 ntosubmit = 1;
3571 if (!IsUseSubmitPolicy()) {
90d50a8c 3572 if (nmasterjobs>5)
3bdcb562 3573 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3574 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3575 ntosubmit = nmasterjobs;
3576 }
3577 } else {
d2a409b2 3578 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3579 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3580 // If last master not split, just return
a03be957 3581 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3582 // No more than 100 waiting jobs
08d5b699 3583 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3584 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3585 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3586 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3587 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3588 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3589 }
d2a409b2 3590 for (Int_t i=0; i<ntosubmit; i++) {
3591 // Submit for a range of enumeration of runs.
a03be957 3592 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3593 TString query;
cd11251e 3594 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3595 runOutDir.ReplaceAll(".xml", "");
3596 if (fOutputToRunNo)
3597 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3598 else
3599 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3600 printf("********* %s\n",query.Data());
d2a409b2 3601 res = gGrid->Command(query);
3602 if (res) {
98ca124f 3603 TString cjobId1 = res->GetKey(0,"jobId");
3604 if (!cjobId1.Length()) {
d2a409b2 3605 iscalled = kFALSE;
a03be957 3606 gGrid->Stdout();
3607 gGrid->Stderr();
3608 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3609 return kFALSE;
d2a409b2 3610 } else {
3611 Info("StartAnalysis", "\n_______________________________________________________________________ \
3612 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3613 \n_______________________________________________________________________",
98ca124f 3614 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 3615 jobID += cjobId1;
3616 jobID += " ";
98ca124f 3617 lastmaster = cjobId1.Atoi();
d2a409b2 3618 if (!firstmaster) firstmaster = lastmaster;
3619 fNsubmitted++;
3620 }
3621 delete res;
a03be957 3622 } else {
3623 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3624 return kFALSE;
d2a409b2 3625 }
3626 }
3627 iscalled = kFALSE;
a03be957 3628 return kTRUE;
d2a409b2 3629}
3630
c57f56b7 3631//______________________________________________________________________________
3632void AliAnalysisAlien::WriteAnalysisFile()
3633{
f10e8481 3634// Write current analysis manager into the file <analysisFile>
3635 TString analysisFile = fExecutable;
3636 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3637 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3638 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3639 if (!mgr || !mgr->IsInitialized()) {
3640 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3641 return;
3642 }
3643 // Check analysis type
3644 TObject *handler;
3645 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3646 handler = (TObject*)mgr->GetInputEventHandler();
3647 if (handler) {
65eb22f8 3648 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3649 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3650 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3651 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3652 } else {
3653 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3654 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3655 }
c57f56b7 3656 }
3657 TDirectory *cdir = gDirectory;
f10e8481 3658 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3659 if (file) {
e16a394c 3660 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3661 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3662 // Unless merging makes no sense
3663 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3664 mgr->Write();
3665 delete file;
fe2d7fc2 3666 // Enable termination for local jobs
3667 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3668 }
3669 if (cdir) cdir->cd();
f10e8481 3670 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3671 }
3672 Bool_t copy = kTRUE;
d3b18c4c 3673 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3674 if (copy) {
3675 CdWork();
3676 TString workdir = gGrid->GetHomeDirectory();
3677 workdir += fGridWorkingDir;
d3b18c4c 3678 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3679 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3680 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3681 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3682 }
3683}
3684
3685//______________________________________________________________________________
3686void AliAnalysisAlien::WriteAnalysisMacro()
3687{
3688// Write the analysis macro that will steer the analysis in grid mode.
3689 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3690 ofstream out;
3691 out.open(fAnalysisMacro.Data(), ios::out);
3692 if (!out.good()) {
3693 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3694 return;
3695 }
5b273635 3696 Bool_t hasSTEERBase = kFALSE;
3697 Bool_t hasESD = kFALSE;
3698 Bool_t hasAOD = kFALSE;
3699 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3700 Bool_t hasOADB = kFALSE;
5b273635 3701 Bool_t hasANALYSISalice = kFALSE;
3702 Bool_t hasCORRFW = kFALSE;
c57f56b7 3703 TString func = fAnalysisMacro;
3704 TString type = "ESD";
3705 TString comment = "// Analysis using ";
c85cfc0f 3706 if (IsUseMCchain()) {
3707 type = "MC";
3708 comment += "MC";
3709 } else {
3710 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3711 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3712 type = "AOD";
3713 comment += "AOD";
3714 }
3715 }
0df6ccf2 3716 if (type!="AOD" && fFriendChainName!="") {
3717 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3718 return;
3719 }
c57f56b7 3720 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3721 else comment += " data";
3722 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3723 func.ReplaceAll(".C", "");
3724 out << "void " << func.Data() << "()" << endl;
3725 out << "{" << endl;
3726 out << comment.Data() << endl;
3727 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3728 out << " TStopwatch timer;" << endl;
3729 out << " timer.Start();" << endl << endl;
13f0c00b 3730 // Change temp directory to current one
3731 out << "// Set temporary merging directory to current one" << endl;
3732 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3733 out << "// Set temporary compilation directory to current one" << endl;
3734 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3735 // Reset existing include path
3736 out << "// Reset existing include path and add current directory first in the search" << endl;
3737 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3738 if (!fExecutableCommand.Contains("aliroot")) {
3739 out << "// load base root libraries" << endl;
3740 out << " gSystem->Load(\"libTree\");" << endl;
3741 out << " gSystem->Load(\"libGeom\");" << endl;
3742 out << " gSystem->Load(\"libVMC\");" << endl;
3743 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3744 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3745 }
d5c6455a 3746 if (fAdditionalRootLibs.Length()) {
47a4137d 3747 // in principle libtree /lib geom libvmc etc. can go into this list, too
3748 out << "// Add aditional libraries" << endl;
3749 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3750 TIter next(list);
3751 TObjString *str;
3752 while((str=(TObjString*)next())) {
3753 if (str->GetString().Contains(".so"))
3754 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3755 }
47a4137d 3756 if (list) delete list;
d5c6455a 3757 }
57377eb5 3758 out << "// Load analysis framework libraries" << endl;
652561d5 3759 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3760 if (!fPackages) {
f790bc1b 3761 if (!fExecutableCommand.Contains("aliroot")) {
3762 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3763 out << " gSystem->Load(\"libESD\");" << endl;
3764 out << " gSystem->Load(\"libAOD\");" << endl;
3765 }
4e5c5506 3766 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3767 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3768 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3769 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3770 } else {
4e5c5506 3771 TIter next(fPackages);
3772 TObject *obj;
57377eb5 3773 TString pkgname;
57377eb5 3774 while ((obj=next())) {
3775 pkgname = obj->GetName();
4478e6f1 3776 if (pkgname == "STEERBase" ||
3777 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3778 if (pkgname == "ESD" ||
3779 pkgname == "ESD.par") hasESD = kTRUE;
3780 if (pkgname == "AOD" ||
3781 pkgname == "AOD.par") hasAOD = kTRUE;
3782 if (pkgname == "ANALYSIS" ||
3783 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 3784 if (pkgname == "OADB" ||
3785 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 3786 if (pkgname == "ANALYSISalice" ||
3787 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3788 if (pkgname == "CORRFW" ||
3789 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3790 }
3791 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3792 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3793 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3794 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3795 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3796 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3797 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3798 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3799 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 3800 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
3801 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 3802 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3803 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3804 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3805 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3806 out << "// Compile other par packages" << endl;
3807 next.Reset();
fcc9bb6f 3808 while ((obj=next())) {
3809 pkgname = obj->GetName();
4478e6f1 3810 if (pkgname == "STEERBase" ||
3811 pkgname == "STEERBase.par" ||
3812 pkgname == "ESD" ||
3813 pkgname == "ESD.par" ||
3814 pkgname == "AOD" ||
3815 pkgname == "AOD.par" ||
3816 pkgname == "ANALYSIS" ||
3817 pkgname == "ANALYSIS.par" ||
7ea40ac8 3818 pkgname == "OADB" ||
3819 pkgname == "OADB.par" ||
4478e6f1 3820 pkgname == "ANALYSISalice" ||
3821 pkgname == "ANALYSISalice.par" ||
3822 pkgname == "CORRFW" ||
3823 pkgname == "CORRFW.par") continue;
5b273635 3824 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3825 }
4e5c5506 3826 }
b3e07543 3827 out << "// include path" << endl;
3828 // Get the include path from the interpreter and remove entries pointing to AliRoot
3829 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3830 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3831 out << " TIter nextpath(listpaths);" << endl;
3832 out << " TObjString *pname;" << endl;
3833 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3834 out << " TString current = pname->GetName();" << endl;
3835 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3836 out << " gSystem->AddIncludePath(current);" << endl;
3837 out << " }" << endl;
3838 out << " if (listpaths) delete listpaths;" << endl;
3839 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3840 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3841 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3842 if (fAdditionalLibs.Length()) {
3843 out << "// Add aditional AliRoot libraries" << endl;
3844 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3845 TIter next(list);
3846 TObjString *str;
3847 while((str=(TObjString*)next())) {
3848 if (str->GetString().Contains(".so"))
3849 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3850 if (str->GetString().Contains(".par"))
3851 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3852 }
3853 if (list) delete list;
3854 }
3855 out << endl;
c57f56b7 3856 out << "// analysis source to be compiled at runtime (if any)" << endl;
3857 if (fAnalysisSource.Length()) {
3858 TObjArray *list = fAnalysisSource.Tokenize(" ");
3859 TIter next(list);
3860 TObjString *str;
3861 while((str=(TObjString*)next())) {
3862 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3863 }
3864 if (list) delete list;
3865 }
3866 out << endl;
b3e07543 3867// out << " printf(\"Currently load libraries:\\n\");" << endl;
3868// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3869 if (fFastReadOption) {
d3339be3 3870 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3871 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3872 out << "// fast xrootd reading enabled" << endl;
3873 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3874 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3875 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3876 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3877 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3878 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 3879 }
3880 if (!IsLocalTest()) {
3881 out << "// connect to AliEn and make the chain" << endl;
3882 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
0f389141 3883 }
c57f56b7 3884 out << "// read the analysis manager from file" << endl;
f10e8481 3885 TString analysisFile = fExecutable;
3886 analysisFile.ReplaceAll(".sh", ".root");
105631b5 3887 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
3888 << analysisFile << "\");" << endl;
3889 out << " if (!mgr) return;" << endl;
21ca8e59 3890 if (IsLocalTest()) {
3891 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
3892 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 3893 if (fFileForTestMode.IsNull())
3894 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
3895 else
3896 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 3897 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4b5d9e21 3898 if (!fFriendChainName.IsNull())
3899 out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\");" << endl;
21ca8e59 3900 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 3901 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 3902 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 3903 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
3904 } else {
7a366e0b 3905 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 3906 out << " mgr->SetNSysInfo(100);" << endl;
3907 }
21ca8e59 3908 }
c57f56b7 3909 out << " mgr->PrintStatus();" << endl;
52b6a92b 3910 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3911 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3912 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3913 } else {
9a8288ff 3914 if (TestBit(AliAnalysisGrid::kTest))
3915 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3916 else
3917 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3918 }
3919 }
21ca8e59 3920 if (!IsLocalTest()) {
242accb2 3921 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 3922 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
3923 } else {
3924 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 3925 }
f7498086 3926 out << " timer.Stop();" << endl;
3927 out << " timer.Print();" << endl;
c57f56b7 3928 out << "}" << endl << endl;
21ca8e59 3929 if (!IsLocalTest()) {
fcc9bb6f 3930 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3931 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3932 out << "{" << endl;
3933 out << "// Create a chain using url's from xml file" << endl;
242accb2 3934 out << " TString filename;" << endl;
3935 out << " Int_t run = 0;" << endl;
c85cfc0f 3936 if (IsUseMCchain()) {
3937 out << " TString treename = \"TE\";" << endl;
3938 } else {
3939 out << " TString treename = type;" << endl;
3940 out << " treename.ToLower();" << endl;
3941 out << " treename += \"Tree\";" << endl;
3942 }
e02fee64 3943 out << " printf(\"***************************************\\n\");" << endl;
3944 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3945 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3946 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3947 out << " if (!coll) {" << endl;
3948 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3949 out << " return NULL;" << endl;
3950 out << " }" << endl;
242accb2 3951 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3952 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3953 if(fFriendChainName!="") {
3954 out << " TChain *chainFriend = new TChain(treename);" << endl;
3955 }
c57f56b7 3956 out << " coll->Reset();" << endl;
0df6ccf2 3957 out << " while (coll->Next()) {" << endl;
242accb2 3958 out << " filename = coll->GetTURL("");" << endl;
3959 out << " if (mgr) {" << endl;
3960 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3961 out << " if (nrun && nrun != run) {" << endl;
3962 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3963 out << " mgr->SetRunFromPath(nrun);" << endl;
3964 out << " run = nrun;" << endl;
3965 out << " }" << endl;
3966 out << " }" << endl;
3967 out << " chain->Add(filename);" << endl;
0df6ccf2 3968 if(fFriendChainName!="") {
3969 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
4b5d9e21 3970 out << " if (fileFriend.Index(\"#\") > -1) fileFriend.Remove(fileFriend.Index(\"#\"));" << endl;
3971 out << " fileFriend = gSystem->DirName(fileFriend);" << endl;
3972 out << " fileFriend += \"/\";" << endl;
3973 out << " fileFriend += \"" << fFriendChainName << "\";";
3974 out << " TFile *file = TFile::Open(fileFriend);" << endl;
3975 out << " if (file) {" << endl;
3976 out << " file->Close();" << endl;
3977 out << " chainFriend->Add(fileFriend.Data());" << endl;
3978 out << " } else {" << endl;
3979 out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
3980 out << " return 0;" << endl;
3981 out << " }" << endl;
0df6ccf2 3982 }
3983 out << " }" << endl;
c57f56b7 3984 out << " if (!chain->GetNtrees()) {" << endl;
3985 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3986 out << " return NULL;" << endl;
3987 out << " }" << endl;
0df6ccf2 3988 if(fFriendChainName!="") {
3989 out << " chain->AddFriend(chainFriend);" << endl;
3990 }
c57f56b7 3991 out << " return chain;" << endl;
fcc9bb6f 3992 out << "}" << endl << endl;
c57f56b7 3993 }
5b273635 3994 if (hasANALYSISalice) {
3995 out <<"//________________________________________________________________________________" << endl;
3996 out << "Bool_t SetupPar(const char *package) {" << endl;
3997 out << "// Compile the package and set it up." << endl;
3998 out << " TString pkgdir = package;" << endl;
3999 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4000 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4001 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4002 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4003 out << " // Check for BUILD.sh and execute" << endl;
4004 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4005 out << " printf(\"*******************************\\n\");" << endl;
4006 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4007 out << " printf(\"*******************************\\n\");" << endl;
4008 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4009 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4010 out << " gSystem->ChangeDirectory(cdir);" << endl;
4011 out << " return kFALSE;" << endl;
4012 out << " }" << endl;
4013 out << " } else {" << endl;
4014 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4015 out << " gSystem->ChangeDirectory(cdir);" << endl;
4016 out << " return kFALSE;" << endl;
4017 out << " }" << endl;
4018 out << " // Check for SETUP.C and execute" << endl;
4019 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4020 out << " printf(\"*******************************\\n\");" << endl;
4021 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4022 out << " printf(\"*******************************\\n\");" << endl;
4023 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4024 out << " } else {" << endl;
4025 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4026 out << " gSystem->ChangeDirectory(cdir);" << endl;
4027 out << " return kFALSE;" << endl;
4028 out << " }" << endl;
4029 out << " // Restore original workdir" << endl;
4030 out << " gSystem->ChangeDirectory(cdir);" << endl;
4031 out << " return kTRUE;" << endl;
4032 out << "}" << endl;
4033 }
c57f56b7 4034 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4035 }
4036 Bool_t copy = kTRUE;
d3b18c4c 4037 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4038 if (copy) {
4039 CdWork();
4040 TString workdir = gGrid->GetHomeDirectory();
4041 workdir += fGridWorkingDir;
4042 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4043 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4044// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4045 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4046 Form("alien://%s/%s", workdir.Data(),
4047 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4048 }
4049}
4050
0f389141 4051//______________________________________________________________________________
4052void AliAnalysisAlien::WriteMergingMacro()
4053{
4054// Write a macro to merge the outputs per master job.
4055 if (!fMergeViaJDL) return;
4056 if (!fOutputFiles.Length()) {
4057 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4058 return;
4059 }
4060 TString mergingMacro = fExecutable;
4061 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4062 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4063 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4064 ofstream out;
4065 out.open(mergingMacro.Data(), ios::out);
4066 if (!out.good()) {
4067 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4068 return;
4069 }
5b273635 4070 Bool_t hasSTEERBase = kFALSE;
4071 Bool_t hasESD = kFALSE;
4072 Bool_t hasAOD = kFALSE;
4073 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4074 Bool_t hasOADB = kFALSE;
5b273635 4075 Bool_t hasANALYSISalice = kFALSE;
4076 Bool_t hasCORRFW = kFALSE;
0f389141 4077 TString func = mergingMacro;
4078 TString comment;
4079 func.ReplaceAll(".C", "");
f47d5cb4 4080 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4081 out << "{" << endl;
4082 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4083 out << " TStopwatch timer;" << endl;
4084 out << " timer.Start();" << endl << endl;
b3e07543 4085 // Reset existing include path
4086 out << "// Reset existing include path and add current directory first in the search" << endl;
4087 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4088 if (!fExecutableCommand.Contains("aliroot")) {
4089 out << "// load base root libraries" << endl;
4090 out << " gSystem->Load(\"libTree\");" << endl;
4091 out << " gSystem->Load(\"libGeom\");" << endl;
4092 out << " gSystem->Load(\"libVMC\");" << endl;
4093 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4094 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4095 }
0f389141 4096 if (fAdditionalRootLibs.Length()) {
4097 // in principle libtree /lib geom libvmc etc. can go into this list, too
4098 out << "// Add aditional libraries" << endl;
4099 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4100 TIter next(list);
4101 TObjString *str;
4102 while((str=(TObjString*)next())) {
4103 if (str->GetString().Contains(".so"))
4104 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4105 }
4106 if (list) delete list;
4107 }
0f389141 4108 out << "// Load analysis framework libraries" << endl;
4109 if (!fPackages) {
7c2cd90a 4110 if (!fExecutableCommand.Contains("aliroot")) {
4111 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4112 out << " gSystem->Load(\"libESD\");" << endl;
4113 out << " gSystem->Load(\"libAOD\");" << endl;
4114 }
0f389141 4115 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4116 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4117 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4118 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4119 } else {
4120 TIter next(fPackages);
4121 TObject *obj;
4122 TString pkgname;
5b273635 4123 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4124 while ((obj=next())) {
4125 pkgname = obj->GetName();
4126 if (pkgname == "STEERBase" ||
4127 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4128 if (pkgname == "ESD" ||
4129 pkgname == "ESD.par") hasESD = kTRUE;
4130 if (pkgname == "AOD" ||
4131 pkgname == "AOD.par") hasAOD = kTRUE;
4132 if (pkgname == "ANALYSIS" ||
4133 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4134 if (pkgname == "OADB" ||
4135 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4136 if (pkgname == "ANALYSISalice" ||
4137 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4138 if (pkgname == "CORRFW" ||
4139 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4140 }
5b273635 4141 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4142 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4143 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4144 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4145 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4146 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4147 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4148 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4149 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4150 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4151 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4152 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4153 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4154 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4155 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4156 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4157 out << "// Compile other par packages" << endl;
4158 next.Reset();
4159 while ((obj=next())) {
4160 pkgname = obj->GetName();
4161 if (pkgname == "STEERBase" ||
4162 pkgname == "STEERBase.par" ||
4163 pkgname == "ESD" ||
4164 pkgname == "ESD.par" ||
4165 pkgname == "AOD" ||
4166 pkgname == "AOD.par" ||
4167 pkgname == "ANALYSIS" ||
4168 pkgname == "ANALYSIS.par" ||
7ea40ac8 4169 pkgname == "OADB" ||
4170 pkgname == "OADB.par" ||
0f389141 4171 pkgname == "ANALYSISalice" ||
4172 pkgname == "ANALYSISalice.par" ||
4173 pkgname == "CORRFW" ||
4174 pkgname == "CORRFW.par") continue;
5b273635 4175 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4176 }
4177 }
b3e07543 4178 out << "// include path" << endl;
4179 // Get the include path from the interpreter and remove entries pointing to AliRoot
4180 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4181 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4182 out << " TIter nextpath(listpaths);" << endl;
4183 out << " TObjString *pname;" << endl;
4184 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4185 out << " TString current = pname->GetName();" << endl;
4186 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4187 out << " gSystem->AddIncludePath(current);" << endl;
4188 out << " }" << endl;
4189 out << " if (listpaths) delete listpaths;" << endl;
4190 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4191 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4192 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4193 if (fAdditionalLibs.Length()) {
4194 out << "// Add aditional AliRoot libraries" << endl;
4195 TObjArray *list = fAdditionalLibs.Tokenize(" ");
4196 TIter next(list);
4197 TObjString *str;
4198 while((str=(TObjString*)next())) {
4199 if (str->GetString().Contains(".so"))
4200 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4201 }
4202 if (list) delete list;
4203 }
4204 out << endl;
4205 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4206 if (fAnalysisSource.Length()) {
4207 TObjArray *list = fAnalysisSource.Tokenize(" ");
4208 TIter next(list);
4209 TObjString *str;
4210 while((str=(TObjString*)next())) {
4211 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4212 }
4213 if (list) delete list;
4214 }
149d288c 4215 out << endl;
4216
0f389141 4217 if (fFastReadOption) {
4218 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4219 out << "// fast xrootd reading enabled" << endl;
4220 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4221 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4222 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4223 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4224 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4225 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4226 }
4227 // Change temp directory to current one
4228 out << "// Set temporary merging directory to current one" << endl;
4229 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4230 out << "// Set temporary compilation directory to current one" << endl;
4231 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
0f389141 4232 out << "// Connect to AliEn" << endl;
4233 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 4234 out << " TString outputDir = dir;" << endl;
f790bc1b 4235 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4236 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4237 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4238 out << " TIter *iter = new TIter(list);" << endl;
4239 out << " TObjString *str;" << endl;
a2f5fc01 4240 out << " TString outputFile;" << endl;
0f389141 4241 out << " Bool_t merged = kTRUE;" << endl;
4242 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4243 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4244 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4245 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4246 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4247 out << " // Skip already merged outputs" << endl;
a2f5fc01 4248 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
4249 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 4250 out << " continue;" << endl;
4251 out << " }" << endl;
a2f5fc01 4252 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4253 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4254 out << " if (!merged) {" << endl;
a2f5fc01 4255 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4256 out << " return;" << endl;
0f389141 4257 out << " }" << endl;
4258 out << " }" << endl;
7c2cd90a 4259 out << " // all outputs merged, validate" << endl;
4260 out << " ofstream out;" << endl;
4261 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4262 out << " out.close();" << endl;
4263 out << " // read the analysis manager from file" << endl;
f866cba5 4264 TString analysisFile = fExecutable;
4265 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 4266 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
105631b5 4267 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4268 << analysisFile << "\");" << endl;
4269 out << " if (!mgr) return;" << endl;
5c4250fc 4270 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4271 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4272 out << " mgr->PrintStatus();" << endl;
4273 if (AliAnalysisManager::GetAnalysisManager()) {
4274 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
4275 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4276 } else {
9a8288ff 4277 if (TestBit(AliAnalysisGrid::kTest))
4278 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4279 else
4280 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4281 }
4282 }
9a8288ff 4283 out << " TTree *tree = NULL;" << endl;
4284 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4285 out << "}" << endl << endl;
5b273635 4286 if (hasANALYSISalice) {
4287 out <<"//________________________________________________________________________________" << endl;
4288 out << "Bool_t SetupPar(const char *package) {" << endl;
4289 out << "// Compile the package and set it up." << endl;
4290 out << " TString pkgdir = package;" << endl;
4291 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4292 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4293 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4294 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4295 out << " // Check for BUILD.sh and execute" << endl;
4296 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4297 out << " printf(\"*******************************\\n\");" << endl;
4298 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4299 out << " printf(\"*******************************\\n\");" << endl;
4300 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4301 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4302 out << " gSystem->ChangeDirectory(cdir);" << endl;
4303 out << " return kFALSE;" << endl;
4304 out << " }" << endl;
4305 out << " } else {" << endl;
4306 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4307 out << " gSystem->ChangeDirectory(cdir);" << endl;
4308 out << " return kFALSE;" << endl;
4309 out << " }" << endl;
4310 out << " // Check for SETUP.C and execute" << endl;
4311 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4312 out << " printf(\"*******************************\\n\");" << endl;
4313 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4314 out << " printf(\"*******************************\\n\");" << endl;
4315 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4316 out << " } else {" << endl;
4317 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4318 out << " gSystem->ChangeDirectory(cdir);" << endl;
4319 out << " return kFALSE;" << endl;
4320 out << " }" << endl;
4321 out << " // Restore original workdir" << endl;
4322 out << " gSystem->ChangeDirectory(cdir);" << endl;
4323 out << " return kTRUE;" << endl;
4324 out << "}" << endl;
4325 }
0f389141 4326 }
4327 Bool_t copy = kTRUE;
d3b18c4c 4328 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4329 if (copy) {
4330 CdWork();
4331 TString workdir = gGrid->GetHomeDirectory();
4332 workdir += fGridWorkingDir;
4333 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4334 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4335// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4336 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4337 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4338 }
4339}
4340
4341//______________________________________________________________________________
4342Bool_t AliAnalysisAlien::SetupPar(const char *package)
4343{
205b201f 4344// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4345// Note that for loading the compiled library. The current directory should have precedence in
4346// LD_LIBRARY_PATH
4347 TString pkgdir = package;
4348 pkgdir.ReplaceAll(".par","");
b50a41c3 4349 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4350 TString cdir = gSystem->WorkingDirectory();
4351 gSystem->ChangeDirectory(pkgdir);
4352 // Check for BUILD.sh and execute
4353 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4354 printf("**************************************************\n");
4355 printf("*** Building PAR archive %s\n", package);
4356 printf("**************************************************\n");
4357 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4358 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4359 gSystem->ChangeDirectory(cdir);
4360 return kFALSE;
4361 }
4362 } else {
4363 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4364 gSystem->ChangeDirectory(cdir);
4365 return kFALSE;
4366 }
4367 // Check for SETUP.C and execute
4368 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4369 printf("**************************************************\n");
4370 printf("*** Setup PAR archive %s\n", package);
4371 printf("**************************************************\n");
4372 gROOT->Macro("PROOF-INF/SETUP.C");
4373 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4374 } else {
4375 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4376 gSystem->ChangeDirectory(cdir);
4377 return kFALSE;
4378 }
4379 // Restore original workdir
4380 gSystem->ChangeDirectory(cdir);
4381 return kTRUE;
4382}
4383
c57f56b7 4384//______________________________________________________________________________
4385void AliAnalysisAlien::WriteExecutable()
4386{
4387// Generate the alien executable script.
4388 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4389 ofstream out;
4390 out.open(fExecutable.Data(), ios::out);
4391 if (out.bad()) {
5513444a 4392 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4393 return;
4394 }
4395 out << "#!/bin/bash" << endl;
2a4d5166 4396 // Make sure we can properly compile par files
4397 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4398 out << "echo \"=========================================\"" << endl;
4399 out << "echo \"############## PATH : ##############\"" << endl;
4400 out << "echo $PATH" << endl;
4401 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4402 out << "echo $LD_LIBRARY_PATH" << endl;
4403 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4404 out << "echo $ROOTSYS" << endl;
4405 out << "echo \"############## which root : ##############\"" << endl;
4406 out << "which root" << endl;
4407 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4408 out << "echo $ALICE_ROOT" << endl;
4409 out << "echo \"############## which aliroot : ##############\"" << endl;
4410 out << "which aliroot" << endl;
9c5ddadc 4411 out << "echo \"############## system limits : ##############\"" << endl;
4412 out << "ulimit -a" << endl;
4413 out << "echo \"############## memory : ##############\"" << endl;
4414 out << "free -m" << endl;
c57f56b7 4415 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4416 out << fExecutableCommand << " ";
370c9880 4417 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4418 out << "RET=$?" << endl;
4419 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4420 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4421 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4422 out << " let sig=\"$RET - 128\""<<endl;
4423 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4424 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4425 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4426 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4427 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4428 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4429 out << " fi"<<endl;
4430 out << " exit $RET"<< endl;
4431 out << "fi" << endl << endl ;
4432 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4433 out << "echo \"############## memory after: ##############\"" << endl;
4434 out << "free -m" << endl;
c57f56b7 4435 }
4436 Bool_t copy = kTRUE;
d3b18c4c 4437 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4438 if (copy) {
4439 CdWork();
4440 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4441 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4442 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4443 workdir += fGridWorkingDir;
4444 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4445 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4446 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4447// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4448 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4449 executable.Data())) Fatal("","Terminating");
c57f56b7 4450 }
4451}
4452
0f389141 4453//______________________________________________________________________________
4454void AliAnalysisAlien::WriteMergeExecutable()
4455{
4456// Generate the alien executable script for the merging job.
4457 if (!fMergeViaJDL) return;
4458 TString mergeExec = fExecutable;
4459 mergeExec.ReplaceAll(".sh", "_merge.sh");
4460 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4461 ofstream out;
4462 out.open(mergeExec.Data(), ios::out);
4463 if (out.bad()) {
4464 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4465 return;
4466 }
4467 out << "#!/bin/bash" << endl;
2a4d5166 4468 // Make sure we can properly compile par files
4469 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4470 out << "echo \"=========================================\"" << endl;
4471 out << "echo \"############## PATH : ##############\"" << endl;
4472 out << "echo $PATH" << endl;
4473 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4474 out << "echo $LD_LIBRARY_PATH" << endl;
4475 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4476 out << "echo $ROOTSYS" << endl;
4477 out << "echo \"############## which root : ##############\"" << endl;
4478 out << "which root" << endl;
4479 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4480 out << "echo $ALICE_ROOT" << endl;
4481 out << "echo \"############## which aliroot : ##############\"" << endl;
4482 out << "which aliroot" << endl;
4483 out << "echo \"############## system limits : ##############\"" << endl;
4484 out << "ulimit -a" << endl;
4485 out << "echo \"############## memory : ##############\"" << endl;
4486 out << "free -m" << endl;
4487 out << "echo \"=========================================\"" << endl << endl;
0f389141 4488 TString mergeMacro = fExecutable;
4489 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4490 if (IsOneStageMerging())
4491 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4492 else
f47d5cb4 4493 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4494 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4495 out << "RET=$?" << endl;
4496 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4497 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4498 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4499 out << " let sig=\"$RET - 128\""<<endl;
4500 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4501 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4502 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4503 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4504 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4505 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4506 out << " fi"<<endl;
4507 out << " exit $RET"<< endl;
4508 out << "fi" << endl << endl ;
0f389141 4509 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4510 out << "echo \"############## memory after: ##############\"" << endl;
4511 out << "free -m" << endl;
0f389141 4512 }
4513 Bool_t copy = kTRUE;
d3b18c4c 4514 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4515 if (copy) {
4516 CdWork();
4517 TString workdir = gGrid->GetHomeDirectory();
4518 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4519 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4520 workdir += fGridWorkingDir;
4521 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4522 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4523 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4524// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4525 if (!copyLocal2Alien("WriteMergeExecutable",
4526 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4527 }
4528}
4529
c57f56b7 4530//______________________________________________________________________________
5513444a 4531void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4532{
4533// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4534// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4535// Next lines: full_path_to_dataset XXX (XXX is a string)
4536// To submit, one has to: submit jdl XXX for all lines
4537 ofstream out;
4538 out.open(filename, ios::out);
4539 if (out.bad()) {
4540 Error("WriteProductionFile", "Bad file name: %s", filename);
4541 return;
4542 }
d3b18c4c 4543 TString workdir;
4544 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4545 workdir = gGrid->GetHomeDirectory();
5513444a 4546 workdir += fGridWorkingDir;
f5e8c702 4547 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4548 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4549 out << locjdl << " " << njobspermaster << endl;
5513444a 4550 Int_t nmasterjobs = fInputFiles->GetEntries();
4551 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4552 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4553 runOutDir.ReplaceAll(".xml", "");
4554 if (fOutputToRunNo)
4555 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4556 else
4557 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4558 }
d3b18c4c 4559 if (gGrid) {
4560 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4561 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4562// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4563 if (!copyLocal2Alien("WriteProductionFile", filename,
4564 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4565 }
5513444a 4566}
4567
4568//______________________________________________________________________________
0f389141 4569void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4570{
4571// Generate the alien validation script.
4572 // Generate the validation script
4573 TObjString *os;
d3b18c4c 4574 if (fValidationScript.IsNull()) {
4575 fValidationScript = fExecutable;
4576 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4577 }
4578 TString validationScript = fValidationScript;
4579 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4580 if (!Connect()) {
4581 Error("WriteValidationScript", "Alien connection required");
4582 return;
4583 }
d3b18c4c 4584 if (!fTerminateFiles.IsNull()) {
4585 fTerminateFiles.Strip();
4586 fTerminateFiles.ReplaceAll(" ",",");
4587 }
a2f5fc01 4588 TString outStream = "";
4589 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4590 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4591 ofstream out;
0d5d317c 4592 out.open(validationScript, ios::out);
c57f56b7 4593 out << "#!/bin/bash" << endl;
4594 out << "##################################################" << endl;
4595 out << "validateout=`dirname $0`" << endl;
4596 out << "validatetime=`date`" << endl;
4597 out << "validated=\"0\";" << endl;
4598 out << "error=0" << endl;
4599 out << "if [ -z $validateout ]" << endl;
4600 out << "then" << endl;
4601 out << " validateout=\".\"" << endl;
4602 out << "fi" << endl << endl;
4603 out << "cd $validateout;" << endl;
4604 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4605 out << "echo \"*******************************************************\"" << outStream << endl;
4606 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4607 out << "" << endl;
a2f5fc01 4608 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4609 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4610 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4611 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4612 out << "ls -la ./" << outStream << endl;
4613 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4614 out << "##################################################" << endl;
ebec370a 4615 out << "" << endl;
4616
4617 out << "if [ ! -f stderr ] ; then" << endl;
4618 out << " error=1" << endl;
a2f5fc01 4619 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4620 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4621 out << "fi" << endl;
4622
b34c9f51 4623 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4624 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4625 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4626 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4627 out << "" << endl;
4628
ebec370a 4629 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4630 out << " error=1" << endl;
a2f5fc01 4631 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4632 out << " echo \"$parArch\" " << outStream << endl;
4633 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4634 out << "fi" << endl;
4635
4636 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4637 out << " error=1" << endl;
a2f5fc01 4638 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4639 out << " echo \"$segViol\" " << outStream << endl;
4640 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4641 out << "fi" << endl;
4642
4643 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4644 out << " error=1" << endl;
a2f5fc01 4645 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4646 out << " echo \"$segFault\" " << outStream << endl;
4647 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4648 out << "fi" << endl;
4649
b34c9f51 4650 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4651 out << " error=1" << endl;
a2f5fc01 4652 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4653 out << " echo \"$glibcErr\" " << outStream << endl;
4654 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4655 out << "fi" << endl;
4656
ebec370a 4657 // Part dedicated to the specific analyses running into the train
4658
d3b18c4c 4659 TString outputFiles = fOutputFiles;
4660 if (merge && !fTerminateFiles.IsNull()) {
4661 outputFiles += ",";
4662 outputFiles += fTerminateFiles;
4663 }
4664 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4665 TIter next1(arr);
a2f5fc01 4666 TString outputFile;
bbd88db8 4667 while (!merge && (os=(TObjString*)next1())) {
4668 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4669 outputFile = os->GetString();
4670 Int_t index = outputFile.Index("@");
4671 if (index > 0) outputFile.Remove(index);
bbd88db8 4672 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4673 if (outputFile.Contains("*")) continue;
4674 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4675 out << " error=1" << endl;
7c2cd90a 4676 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4677 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4678 out << "fi" << endl;
4679 }
4680 delete arr;
7c2cd90a 4681 out << "if ! [ -f outputs_valid ] ; then" << endl;
4682 out << " error=1" << endl;
4683 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4684 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4685 out << "fi" << endl;
923e2ca5 4686
c57f56b7 4687 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4688 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4689 if (!IsKeepLogs()) {
4690 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4691 outStream = "";
149d288c 4692 out << " rm -f std*" << endl;
4693 }
c57f56b7 4694 out << "fi" << endl;
4695
a2f5fc01 4696 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4697 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4698 out << "cd -" << endl;
4699 out << "exit $error" << endl;
4700 }
4701 Bool_t copy = kTRUE;
d3b18c4c 4702 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4703 if (copy) {
4704 CdWork();
4705 TString workdir = gGrid->GetHomeDirectory();
4706 workdir += fGridWorkingDir;
d3b18c4c 4707 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4708 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 4709// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4710 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
4711 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 4712 }
4713}